Use `State` for `CREATE` and `ALTER` statements

server/src/engine/ql/ast.rs

@@ -202,6 +202,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
             & (self.t[idx_c] == Token![() close])
             & rem
     }
+    #[inline(always)]
     /// Reads a lit using the given token and the internal data source and return a data type
     ///
     /// ## Safety
@@ -211,6 +212,24 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     pub unsafe fn read_lit_into_data_type_unchecked_from(&mut self, tok: &'a Token) -> DataType {
         self.d.read_data_type(tok)
     }
+    #[inline(always)]
+    /// Loop condition for tt and non-poisoned state only
+    pub fn loop_tt(&self) -> bool {
+        self.not_exhausted() && self.okay()
+    }
+    #[inline(always)]
+    /// Loop condition for tt and non-poisoned state only
+    pub fn loop_data_tt(&self) -> bool {
+        self.not_exhausted() && self.okay() && self.d.nonzero()
+    }
+    #[inline(always)]
+    pub(crate) fn consumed(&self) -> usize {
+        self.t.len() - self.remaining()
+    }
+    #[inline(always)]
+    pub(crate) fn cursor(&self) -> usize {
+        self.i
+    }
 }
 
 pub trait QueryData<'a> {
@@ -226,6 +245,8 @@ pub trait QueryData<'a> {
     /// ## Safety
     /// The current token must match the signature of a lit
     unsafe fn read_data_type(&mut self, tok: &'a Token) -> DataType;
+    /// Returns true if the data source has enough data
+    fn nonzero(&self) -> bool;
 }
 
 #[derive(Debug)]
@@ -250,6 +271,10 @@ impl<'a> QueryData<'a> for InplaceData {
     unsafe fn read_data_type(&mut self, tok: &'a Token) -> DataType {
         DataType::clone_from_lit(extract!(tok, Token::Lit(ref l) => l))
     }
+    #[inline(always)]
+    fn nonzero(&self) -> bool {
+        true
+    }
 }
 
 #[derive(Debug)]
@@ -266,7 +291,7 @@ impl<'a> SubstitutedData<'a> {
 impl<'a> QueryData<'a> for SubstitutedData<'a> {
     #[inline(always)]
     fn can_read_lit_from(&self, tok: &Token) -> bool {
-        Token![?].eq(tok) && !self.data.is_empty()
+        Token![?].eq(tok) && self.nonzero()
     }
     #[inline(always)]
     unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a> {
@@ -282,6 +307,10 @@ impl<'a> QueryData<'a> for SubstitutedData<'a> {
         self.data = &self.data[1..];
         DataType::clone_from_litir(ret)
     }
+    #[inline(always)]
+    fn nonzero(&self) -> bool {
+        !self.data.is_empty()
+    }
 }
 
 /*
@@ -443,45 +472,41 @@ pub enum Statement<'a> {
     Delete(dml::delete::DeleteStatement<'a>),
 }
 
-pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], mut d: Qd) -> LangResult<Statement<'a>> {
+pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], d: Qd) -> LangResult<Statement<'a>> {
     let mut i = 0;
-    let ref mut qd = d;
     if compiler::unlikely(tok.len() < 2) {
         return Err(LangError::UnexpectedEndofStatement);
     }
     match tok[0] {
         // DDL
         Token![use] => Entity::parse_from_tokens(&tok[1..], &mut i).map(Statement::Use),
-        Token![create] => match tok[1] {
-            Token![model] => schema::parse_schema_from_tokens(&tok[2..], qd).map(|(q, c)| {
-                i += c;
-                Statement::CreateModel(q)
-            }),
-            Token![space] => schema::parse_space_from_tokens(&tok[2..], qd).map(|(q, c)| {
-                i += c;
-                Statement::CreateSpace(q)
-            }),
-            _ => compiler::cold_rerr(LangError::UnknownCreateStatement),
-        },
         Token![drop] if tok.len() >= 3 => ddl::parse_drop(&tok[1..], &mut i),
-        Token![alter] => match tok[1] {
-            Token![model] => schema::parse_alter_kind_from_tokens(&tok[2..], qd, &mut i)
-                .map(Statement::AlterModel),
-            Token![space] => {
-                schema::parse_alter_space_from_tokens(&tok[2..], qd).map(|(q, incr)| {
-                    i += incr;
-                    Statement::AlterSpace(q)
-                })
-            }
-            _ => compiler::cold_rerr(LangError::UnknownAlterStatement),
-        },
         Token::Ident(id) if id.eq_ignore_ascii_case(b"inspect") => {
             ddl::parse_inspect(&tok[1..], &mut i)
         }
-        // DML
         ref stmt => {
             let mut state = State::new(&tok[1..], d);
             match stmt {
+                // DDL
+                Token![create] => {
+                    match tok[1] {
+                        Token![model] => schema::parse_model_from_tokens(&mut state)
+                            .map(Statement::CreateModel),
+                        Token![space] => schema::parse_space_from_tokens(&mut state)
+                            .map(Statement::CreateSpace),
+                        _ => compiler::cold_rerr(LangError::UnknownCreateStatement),
+                    }
+                }
+                Token![alter] => match tok[1] {
+                    Token![model] => {
+                        schema::parse_alter_kind_from_tokens(&mut state).map(Statement::AlterModel)
+                    }
+                    Token![space] => {
+                        schema::parse_alter_space_from_tokens(&mut state).map(Statement::AlterSpace)
+                    }
+                    _ => compiler::cold_rerr(LangError::UnknownAlterStatement),
+                },
+                // DML
                 Token![insert] => {
                     dml::insert::InsertStatement::parse_insert(&mut state).map(Statement::Insert)
                 }

server/src/engine/ql/lexer.rs

@@ -81,6 +81,7 @@ enum_impls! {
 
 #[derive(Debug, PartialEq, Clone)]
 #[repr(u8)]
+/// A [`Lit`] as represented by an insecure token stream
 pub enum Lit<'a> {
     Str(Box<str>),
     Bool(bool),

server/src/engine/ql/tests/schema_tests.rs

@@ -119,30 +119,30 @@ mod tymeta {
     #[test]
     fn tymeta_mini() {
         let tok = lex_insecure(b"}").unwrap();
-        let (res, ret) = schema::fold_tymeta(&tok);
-        assert!(res.is_okay());
-        assert!(!res.has_more());
-        assert_eq!(res.pos(), 1);
-        assert_eq!(ret, nullable_dict!());
+        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
+        assert!(okay);
+        assert!(!tymeta.has_more());
+        assert_eq!(cursor, 1);
+        assert_eq!(data, nullable_dict!());
     }
     #[test]
     fn tymeta_mini_fail() {
         let tok = lex_insecure(b",}").unwrap();
-        let (res, ret) = schema::fold_tymeta(&tok);
-        assert!(!res.is_okay());
-        assert!(!res.has_more());
-        assert_eq!(res.pos(), 0);
-        assert_eq!(ret, nullable_dict!());
+        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
+        assert!(!okay);
+        assert!(!tymeta.has_more());
+        assert_eq!(cursor, 0);
+        assert_eq!(data, nullable_dict!());
     }
     #[test]
     fn tymeta() {
         let tok = lex_insecure(br#"hello: "world", loading: true, size: 100 }"#).unwrap();
-        let (res, ret) = schema::fold_tymeta(&tok);
-        assert!(res.is_okay());
-        assert!(!res.has_more());
-        assert_eq!(res.pos(), tok.len());
+        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
+        assert!(okay);
+        assert!(!tymeta.has_more());
+        assert_eq!(cursor, tok.len());
         assert_eq!(
-            ret,
+            data,
             nullable_dict! {
                 "hello" => Lit::Str("world".into()),
                 "loading" => Lit::Bool(true),
@@ -155,17 +155,17 @@ mod tymeta {
         // list { maxlen: 100, type string, unique: true }
         //        ^^^^^^^^^^^^^^^^^^ cursor should be at string
         let tok = lex_insecure(br#"maxlen: 100, type string, unique: true }"#).unwrap();
-        let (res1, ret1) = schema::fold_tymeta(&tok);
-        assert!(res1.is_okay());
-        assert!(res1.has_more());
-        assert_eq!(res1.pos(), 5);
-        let remslice = &tok[res1.pos() + 2..];
-        let (res2, ret2) = schema::fold_tymeta(remslice);
-        assert!(res2.is_okay());
-        assert!(!res2.has_more());
-        assert_eq!(res2.pos() + res1.pos() + 2, tok.len());
-        let mut final_ret = ret1;
-        final_ret.extend(ret2);
+        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
+        assert!(okay);
+        assert!(tymeta.has_more());
+        assert_eq!(cursor, 5);
+        let remslice = &tok[cursor + 2..];
+        let (tymeta2, okay2, cursor2, data2) = schema::fold_tymeta(remslice);
+        assert!(okay2);
+        assert!(!tymeta2.has_more());
+        assert_eq!(cursor2 + cursor + 2, tok.len());
+        let mut final_ret = data;
+        final_ret.extend(data2);
         assert_eq!(
             final_ret,
             nullable_dict! {
@@ -181,17 +181,17 @@ mod tymeta {
         let tok =
             lex_insecure(br#"maxlen: 100, this: { is: "cool" }, type string, unique: true }"#)
                 .unwrap();
-        let (res1, ret1) = schema::fold_tymeta(&tok);
-        assert!(res1.is_okay());
-        assert!(res1.has_more());
-        assert_eq!(res1.pos(), 13);
-        let remslice = &tok[res1.pos() + 2..];
-        let (res2, ret2) = schema::fold_tymeta(remslice);
-        assert!(res2.is_okay());
-        assert!(!res2.has_more());
-        assert_eq!(res2.pos() + res1.pos() + 2, tok.len());
-        let mut final_ret = ret1;
-        final_ret.extend(ret2);
+        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
+        assert!(okay);
+        assert!(tymeta.has_more());
+        assert_eq!(cursor, 13);
+        let remslice = &tok[cursor + 2..];
+        let (tymeta2, okay2, cursor2, data2) = schema::fold_tymeta(remslice);
+        assert!(okay2);
+        assert!(!tymeta2.has_more());
+        assert_eq!(cursor2 + cursor + 2, tok.len());
+        let mut final_ret = data;
+        final_ret.extend(data2);
         assert_eq!(
             final_ret,
             nullable_dict! {
@@ -228,13 +228,13 @@ mod tymeta {
             "users" => Lit::Str("sayan".into())
         };
         fuzz_tokens(&tok, |should_pass, new_src| {
-            let (ret, dict) = schema::fold_tymeta(&new_src);
+            let (tymeta, okay, cursor, data) = schema::fold_tymeta(&new_src);
             if should_pass {
-                assert!(ret.is_okay(), "{:?}", &new_src);
-                assert!(!ret.has_more());
-                assert_eq!(ret.pos(), new_src.len());
-                assert_eq!(dict, expected);
-            } else if ret.is_okay() {
+                assert!(okay, "{:?}", &new_src);
+                assert!(!tymeta.has_more());
+                assert_eq!(cursor, new_src.len());
+                assert_eq!(data, expected);
+            } else if okay {
                 panic!(
                     "Expected failure but passed for token stream: `{:?}`",
                     new_src
@@ -267,13 +267,13 @@ mod tymeta {
             },
         };
         fuzz_tokens(&tok, |should_pass, new_src| {
-            let (ret, dict) = schema::fold_tymeta(&new_src);
+            let (tymeta, okay, cursor, data) = schema::fold_tymeta(&new_src);
             if should_pass {
-                assert!(ret.is_okay());
-                assert!(ret.has_more());
-                assert!(new_src[ret.pos()] == Token::Ident(b"string"));
-                assert_eq!(dict, expected);
-            } else if ret.is_okay() {
+                assert!(okay);
+                assert!(tymeta.has_more());
+                assert!(new_src[cursor] == Token::Ident(b"string"));
+                assert_eq!(data, expected);
+            } else if okay {
                 panic!("Expected failure but passed for token stream: `{:?}`", tok);
             }
         });
@@ -895,7 +895,7 @@ mod alter_model_remove {
     fn alter_mini() {
         let tok = lex_insecure(b"alter model mymodel remove myfield").unwrap();
         let mut i = 4;
-        let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+        let remove = schema::alter_remove_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(remove, [b"myfield".as_slice()].into());
     }
@@ -903,7 +903,7 @@ mod alter_model_remove {
     fn alter_mini_2() {
         let tok = lex_insecure(b"alter model mymodel remove (myfield)").unwrap();
         let mut i = 4;
-        let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+        let remove = schema::alter_remove_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(remove, [b"myfield".as_slice()].into());
     }
@@ -913,7 +913,7 @@ mod alter_model_remove {
             lex_insecure(b"alter model mymodel remove (myfield1, myfield2, myfield3, myfield4)")
                 .unwrap();
         let mut i = 4;
-        let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+        let remove = schema::alter_remove_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             remove,