Keep `UnsafeLit` as a `Lit` variant

Per our language specification, unsafe literals are by their inherent
name literals, hence keep them under this variant. Also, it simplifies
a bunch of implementations. Although I'm sure we can do something for
performance to simply provide an error case when we encounter a
disallowed literal.
next
Sayan Nandan 2 years ago
parent 9c28c97830
commit 2d1fdd3417
No known key found for this signature in database
GPG Key ID: 8BC07A0A4D41DD52

@ -33,6 +33,10 @@ use {
}, },
}; };
/*
Impls for insert
*/
/// Parse a list /// Parse a list
/// ///
/// **NOTE:** This function will error if the `[` token is passed. Make sure this is forwarded by the caller /// **NOTE:** This function will error if the `[` token is passed. Make sure this is forwarded by the caller

@ -38,7 +38,6 @@ use {
pub enum Token { pub enum Token {
Symbol(Symbol), Symbol(Symbol),
Keyword(Keyword), Keyword(Keyword),
UnsafeLit(RawSlice),
Ident(RawSlice), Ident(RawSlice),
#[cfg(test)] #[cfg(test)]
/// A comma that can be ignored (used for fuzzing) /// A comma that can be ignored (used for fuzzing)
@ -76,6 +75,7 @@ pub enum Lit {
Str(Box<str>), Str(Box<str>),
Bool(bool), Bool(bool),
Num(u64), Num(u64),
UnsafeLit(RawSlice),
} }
enum_impls! { enum_impls! {
@ -599,7 +599,7 @@ impl<'a> Lexer<'a> {
okay &= self.remaining() >= size; okay &= self.remaining() >= size;
if compiler::likely(okay) { if compiler::likely(okay) {
unsafe { unsafe {
self.push_token(Token::UnsafeLit(RawSlice::new(self.cursor(), size))); self.push_token(Lit::UnsafeLit(RawSlice::new(self.cursor(), size)));
self.incr_cursor_by(size); self.incr_cursor_by(size);
} }
} else { } else {

@ -146,19 +146,22 @@ mod lexer_tests {
fn lex_unsafe_literal_mini() { fn lex_unsafe_literal_mini() {
let usl = lex("\r0\n".as_bytes()).unwrap(); let usl = lex("\r0\n".as_bytes()).unwrap();
assert_eq!(usl.len(), 1); assert_eq!(usl.len(), 1);
assert_eq!(Token::UnsafeLit("".into()), usl[0]); assert_eq!(Token::Lit(Lit::UnsafeLit("".into())), usl[0]);
} }
#[test] #[test]
fn lex_unsafe_literal() { fn lex_unsafe_literal() {
let usl = lex("\r9\nabcdefghi".as_bytes()).unwrap(); let usl = lex("\r9\nabcdefghi".as_bytes()).unwrap();
assert_eq!(usl.len(), 1); assert_eq!(usl.len(), 1);
assert_eq!(Token::UnsafeLit("abcdefghi".into()), usl[0]); assert_eq!(Token::Lit(Lit::UnsafeLit("abcdefghi".into())), usl[0]);
} }
#[test] #[test]
fn lex_unsafe_literal_pro() { fn lex_unsafe_literal_pro() {
let usl = lex("\r18\nabcdefghi123456789".as_bytes()).unwrap(); let usl = lex("\r18\nabcdefghi123456789".as_bytes()).unwrap();
assert_eq!(usl.len(), 1); assert_eq!(usl.len(), 1);
assert_eq!(Token::UnsafeLit("abcdefghi123456789".into()), usl[0]); assert_eq!(
Token::Lit(Lit::UnsafeLit("abcdefghi123456789".into())),
usl[0]
);
} }
} }

Loading…
Cancel
Save