Keep `UnsafeLit` as a `Lit` variant

Per our language specification, unsafe literals are by their inherent
name literals, hence keep them under this variant. Also, it simplifies
a bunch of implementations. Although I'm sure we can do something for
performance to simply provide an error case when we encounter a
disallowed literal.
next
Sayan Nandan 2 years ago
parent 9c28c97830
commit 2d1fdd3417
No known key found for this signature in database
GPG Key ID: 8BC07A0A4D41DD52

@ -33,6 +33,10 @@ use {
},
};
/*
Impls for insert
*/
/// Parse a list
///
/// **NOTE:** This function will error if the `[` token is passed. Make sure this is forwarded by the caller

@ -38,7 +38,6 @@ use {
pub enum Token {
Symbol(Symbol),
Keyword(Keyword),
UnsafeLit(RawSlice),
Ident(RawSlice),
#[cfg(test)]
/// A comma that can be ignored (used for fuzzing)
@ -76,6 +75,7 @@ pub enum Lit {
Str(Box<str>),
Bool(bool),
Num(u64),
UnsafeLit(RawSlice),
}
enum_impls! {
@ -599,7 +599,7 @@ impl<'a> Lexer<'a> {
okay &= self.remaining() >= size;
if compiler::likely(okay) {
unsafe {
self.push_token(Token::UnsafeLit(RawSlice::new(self.cursor(), size)));
self.push_token(Lit::UnsafeLit(RawSlice::new(self.cursor(), size)));
self.incr_cursor_by(size);
}
} else {

@ -146,19 +146,22 @@ mod lexer_tests {
fn lex_unsafe_literal_mini() {
let usl = lex("\r0\n".as_bytes()).unwrap();
assert_eq!(usl.len(), 1);
assert_eq!(Token::UnsafeLit("".into()), usl[0]);
assert_eq!(Token::Lit(Lit::UnsafeLit("".into())), usl[0]);
}
#[test]
fn lex_unsafe_literal() {
let usl = lex("\r9\nabcdefghi".as_bytes()).unwrap();
assert_eq!(usl.len(), 1);
assert_eq!(Token::UnsafeLit("abcdefghi".into()), usl[0]);
assert_eq!(Token::Lit(Lit::UnsafeLit("abcdefghi".into())), usl[0]);
}
#[test]
fn lex_unsafe_literal_pro() {
let usl = lex("\r18\nabcdefghi123456789".as_bytes()).unwrap();
assert_eq!(usl.len(), 1);
assert_eq!(Token::UnsafeLit("abcdefghi123456789".into()), usl[0]);
assert_eq!(
Token::Lit(Lit::UnsafeLit("abcdefghi123456789".into())),
usl[0]
);
}
}

Loading…
Cancel
Save