forked from argotorg/fe
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathlexer.rs
More file actions
98 lines (84 loc) · 2.26 KB
/
Copy pathlexer.rs
File metadata and controls
98 lines (84 loc) · 2.26 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
mod token;
use crate::node::Span;
use logos::Logos;
pub use token::{Token, TokenKind};
#[derive(Clone)]
pub struct Lexer<'a> {
inner: logos::Lexer<'a, TokenKind>,
}
impl<'a> Lexer<'a> {
/// Create a new lexer with the given source code string.
pub fn new(src: &'a str) -> Lexer {
Lexer {
inner: TokenKind::lexer(src),
}
}
/// Return the full source code string that's being tokenized.
pub fn source(&self) -> &'a str {
self.inner.source()
}
}
impl<'a> Iterator for Lexer<'a> {
type Item = Token<'a>;
fn next(&mut self) -> Option<Self::Item> {
let kind = self.inner.next()?;
let text = self.inner.slice();
let span = self.inner.span();
Some(Token {
kind,
text,
span: Span {
start: span.start,
end: span.end,
},
})
}
}
#[cfg(test)]
mod tests {
use crate::lexer::{Lexer, TokenKind};
use TokenKind::*;
fn check(input: &str, expected: &[TokenKind]) {
let lex = Lexer::new(input);
let actual = lex.map(|t| t.kind).collect::<Vec<_>>();
assert!(
actual.iter().eq(expected.iter()),
"\nexpected: {:?}\n actual: {:?}",
expected,
actual
);
}
#[test]
fn basic() {
check(
"contract Foo:\n x: u32\n def f() -> u32: not x",
&[
Contract, Name, Colon, Newline, Name, Colon, Name, Newline, Def, Name, ParenOpen,
ParenClose, Arrow, Name, Colon, Not, Name,
],
);
}
#[test]
fn strings() {
let rawstr = r#""string \t with \n escapes \" \"""#;
let mut lex = Lexer::new(rawstr);
let lexedstr = lex.next().unwrap();
assert!(lexedstr.kind == Text);
assert!(lexedstr.text == rawstr);
assert!(lex.next() == None);
}
#[test]
fn errors() {
check(
"contract Foo@ 5u8 \n self.bar",
&[Contract, Name, Error, Int, Name, Newline, Name, Dot, Name],
);
}
#[test]
fn tabs_and_comment() {
check(
"\n\t \tcontract\n\tFoo # hi mom!\n ",
&[Newline, Contract, Newline, Name, Newline],
);
}
}