forked from argotorg/fe
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtypes.rs
More file actions
59 lines (48 loc) · 1.12 KB
/
Copy pathtypes.rs
File metadata and controls
59 lines (48 loc) · 1.12 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
use serde::{
Deserialize,
Serialize,
};
use crate::span::{
Span,
Spanned,
};
/// Indicates the basic syntactic element represented by a token.
#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone)]
pub enum TokenType {
NAME,
NUMBER,
STRING,
OP,
COMMENT,
INDENT,
DEDENT,
NEWLINE, // Grammatically significant newlines
NL, // Whitespace newlines
ENDMARKER,
ERRORTOKEN,
}
/// A token parsed from a source string.
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Copy)]
pub struct Token<'a> {
/// The type of a token.
pub typ: TokenType,
/// The text content of a parsed token.
pub string: &'a str,
/// The span of source text covered by a token.
pub span: Span,
/// The text content of the line from which a token was parsed.
pub line: &'a str,
}
impl<'a> From<&Token<'a>> for Span {
fn from(tok: &Token) -> Self {
tok.span
}
}
impl<'a> From<&Token<'a>> for Spanned<&'a str> {
fn from(tok: &Token<'a>) -> Self {
Spanned {
node: tok.string,
span: tok.span,
}
}
}