forked from argotorg/fe
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_tokenizer.rs
More file actions
126 lines (112 loc) · 3.19 KB
/
Copy pathtest_tokenizer.rs
File metadata and controls
126 lines (112 loc) · 3.19 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
extern crate wasm_bindgen_test;
#[macro_use]
mod utils;
use serde::Serialize;
use wasm_bindgen_test::wasm_bindgen_test;
use fe_parser::string_utils::StringPositions;
use fe_parser::tokenizer::{
tokenize,
Token,
TokenType,
TokenizeError,
};
/// A python token object similar to those defined in python's stdlib `tokenize`
/// module.
#[derive(Serialize)]
struct PythonTokenInfo<'a> {
pub typ: TokenType,
pub string: &'a str,
pub start: (usize, usize),
pub end: (usize, usize),
pub line: &'a str,
}
impl<'a> PythonTokenInfo<'a> {
pub fn from_token_and_positions(
tok: &'a Token<'a>,
string_pos: &mut StringPositions<'_>,
) -> Self {
let start_pos = match string_pos.get_pos(tok.span.start) {
Some(pos) => pos,
None => string_pos.get_eof(),
};
let end_pos = match string_pos.get_pos(tok.span.end) {
Some(pos) => pos,
None => string_pos.get_eof(),
};
Self {
typ: tok.typ,
string: tok.string,
start: (start_pos.line, start_pos.col),
end: (end_pos.line, end_pos.col),
line: tok.line,
}
}
}
fn get_rust_token_json(input: &str) -> String {
let tokens = tokenize(input).unwrap();
let mut string_pos = StringPositions::new(input);
// Convert Fe tokens into python tokens
let python_tokens = tokens
.iter()
.map(|tok| PythonTokenInfo::from_token_and_positions(tok, &mut string_pos))
.collect::<Vec<PythonTokenInfo>>();
serde_json::to_string_pretty(&python_tokens).unwrap()
}
fn assert_fixture_is_valid(filename: &str, input: &str, expected_ser: &str) {
let actual_ser = get_rust_token_json(input);
assert_strings_eq!(
actual_ser,
expected_ser,
"\nTokenizations did not match for {}",
filename,
);
}
#[test]
#[wasm_bindgen_test]
fn test_tokenize_fixtures() {
do_with_fixtures!(
assert_fixture_is_valid,
"fixtures/tokenizer/basic.py.json",
"fixtures/tokenizer/triple_quote_strings.py.json",
"fixtures/tokenizer/single_quote_strings.py.json",
"fixtures/tokenizer/continued_statements.py.json",
"fixtures/tokenizer/validator_registration.v.py.json",
"fixtures/tokenizer/tokenize.py.json",
"fixtures/tokenizer/one_stmt_form_feed.v.py.json",
"fixtures/tokenizer/zero_length_pseudotoken.py.json",
);
}
#[test]
#[wasm_bindgen_test]
fn test_tokenize_errors() {
let examples = vec![
(
r#"
event Test:
field1: uint128
field2: uint128
"#,
Err(TokenizeError {
msg: "unindent does not match any outer indentation level",
offset: 36,
}),
),
(
r#"s = """"#,
Err(TokenizeError {
msg: "EOF in multi-line string",
offset: 7,
}),
),
(
"s = 3 + \\\n",
Err(TokenizeError {
msg: "EOF in multi-line statement",
offset: 10,
}),
),
];
for (input, expected) in examples {
assert_eq!(tokenize(input), expected);
}
}