This repository was archived by the owner on Jul 27, 2023. It is now read-only.
Repository navigation
Expand file tree
/
Copy pathsoft_keywords.rs
More file actions
119 lines (111 loc) · 4.39 KB
/
Copy pathsoft_keywords.rs
File metadata and controls
119 lines (111 loc) · 4.39 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
use crate::{lexer::LexResult, token::Tok, Mode};
use itertools::{Itertools, MultiPeek};
/// An [`Iterator`] that transforms a token stream to accommodate soft keywords (namely, `match`
/// and `case`).
///
/// [PEP 634](https://www.python.org/dev/peps/pep-0634/) introduced the `match` and `case` keywords
/// as soft keywords, meaning that they can be used as identifiers (e.g., variable names) in certain
/// contexts.
///
/// This function modifies a token stream to accommodate this change. In particular, it replaces
/// `match` and `case` tokens with `identifier` tokens if they are used as identifiers.
///
/// Handling soft keywords in this intermediary pass allows us to simplify both the lexer and
/// parser, as neither of them need to be aware of soft keywords.
pub struct SoftKeywordTransformer<I>
where
I: Iterator<Item = LexResult>,
{
underlying: MultiPeek<I>,
start_of_line: bool,
}
impl<I> SoftKeywordTransformer<I>
where
I: Iterator<Item = LexResult>,
{
pub fn new(lexer: I, mode: Mode) -> Self {
Self {
underlying: lexer.multipeek(), // spell-checker:ignore multipeek
start_of_line: matches!(mode, Mode::Interactive | Mode::Module),
}
}
}
impl<I> Iterator for SoftKeywordTransformer<I>
where
I: Iterator<Item = LexResult>,
{
type Item = LexResult;
#[inline]
fn next(&mut self) -> Option<LexResult> {
let mut next = self.underlying.next();
if let Some(Ok((tok, range))) = next.as_ref() {
// If the token is a `match` or `case` token, check if it's used as an identifier.
// We assume every `match` or `case` is an identifier unless both of the following
// conditions are met:
// 1. The token is at the start of a logical line.
// 2. The logical line contains a top-level colon (that is, a colon that is not nested
// inside a parenthesized expression, list, or dictionary).
// 3. The top-level colon is not the immediate sibling of a `match` or `case` token.
// (This is to avoid treating `match` and `case` as identifiers when annotated with
// type hints.)
if matches!(tok, Tok::Match | Tok::Case) {
if !self.start_of_line {
next = Some(Ok((soft_to_name(tok), *range)));
} else {
let mut nesting = 0;
let mut first = true;
let mut seen_colon = false;
let mut seen_lambda = false;
while let Some(Ok((tok, _))) = self.underlying.peek() {
match tok {
Tok::Newline => break,
Tok::Lambda if nesting == 0 => seen_lambda = true,
Tok::Colon if nesting == 0 => {
if seen_lambda {
seen_lambda = false;
} else if !first {
seen_colon = true;
}
}
Tok::Lpar | Tok::Lsqb | Tok::Lbrace => nesting += 1,
Tok::Rpar | Tok::Rsqb | Tok::Rbrace => nesting -= 1,
_ => {}
}
first = false;
}
if !seen_colon {
next = Some(Ok((soft_to_name(tok), *range)));
}
}
}
}
self.start_of_line = next.as_ref().map_or(false, |lex_result| {
lex_result.as_ref().map_or(false, |(tok, _)| {
#[cfg(feature = "full-lexer")]
if matches!(tok, Tok::NonLogicalNewline | Tok::Comment { .. }) {
return self.start_of_line;
}
matches!(
tok,
Tok::StartModule
| Tok::StartInteractive
| Tok::Newline
| Tok::Indent
| Tok::Dedent
)
})
});
next
}
}
#[inline]
fn soft_to_name(tok: &Tok) -> Tok {
let name = match tok {
Tok::Match => "match",
Tok::Case => "case",
_ => unreachable!("other tokens never reach here"),
};
Tok::Name {
name: name.to_owned(),
}
}