forked from argotorg/fe
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtoken_stream.rs
More file actions
118 lines (100 loc) · 3.5 KB
/
Copy pathtoken_stream.rs
File metadata and controls
118 lines (100 loc) · 3.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
use std::collections::VecDeque;
use crate::SyntaxKind;
/// This trait works as an abstraction layer to encapsulate the differences
/// between input sources. There are mainly two types of input sources,
/// 1. text in source file
/// 2. tokens stream produced by procedural macros.
pub trait TokenStream {
type Token: LexicalToken;
/// Returns the next token in the stream.
fn next(&mut self) -> Option<Self::Token>;
/// Returns the next token in the stream without consuming it.
fn peek(&mut self) -> Option<&Self::Token>;
}
/// This trait represents a single token in the token stream.
pub trait LexicalToken: Clone {
/// Returns `SyntaxKind` of the token.
fn syntax_kind(&self) -> SyntaxKind;
/// Returns raw text of the token.
fn text(&self) -> &str;
}
/// This struct is a thin wrapper around `TokenStream` which allows the parser
/// to backtrack.
pub struct BackTrackableTokenStream<T: TokenStream> {
stream: T,
/// Backtrack buffer which stores tokens that have been already consumed.
bt_buffer: VecDeque<T::Token>,
bt_points: Vec<usize>,
/// Points to the current position of the backtrack buffer.
bt_cursor: Option<usize>,
}
impl<T: TokenStream> BackTrackableTokenStream<T> {
/// Creates a new `BackTrackableTokenStream` from the given `TokenStream`.
pub fn new(stream: T) -> Self {
Self {
stream,
bt_buffer: VecDeque::new(),
bt_points: Vec::new(),
bt_cursor: None,
}
}
/// Returns the next token in the stream.
#[allow(clippy::should_implement_trait)]
pub fn next(&mut self) -> Option<T::Token> {
if !self.has_parent() {
if let Some(bt_buffer) = self.bt_buffer.pop_front() {
return Some(bt_buffer);
} else {
return self.stream.next();
}
}
if let Some(cursor) = self.bt_cursor {
if cursor < self.bt_buffer.len() {
let token = self.bt_buffer.get(cursor).cloned();
self.bt_cursor = Some(cursor + 1);
return token;
} else {
self.bt_cursor = Some(cursor + 1);
}
}
let token = self.stream.next()?;
if self.has_parent() {
self.bt_buffer.push_back(token.clone());
}
Some(token)
}
/// Returns the next token in the stream without consuming it.
pub fn peek(&mut self) -> Option<&T::Token> {
if let Some(cursor) = self.bt_cursor
&& cursor < self.bt_buffer.len()
{
return self.bt_buffer.get(cursor);
}
self.stream.peek()
}
/// Set a backtrack point which allows the parser to backtrack to this
/// point.
pub fn set_bt_point(&mut self) {
if self.has_parent() {
self.bt_points.push(self.bt_cursor.unwrap());
} else {
self.bt_points.push(0);
self.bt_cursor = Some(0);
}
}
/// Backtracks to the last backtrack point.
///
/// # Panics
/// Panics if the `set_bt_point` method has not been called before.
pub fn backtrack(&mut self) {
debug_assert!(self.has_bt_point(), "backtrack without `bt_point`");
self.bt_cursor = Some(self.bt_points.pop().unwrap());
}
/// Returns `true` if the stream has a backtrack point.
pub fn has_bt_point(&mut self) -> bool {
!self.bt_points.is_empty()
}
pub fn has_parent(&mut self) -> bool {
!self.bt_points.is_empty()
}
}