fn compare_strict(

&self,

other: &IndentationLevel,

location: Location,

) -> Result<Ordering, LexicalError> {

// We only know for sure that we're smaller or bigger if tabs

// and spaces both differ in the same direction. Otherwise we're

// dependent on the size of tabs.

match self.tabs.cmp(&other.tabs) {

Ordering::Less => {

if self.spaces <= other.spaces {

Ok(Ordering::Less)

} else {

Err(LexicalError {

location,

error: LexicalErrorType::TabError,

})

}

}

Ordering::Greater => {

if self.spaces >= other.spaces {

Ok(Ordering::Greater)

} else {

Err(LexicalError {

location,

error: LexicalErrorType::TabError,

})

}

}

Ordering::Equal => Ok(self.spaces.cmp(&other.spaces)),

}

}

fn is_empty(&self) -> bool {

self.indent_stack.len() == 1

}

fn push(&mut self, indent: IndentationLevel) {

self.indent_stack.push(indent);

}

fn pop(&mut self) -> Option<IndentationLevel> {

if self.is_empty() {

return None;

}

self.indent_stack.pop()

}

fn current(&self) -> &IndentationLevel {

self.indent_stack

.last()

.expect("Indentations must have at least one level")

}

fn default() -> Self {

Self {

indent_stack: vec![IndentationLevel::default()],

}

}

T: Iterator<Item = char>,

fn new(source: T) -> Self {

Self {

source,

window: [None; N],

}

}

fn slide(&mut self) -> Option<char> {

self.window.rotate_left(1);

let next = self.source.next();

*self.window.last_mut().expect("never empty") = next;

next

}

T: Iterator<Item = char>,

Idx: SliceIndex<[Option<char>]>,

type Output = Idx::Output;

fn index(&self, index: Idx) -> &Self::Output {

&self.window[index]

}

// Contains the source code to be lexed.

window: CharWindow<T, 3>,

// Are we at the beginning of a line?

at_begin_of_line: bool,

// Amount of parenthesis.

nesting: usize,

// Indentation levels.

indentations: Indentations,

// Pending list of tokens to be returned.

pending: Vec<Spanned>,

// The current location.

location: Location,

source: &str,

mode: Mode,

start_location: Location,

) -> impl Iterator<Item = LexResult> + '_ {

SoftKeywordTransformer::new(Lexer::new(source.chars(), start_location), mode)

T: Iterator<Item = char>,

/// Create a new lexer from T and a starting location. You probably want to use

/// [`lex`] instead.

pub fn new(input: T, start: Location) -> Self {

let mut lxr = Lexer {

at_begin_of_line: true,

nesting: 0,

indentations: Indentations::default(),

// Usually we have less than 5 tokens pending.

pending: Vec::with_capacity(5),

location: start,

window: CharWindow::new(input),

};

// Fill the window.

lxr.window.slide();

lxr.window.slide();

lxr.window.slide();

// TODO: Handle possible mismatch between BOM and explicit encoding declaration.

// spell-checker:ignore feff

if let Some('\u{feff}') = lxr.window[0] {

lxr.window.slide();

}

lxr

}

/// Lex an identifier. Also used for keywords and string/bytes literals with a prefix.

fn lex_identifier(&mut self) -> LexResult {

// Detect potential string like rb'' b'' f'' u'' r''

match self.window[..3] {

[Some(c), Some('"' | '\''), ..] => {

if let Ok(kind) = StringKind::try_from(c) {

return self.lex_string(kind);

}

}

[Some(c1), Some(c2), Some('"' | '\'')] => {

if let Ok(kind) = StringKind::try_from([c1, c2]) {

return self.lex_string(kind);

}

}

_ => {}

};

let start_pos = self.get_pos();

let mut name = String::with_capacity(8);

while self.is_identifier_continuation() {

name.push(self.next_char().unwrap());

}

let end_pos = self.get_pos();

if let Some(tok) = KEYWORDS.get(&name) {

Ok((start_pos, tok.clone(), end_pos))

} else {

Ok((start_pos, Tok::Name { name }, end_pos))

}

}

/// Numeric lexing. The feast can start!

fn lex_number(&mut self) -> LexResult {

let start_pos = self.get_pos();

match self.window[..2] {

[Some('0'), Some('x' | 'X')] => {

// Hex! (0xdeadbeef)

self.next_char();

self.next_char();

self.lex_number_radix(start_pos, 16)

}

[Some('0'), Some('o' | 'O')] => {

// Octal style! (0o377)

self.next_char();

self.next_char();

self.lex_number_radix(start_pos, 8)

}

[Some('0'), Some('b' | 'B')] => {

// Binary! (0b_1110_0101)

self.next_char();

self.next_char();

self.lex_number_radix(start_pos, 2)

}

_ => self.lex_normal_number(),

}

}

/// Lex a hex/octal/decimal/binary number without a decimal point.

fn lex_number_radix(&mut self, start_pos: Location, radix: u32) -> LexResult {

let value_text = self.radix_run(radix);

let end_pos = self.get_pos();

let value = BigInt::from_str_radix(&value_text, radix).map_err(|e| LexicalError {

error: LexicalErrorType::OtherError(format!("{e:?}")),

location: start_pos,

})?;

Ok((start_pos, Tok::Int { value }, end_pos))

}

/// Lex a normal number, that is, no octal, hex or binary number.

fn lex_normal_number(&mut self) -> LexResult {

let start_pos = self.get_pos();

let start_is_zero = self.window[0] == Some('0');

// Normal number:

let mut value_text = self.radix_run(10);

// If float:

if self.window[0] == Some('.') || self.at_exponent() {

// Take '.':

if self.window[0] == Some('.') {

if self.window[1] == Some('_') {

return Err(LexicalError {

error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),

location: self.get_pos(),

});

}

value_text.push(self.next_char().unwrap());

value_text.push_str(&self.radix_run(10));

}

// 1e6 for example:

if let Some('e' | 'E') = self.window[0] {

if self.window[1] == Some('_') {

return Err(LexicalError {

error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),

location: self.get_pos(),

});

}

value_text.push(self.next_char().unwrap().to_ascii_lowercase());

// Optional +/-

if matches!(self.window[0], Some('-' | '+')) {

if self.window[1] == Some('_') {

return Err(LexicalError {

error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),

location: self.get_pos(),

});

}

value_text.push(self.next_char().unwrap());

}

value_text.push_str(&self.radix_run(10));

}

let value = f64::from_str(&value_text).map_err(|_| LexicalError {

error: LexicalErrorType::OtherError("Invalid decimal literal".to_owned()),

location: self.get_pos(),

})?;

// Parse trailing 'j':

if matches!(self.window[0], Some('j' | 'J')) {

self.next_char();

let end_pos = self.get_pos();

Ok((

start_pos,

Tok::Complex {

real: 0.0,

imag: value,

},

end_pos,

))

} else {

let end_pos = self.get_pos();

Ok((start_pos, Tok::Float { value }, end_pos))

}

} else {

// Parse trailing 'j':

if matches!(self.window[0], Some('j' | 'J')) {

self.next_char();

let end_pos = self.get_pos();

let imag = f64::from_str(&value_text).unwrap();

Ok((start_pos, Tok::Complex { real: 0.0, imag }, end_pos))

} else {

let end_pos = self.get_pos();

let value = value_text.parse::<BigInt>().unwrap();

if start_is_zero && !value.is_zero() {

// leading zeros in decimal integer literals are not permitted

return Err(LexicalError {

error: LexicalErrorType::OtherError("Invalid Token".to_owned()),

location: self.get_pos(),

});

}

Ok((start_pos, Tok::Int { value }, end_pos))

}

}

}

/// Consume a sequence of numbers with the given radix,

/// the digits can be decorated with underscores

/// like this: '1_2_3_4' == '1234'

fn radix_run(&mut self, radix: u32) -> String {

let mut value_text = String::new();

loop {

if let Some(c) = self.take_number(radix) {

value_text.push(c);

} else if self.window[0] == Some('_')

&& Lexer::<T>::is_digit_of_radix(self.window[1], radix)

{

self.next_char();

} else {

break;

}

}

value_text

}

/// Consume a single character with the given radix.

fn take_number(&mut self, radix: u32) -> Option<char> {

let take_char = Lexer::<T>::is_digit_of_radix(self.window[0], radix);

take_char.then(|| self.next_char().unwrap())

}

/// Test if a digit is of a certain radix.

fn is_digit_of_radix(c: Option<char>, radix: u32) -> bool {

match radix {

2 => matches!(c, Some('0'..='1')),

8 => matches!(c, Some('0'..='7')),

10 => matches!(c, Some('0'..='9')),

16 => matches!(c, Some('0'..='9') | Some('a'..='f') | Some('A'..='F')),

other => unimplemented!("Radix not implemented: {}", other),

}

}

/// Test if we face '[eE][-+]?[0-9]+'

fn at_exponent(&self) -> bool {

match self.window[..2] {

[Some('e' | 'E'), Some('+' | '-')] => matches!(self.window[2], Some('0'..='9')),

[Some('e' | 'E'), Some('0'..='9')] => true,

_ => false,

}

}

/// Lex a single comment.

fn lex_comment(&mut self) -> LexResult {

let start_pos = self.get_pos();

let mut value = String::new();

loop {

match self.window[0] {

Some('\n' | '\r') | None => {

let end_pos = self.get_pos();

return Ok((start_pos, Tok::Comment(value), end_pos));

}

Some(_) => {}

}

value.push(self.next_char().unwrap());

}

}

/// Lex a string literal.

fn lex_string(&mut self, kind: StringKind) -> LexResult {

let start_pos = self.get_pos();

for _ in 0..kind.prefix_len() {

self.next_char();

}

let quote_char = self.next_char().unwrap();

let mut string_content = String::with_capacity(5);

// If the next two characters are also the quote character, then we have a triple-quoted

// string; consume those two characters and ensure that we require a triple-quote to close

let triple_quoted = if self.window[..2] == [Some(quote_char); 2] {

self.next_char();

self.next_char();

true

} else {

false

};

loop {

match self.next_char() {

Some(c) => {

if c == '\\' {

if let Some(next_c) = self.next_char() {

string_content.push('\\');

string_content.push(next_c);

continue;

}

}

if c == '\n' && !triple_quoted {

return Err(LexicalError {

error: LexicalErrorType::OtherError(

"EOL while scanning string literal".to_owned(),

),

location: self.get_pos(),

});

}

if c == quote_char {

if triple_quoted {

// Look ahead at the next two characters; if we have two more

// quote_chars, it's the end of the string; consume the remaining

// closing quotes and break the loop

if self.window[..2] == [Some(quote_char); 2] {

self.next_char();

self.next_char();

break;

}

} else {

break;

}

}

string_content.push(c);

}

None => {

return Err(LexicalError {

error: if triple_quoted {

LexicalErrorType::Eof

} else {

LexicalErrorType::StringError

},

location: self.get_pos(),

});

}

}

}

let end_pos = self.get_pos();

let tok = Tok::String {

value: string_content,

kind,

triple_quoted,

};

Ok((start_pos, tok, end_pos))

}

// Checks if the character c is a valid starting character as described

// in https://docs.python.org/3/reference/lexical_analysis.html#identifiers

fn is_identifier_start(&self, c: char) -> bool {

match c {

'a'..='z' | 'A'..='Z' | '_' => true,

_ => is_xid_start(c),

}

}

// Checks if the character c is a valid continuation character as described

// in https://docs.python.org/3/reference/lexical_analysis.html#identifiers

fn is_identifier_continuation(&self) -> bool {

match self.window[0] {

Some('a'..='z' | 'A'..='Z' | '_' | '0'..='9') => true,

Some(c) => is_xid_continue(c),

_ => false,

}

}

// This is the main entry point. Call this function to retrieve the next token.

// This function is used by the iterator implementation.

fn inner_next(&mut self) -> LexResult {

// top loop, keep on processing, until we have something pending.

while self.pending.is_empty() {

// Detect indentation levels

if self.at_begin_of_line {

self.handle_indentations()?;

}

self.consume_normal()?;

}

Ok(self.pending.remove(0))

}

// Given we are at the start of a line, count the number of spaces and/or tabs until the first character.

fn eat_indentation(&mut self) -> Result<IndentationLevel, LexicalError> {

// Determine indentation:

let mut spaces: u32 = 0;

let mut tabs: u32 = 0;

loop {

match self.window[0] {

Some(' ') => {

/*

self.next_char();

spaces += 1;

}

Some('\t') => {

if spaces != 0 {

// Don't allow tabs after spaces as part of indentation.

// This is technically stricter than python3 but spaces before

// tabs is even more insane than mixing spaces and tabs.

return Err(LexicalError {

error: LexicalErrorType::TabsAfterSpaces,

location: self.get_pos(),

});

}

self.next_char();

tabs += 1;

}

Some('#') => {

let comment = self.lex_comment()?;

self.emit(comment);

spaces = 0;

tabs = 0;

}

Some('\x0C') => {

// Form feed character!

// Reset indentation for the Emacs user.

self.next_char();

spaces = 0;

tabs = 0;

}

Some('\n' | '\r') => {

// Empty line!

self.next_char();

spaces = 0;

tabs = 0;

}

None => {

spaces = 0;

tabs = 0;

break;

}

_ => {

self.at_begin_of_line = false;

break;

}

}

}

Ok(IndentationLevel { tabs, spaces })

}

// Push/pop indents/dedents based on the current indentation level.

fn handle_indentations(&mut self) -> Result<(), LexicalError> {

let indentation_level = self.eat_indentation()?;

if self.nesting != 0 {

return Ok(());

}

// Determine indent or dedent:

let current_indentation = self.indentations.current();

let ordering = indentation_level.compare_strict(current_indentation, self.get_pos())?;

match ordering {

Ordering::Equal => {

// Same same

}

Ordering::Greater => {

// New indentation level:

self.indentations.push(indentation_level);

let tok_pos = self.get_pos();

self.emit((tok_pos, Tok::Indent, tok_pos));

}

Ordering::Less => {

// One or more dedentations

// Pop off other levels until col is found:

loop {

let current_indentation = self.indentations.current();

let ordering =

indentation_level.compare_strict(current_indentation, self.get_pos())?;

match ordering {

Ordering::Less => {

self.indentations.pop();

let tok_pos = self.get_pos();

self.emit((tok_pos, Tok::Dedent, tok_pos));

}

Ordering::Equal => {

// We arrived at proper level of indentation.

break;

}

Ordering::Greater => {

return Err(LexicalError {

error: LexicalErrorType::IndentationError,

location: self.get_pos(),

});

}

}

}

}

}

Ok(())

}

// Take a look at the next character, if any, and decide upon the next steps.

fn consume_normal(&mut self) -> Result<(), LexicalError> {

if let Some(c) = self.window[0] {

// Identifiers are the most common case.

if self.is_identifier_start(c) {

let identifier = self.lex_identifier()?;

self.emit(identifier);

} else {

self.consume_character(c)?;

}

} else {

// We reached end of file.

let tok_pos = self.get_pos();

// First of all, we need all nestings to be finished.

if self.nesting > 0 {

return Err(LexicalError {

error: LexicalErrorType::Eof,

location: tok_pos,

});

}

// Next, insert a trailing newline, if required.

if !self.at_begin_of_line {

self.at_begin_of_line = true;

self.emit((tok_pos, Tok::Newline, tok_pos));

}

// Next, flush the indentation stack to zero.

while !self.indentations.is_empty() {

self.indentations.pop();

self.emit((tok_pos, Tok::Dedent, tok_pos));

}

self.emit((tok_pos, Tok::EndOfFile, tok_pos));

}

Ok(())

}

// Dispatch based on the given character.

fn consume_character(&mut self, c: char) -> Result<(), LexicalError> {

match c {

'0'..='9' => {

let number = self.lex_number()?;

self.emit(number);

}

'#' => {

let comment = self.lex_comment()?;

self.emit(comment);

}

'"' | '\'' => {

let string = self.lex_string(StringKind::String)?;

self.emit(string);

}

'=' => {

let tok_start = self.get_pos();

self.next_char();

match self.window[0] {

Some('=') => {

self.next_char();

let tok_end = self.get_pos();

self.emit((tok_start, Tok::EqEqual, tok_end));

}

_ => {

let tok_end = self.get_pos();

self.emit((tok_start, Tok::Equal, tok_end));

}

}

}

'+' => {

let tok_start = self.get_pos();

self.next_char();

if let Some('=') = self.window[0] {

self.next_char();

let tok_end = self.get_pos();

self.emit((tok_start, Tok::PlusEqual, tok_end));

} else {

let tok_end = self.get_pos();

self.emit((tok_start, Tok::Plus, tok_end));

}

}

'*' => {

let tok_start = self.get_pos();

self.next_char();

match self.window[0] {

Some('=') => {

self.next_char();

let tok_end = self.get_pos();

self.emit((tok_start, Tok::StarEqual, tok_end));

}

Some('*') => {

self.next_char();

match self.window[0] {

Some('=') => {

self.next_char();

let tok_end = self.get_pos();

self.emit((tok_start, Tok::DoubleStarEqual, tok_end));

}

_ => {

let tok_end = self.get_pos();

self.emit((tok_start, Tok::DoubleStar, tok_end));

}

}

}

_ => {

let tok_end = self.get_pos();

self.emit((tok_start, Tok::Star, tok_end));

}

}

}

'/' => {

let tok_start = self.get_pos();

self.next_char();

match self.window[0] {

Some('=') => {

self.next_char();

let tok_end = self.get_pos();

self.emit((tok_start, Tok::SlashEqual, tok_end));

}

Some('/') => {

self.next_char();

match self.window[0] {

Some('=') => {

self.next_char();

let tok_end = self.get_pos();

self.emit((tok_start, Tok::DoubleSlashEqual, tok_end));

}

_ => {

let tok_end = self.get_pos();

self.emit((tok_start, Tok::DoubleSlash, tok_end));

}

}

}

_ => {

let tok_end = self.get_pos();

self.emit((tok_start, Tok::Slash, tok_end));

}

}

}

'%' => {

let tok_start = self.get_pos();

self.next_char();

if let Some('=') = self.window[0] {

self.next_char();

let tok_end = self.get_pos();

self.emit((tok_start, Tok::PercentEqual, tok_end));

} else {

let tok_end = self.get_pos();

self.emit((tok_start, Tok::Percent, tok_end));

}

}

'|' => {

let tok_start = self.get_pos();

self.next_char();

if let Some('=') = self.window[0] {

self.next_char();

let tok_end = self.get_pos();

self.emit((tok_start, Tok::VbarEqual, tok_end));

} else {

let tok_end = self.get_pos();

self.emit((tok_start, Tok::Vbar, tok_end));

}

}

'^' => {

let tok_start = self.get_pos();

self.next_char();

if let Some('=') = self.window[0] {

self.next_char();

let tok_end = self.get_pos();

self.emit((tok_start, Tok::CircumflexEqual, tok_end));

} else {

let tok_end = self.get_pos();

self.emit((tok_start, Tok::CircumFlex, tok_end));