/// Source code to be lexed.

source: &'src str,

/// A pointer to the current character of the source code which is being lexed.

cursor: Cursor<'src>,

/// The kind of the current token.

current_kind: TokenKind,

/// The range of the current token.

current_range: TextRange,

/// Flags for the current token.

current_flags: TokenFlags,

/// Lexer state.

state: State,

/// Represents the current level of nesting in the lexer, indicating the depth of parentheses.

/// The lexer is within a parenthesized context if the value is greater than 0.

nesting: u32,

/// The opening brackets that are not closed yet, innermost last. It holds the innermost

/// `nesting` brackets; the implicit outer parentheses of

/// [`Mode::ParenthesizedExpression`] have no entry.

brackets: Vec<OpenBracket>,

/// A stack of indentation representing the current indentation level.

indentations: Indentations,

pending_indentation: Option<Indentation>,

/// Lexer mode.

mode: Mode,

/// F-string and t-string contexts.

interpolated_strings: InterpolatedStrings,

/// Errors encountered while lexing.

errors: Vec<LexicalError>,

/// Create a new lexer for the given input source which starts at the given offset.

///

/// If the start offset is greater than 0, the cursor is moved ahead that many bytes.

/// This means that the input source should be the complete source code and not the

/// sliced version.

pub(crate) fn new(source: &'src str, mode: Mode, start_offset: TextSize) -> Self {

assert!(

u32::try_from(source.len()).is_ok(),

"Lexer only supports files with a size up to 4GB"

);

let (state, nesting) = if mode == Mode::ParenthesizedExpression {

(State::Other, 1)

} else {

(State::AfterNewline, 0)

};

let mut lexer = Lexer {

source,

cursor: Cursor::new(source),

state,

current_kind: TokenKind::EndOfFile,

current_range: TextRange::empty(start_offset),

current_flags: TokenFlags::empty(),

nesting,

brackets: Vec::new(),

indentations: Indentations::default(),

pending_indentation: None,

mode,

interpolated_strings: InterpolatedStrings::default(),

errors: Vec::new(),

};

if start_offset == TextSize::new(0) {

// TODO: Handle possible mismatch between BOM and explicit encoding declaration.

lexer.cursor.eat_char(BOM);

} else {

lexer.cursor.skip_bytes(start_offset.to_usize());

}

lexer

}

/// Returns the kind of the current token.

pub(crate) const fn current_kind(&self) -> TokenKind {

self.current_kind

}

/// Returns the range of the current token.

#[doc(hidden)]

pub const fn current_range(&self) -> TextRange {

self.current_range

}

/// Returns the flags for the current token.

#[doc(hidden)]

pub const fn current_flags(&self) -> TokenFlags {

self.current_flags

}

/// Helper function to push the given error, updating the current range with the error location

/// and return the [`TokenKind::Unknown`] token.

fn push_error(&mut self, error: LexicalError) -> TokenKind {

self.current_range = error.location();

self.errors.push(error);

TokenKind::Unknown

}

/// Pushes an indentation error for the current line.

///

/// An unmatched dedent is reported at the end of the line, tab and depth errors at its start.

fn push_indentation_error(&mut self, error: IndentationError) -> TokenKind {

let offset = self.offset().to_usize();

let line_start = self.source[..offset]

.rfind(['\n', '\r'])

.map_or(0, |index| index + 1);

let line_end = self.source[offset..]

.find(['\n', '\r'])

.map_or(self.source.len(), |index| offset + index);

let (error, position) = match error {

IndentationError::UnmatchedDedent => (LexicalErrorType::IndentationError, line_end),

IndentationError::InconsistentTabs => (LexicalErrorType::TabError, line_start),

IndentationError::TooDeep => (LexicalErrorType::TooDeepIndentation, line_start),

};

let range = TextRange::empty(TextSize::try_from(position).unwrap());

self.push_error(LexicalError::new(error, range));

// The token keeps covering the indentation.

self.current_range = self.token_range();

TokenKind::Unknown

}

/// Lex the next token.

pub fn next_token(&mut self) -> TokenKind {

// `lex_token` marks the start on the path that lexes each token.

self.current_flags = TokenFlags::empty();

self.current_kind = self.lex_token();

// For `Unknown` token, the `push_error` method updates the current range.

if !matches!(self.current_kind, TokenKind::Unknown) {

self.current_range = self.token_range();

}

self.current_kind

}

fn lex_token(&mut self) -> TokenKind {

if let Some(interpolated_string) = self.interpolated_strings.current() {

if !interpolated_string.is_in_interpolation(self.nesting) {

let nesting = interpolated_string.nesting();

self.cursor.start_token();

if let Some(token) = self.lex_interpolated_string_middle_or_end() {

if token.is_interpolated_string_end() {

// Brackets left open in a replacement field end with the string.

self.reset_nesting(nesting);

self.interpolated_strings.pop();

}

return token;

}

}

}

// Return dedent tokens until the current indentation level matches the indentation of the next token.

// Avoid `Option::take` here: this check runs for every token, and `take` writes `None`

// even when there is no pending indentation.

else if let Some(indentation) = self.pending_indentation {

self.pending_indentation = None;

self.cursor.start_token();

match self.indentations.current().try_compare(indentation) {

Ok(Ordering::Greater) => {

self.pending_indentation = Some(indentation);

if let Err(error) = self.indentations.dedent_one(indentation) {

return self.push_indentation_error(error);

}

return TokenKind::Dedent;

}

Ok(_) => {}

Err(error) => return self.push_indentation_error(error),

}

}

if self.state.is_after_newline() {

// Indent and dedent tokens include leading whitespace in their ranges.

self.cursor.start_token();

if let Some(indentation) = self.eat_indentation() {

return indentation;

}

} else {

if let Err(error) = self.skip_whitespace() {

return self.push_error(error);

}

}

// Whitespace between tokens is not part of the next token's range.

self.cursor.start_token();

if let Some(c) = self.cursor.bump() {

if c.is_ascii() {

self.consume_ascii_character(c)

} else if is_unicode_identifier_start(c) {

let identifier = self.lex_identifier(c);

self.state = State::Other;

identifier

} else {

self.push_error(LexicalError::new(

LexicalErrorType::UnrecognizedToken { tok: c },

self.token_range(),

))

}

} else {

// Reached the end of the file. Emit a trailing newline token if not at the beginning of a logical line,

// empty the dedent stack, and finally, return the EndOfFile token.

self.consume_end()

}

}

fn eat_indentation(&mut self) -> Option<TokenKind> {

let mut indentation = Indentation::root();

loop {

match self.cursor.first() {

' ' => {

self.cursor.bump();

indentation = indentation.add_space();

}

'\t' => {

self.cursor.bump();

indentation = indentation.add_tab();

}

'\\' => {

self.cursor.bump();

if self.cursor.eat_char('\r') {

self.cursor.eat_char('\n');

} else if !self.cursor.eat_char('\n') {

return Some(self.push_error(LexicalError::new(

LexicalErrorType::LineContinuationError,

TextRange::empty(self.offset()),

)));

}

if self.cursor.is_eof() {

let error = self.eof_error(self.token_range());

return Some(self.push_error(error));

}

// test_ok backslash_continuation_indentation

// if True:

// \

// 1

// \

// 2

// else:\

// 3

// test_err backslash_continuation_indentation_error

// if True:

// 1

// \

// 2

// > Indentation cannot be split over multiple physical lines using backslashes;

// > the whitespace up to the first backslash determines the indentation.

// >

// > https://docs.python.org/3/reference/lexical_analysis.html#indentation

//

// Skip whitespace after the continuation-line without accumulating it into

// `indentation`. However, if the backslash is at column 0 (no prior

// indentation), let the loop continue so the next line's whitespace is

// accumulated normally.

//

// See also: https://github.com/python/cpython/issues/90249

if indentation != Indentation::root() {

self.cursor.eat_while(is_python_whitespace);

}

}

// Form feed

'\x0C' => {

self.cursor.bump();

indentation = Indentation::root();

}

_ => break,

}

}

// Handle indentation if this is a new, not all empty, logical line

if !matches!(self.cursor.first(), '\n' | '\r' | '#' | EOF_CHAR) {

self.state = State::NonEmptyLogicalLine;

// Set to false so that we don't handle indentation on the next call.

return self.handle_indentation(indentation);

}

None

}

fn handle_indentation(&mut self, indentation: Indentation) -> Option<TokenKind> {

match self.indentations.current().try_compare(indentation) {

// Dedent

Ok(Ordering::Greater) => {

self.pending_indentation = Some(indentation);

if let Err(error) = self.indentations.dedent_one(indentation) {

return Some(self.push_indentation_error(error));

}

// The lexer might've eaten some whitespaces to calculate the `indentation`. For

// example:

//

// ```py

// if first:

// if second:

// pass

// foo

// # ^

// ```

//

// Here, the cursor is at `^` and the `indentation` contains the whitespaces before

// the `pass` token.

self.cursor.start_token();

Some(TokenKind::Dedent)

}

Ok(Ordering::Equal) => None,

// Indent

Ok(Ordering::Less) => {

if let Err(error) = self.indentations.indent(indentation) {

return Some(self.push_indentation_error(error));

}

Some(TokenKind::Indent)

}

Err(error) => Some(self.push_indentation_error(error)),

}

}

fn skip_whitespace(&mut self) -> Result<(), LexicalError> {

let whitespace_start = if matches!(self.cursor.first(), ' ' | '\t' | '\\' | '\x0C') {

self.offset()

} else {

return Ok(());

};

loop {

match self.cursor.first() {

' ' => {

self.cursor.bump();

}

'\t' => {

self.cursor.bump();

}

'\\' => {

self.cursor.bump();

if self.cursor.eat_char('\r') {

self.cursor.eat_char('\n');

} else if !self.cursor.eat_char('\n') {

return Err(LexicalError::new(

LexicalErrorType::LineContinuationError,

TextRange::empty(self.offset()),

));

}

if self.cursor.is_eof() {

return Err(self.eof_error(TextRange::new(whitespace_start, self.offset())));

}

}

// Form feed

'\x0C' => {

self.cursor.bump();

}

_ => break,

}

}

Ok(())

}

// Dispatch based on the given character.

fn consume_ascii_character(&mut self, c: char) -> TokenKind {

let token = match c {

c if is_ascii_identifier_start(c) => self.lex_identifier(c),

'0'..='9' => self.lex_number(c),

'#' => return self.lex_comment(),

'\'' | '"' => self.lex_string(c),

'=' => {

if self.cursor.eat_char('=') {

TokenKind::EqEqual

} else {

self.state = State::AfterEqual;

return TokenKind::Equal;

}

}

'+' => {

if self.cursor.eat_char('=') {

TokenKind::PlusEqual

} else {

TokenKind::Plus

}

}

'*' => {

if self.cursor.eat_char('=') {

TokenKind::StarEqual

} else if self.cursor.eat_char('*') {

if self.cursor.eat_char('=') {

TokenKind::DoubleStarEqual

} else {

TokenKind::DoubleStar

}

} else {

TokenKind::Star

}

}

'%' | '!'

if self.mode == Mode::Ipython

&& self.state.is_after_equal()

&& self.nesting == 0 =>

{

self.lex_ipython_escape_command()

}

'%' | '!' | '?' | '/' | ';' | ','

if self.mode == Mode::Ipython && self.state.is_new_logical_line() =>

{

self.lex_ipython_escape_command()

}

'?' if self.mode == Mode::Ipython => TokenKind::Question,

'/' => {

if self.cursor.eat_char('=') {

TokenKind::SlashEqual

} else if self.cursor.eat_char('/') {

if self.cursor.eat_char('=') {

TokenKind::DoubleSlashEqual

} else {

TokenKind::DoubleSlash

}

} else {

TokenKind::Slash

}

}

'%' => {

if self.cursor.eat_char('=') {

TokenKind::PercentEqual

} else {

TokenKind::Percent

}

}

'|' => {

if self.cursor.eat_char('=') {

TokenKind::VbarEqual

} else {

TokenKind::Vbar

}

}

'^' => {

if self.cursor.eat_char('=') {

TokenKind::CircumflexEqual

} else {

TokenKind::CircumFlex

}

}

'&' => {

if self.cursor.eat_char('=') {

TokenKind::AmperEqual

} else {

TokenKind::Amper

}

}

'-' => {

if self.cursor.eat_char('=') {

TokenKind::MinusEqual

} else if self.cursor.eat_char('>') {

TokenKind::Rarrow

} else {

TokenKind::Minus

}

}

'@' => {

if self.cursor.eat_char('=') {

TokenKind::AtEqual

} else {

TokenKind::At

}

}

'!' => {

if self.cursor.eat_char('=') {

TokenKind::NotEqual

} else {

TokenKind::Exclamation

}

}

'~' => TokenKind::Tilde,

'(' => {

self.open_bracket('(');

TokenKind::Lpar

}

')' => {

self.close_bracket(')');

TokenKind::Rpar

}

'[' => {

self.open_bracket('[');

TokenKind::Lsqb

}

']' => {

self.close_bracket(']');

TokenKind::Rsqb

}

'{' => {

self.open_bracket('{');

TokenKind::Lbrace

}

'}' => {

if let Some(interpolated_string) = self.interpolated_strings.current_mut() {

if interpolated_string.nesting() == self.nesting {

let error_type = LexicalErrorType::from_interpolated_string_error(

InterpolatedStringErrorType::SingleRbrace,

interpolated_string.kind(),

);

return self.push_error(LexicalError::new(error_type, self.token_range()));

}

interpolated_string.try_end_format_spec(self.nesting);

}

self.close_bracket('}');

TokenKind::Rbrace

}

':' => {

if self

.interpolated_strings

.current_mut()

.is_some_and(|interpolated_string| {

interpolated_string.try_start_format_spec(self.nesting)

})

{

TokenKind::Colon

} else if self.cursor.eat_char('=') {

TokenKind::ColonEqual

} else {

TokenKind::Colon

}

}

';' => TokenKind::Semi,

'<' => {

if self.cursor.eat_char('<') {

if self.cursor.eat_char('=') {

TokenKind::LeftShiftEqual

} else {

TokenKind::LeftShift

}

} else if self.cursor.eat_char('=') {

TokenKind::LessEqual

} else {

TokenKind::Less

}

}

'>' => {

if self.cursor.eat_char('>') {

if self.cursor.eat_char('=') {

TokenKind::RightShiftEqual

} else {

TokenKind::RightShift

}

} else if self.cursor.eat_char('=') {

TokenKind::GreaterEqual

} else {

TokenKind::Greater

}

}

',' => TokenKind::Comma,

'.' => {

if self.cursor.first().is_ascii_digit() {

self.lex_decimal_number('.')

} else if self.cursor.eat_char2('.', '.') {

TokenKind::Ellipsis

} else {

TokenKind::Dot

}

}

'\n' => {

return if self.nesting == 0 && !self.state.is_new_logical_line() {

self.state = State::AfterNewline;

TokenKind::Newline

} else {

if let Some(interpolated_string) = self.interpolated_strings.current_mut() {

interpolated_string.try_end_format_spec(self.nesting);

}

TokenKind::NonLogicalNewline

};

}

'\r' => {

self.cursor.eat_char('\n');

return if self.nesting == 0 && !self.state.is_new_logical_line() {

self.state = State::AfterNewline;

TokenKind::Newline

} else {

if let Some(interpolated_string) = self.interpolated_strings.current_mut() {

interpolated_string.try_end_format_spec(self.nesting);

}

TokenKind::NonLogicalNewline

};

}

_ => {

self.state = State::Other;

return self.push_error(LexicalError::new(

LexicalErrorType::UnrecognizedToken { tok: c },

self.token_range(),

));

}

};

self.state = State::Other;

token

}

/// Lex an identifier. Also used for keywords and string/bytes literals with a prefix.

fn lex_identifier(&mut self, first: char) -> TokenKind {

if let Some(error) = self.incompatible_string_prefixes(first) {

self.cursor

.skip_bytes(error.location().len().to_usize() - first.len_utf8());

self.errors.push(error);

self.current_range = self.token_range();

return TokenKind::Unknown;

}

// Detect potential string like rb'' b'' f'' t'' u'' r''

let quote = if let Some(prefix) = single_char_prefix(first) {

match self.cursor.first() {

quote @ ('\'' | '"') => {

self.current_flags |= prefix;

self.cursor.bump();

Some(quote)

}

second

if let quote = self.cursor.second()

&& is_quote(quote) =>

{

self.try_double_char_prefix([first, second]).then(|| {

self.cursor.bump();

self.cursor.bump();

quote

})

}

_ => None,

}

} else {

None

};

if let Some(quote) = quote {

if self.current_flags.is_interpolated_string() {

if let Some(kind) = self.lex_interpolated_string_start(quote) {

return kind;

}

}

return self.lex_string(quote);

}

// Keep track of whether the identifier is ASCII-only or not.

//

// This is important because Python applies NFKC normalization to

// identifiers: https://docs.python.org/3/reference/lexical_analysis.html#identifiers.

// The parser needs to do the same when cooking the name, but applying

// NFKC normalization unconditionally is extremely expensive. If we know

// an identifier is ASCII-only (by far the most common case), the parser

// can skip NFKC normalization.

let mut is_ascii = first.is_ascii();

self.cursor

.eat_while(|c| is_identifier_continuation(c, &mut is_ascii));

if !is_ascii {

self.current_flags |= TokenFlags::NON_ASCII_IDENTIFIER;

return TokenKind::Identifier;

}

let text = self.token_text();

// No Python keyword is longer than eight bytes.

if text.len() > 8 {

return TokenKind::Identifier;

}

match text.as_bytes() {

b"False" => TokenKind::False,

b"None" => TokenKind::None,

b"True" => TokenKind::True,

b"and" => TokenKind::And,

b"as" => TokenKind::As,

b"assert" => TokenKind::Assert,

b"async" => TokenKind::Async,

b"await" => TokenKind::Await,

b"break" => TokenKind::Break,

b"case" => TokenKind::Case,

b"class" => TokenKind::Class,

b"continue" => TokenKind::Continue,

b"def" => TokenKind::Def,

b"del" => TokenKind::Del,

b"elif" => TokenKind::Elif,

b"else" => TokenKind::Else,

b"except" => TokenKind::Except,

b"finally" => TokenKind::Finally,

b"for" => TokenKind::For,

b"from" => TokenKind::From,

b"global" => TokenKind::Global,

b"if" => TokenKind::If,

b"import" => TokenKind::Import,

b"in" => TokenKind::In,

b"is" => TokenKind::Is,

b"lazy" => TokenKind::Lazy,

b"lambda" => TokenKind::Lambda,

b"match" => TokenKind::Match,

b"nonlocal" => TokenKind::Nonlocal,

b"not" => TokenKind::Not,

b"or" => TokenKind::Or,

b"pass" => TokenKind::Pass,

b"raise" => TokenKind::Raise,

b"return" => TokenKind::Return,

b"try" => TokenKind::Try,

b"type" => TokenKind::Type,

b"while" => TokenKind::While,

b"with" => TokenKind::With,

b"yield" => TokenKind::Yield,

_ => TokenKind::Identifier,

}

}

/// Returns the error for a string prefix starting with `first` that combines incompatible

/// prefixes, such as `ub''`. The error covers the prefix.

fn incompatible_string_prefixes(&self, first: char) -> Option<LexicalError> {

const PREFIXES: [char; 5] = ['u', 'b', 'r', 'f', 't'];

const INCOMPATIBLE: [(char, char); 7] = [

('u', 'b'),

('u', 'r'),

('u', 'f'),

('u', 't'),

('b', 'f'),

('b', 't'),

('f', 't'),

];

let bit = |prefix: char| PREFIXES.iter().position(|&c| c == prefix).map(|i| 1u8 << i);

let mut seen = 0u8;

let mut len = TextSize::new(0);

let mut chars = std::iter::once(first).chain(self.cursor.rest().chars());

loop {

let c = chars.next()?;

if is_quote(c) {

break;

}

let prefix = bit(c.to_ascii_lowercase())?;

if seen & prefix != 0 {

return None;

}

seen |= prefix;

len += c.text_len();

}

let has = |prefix: char| bit(prefix).is_some_and(|prefix| seen & prefix != 0);

let (first, second) = INCOMPATIBLE

.into_iter()

.find(|(first, second)| has(*first) && has(*second))?;

Some(LexicalError::new(

LexicalErrorType::IncompatibleStringPrefixes { first, second },

TextRange::at(self.token_range().start(), len),

))

}

/// Try lexing the double character string prefix, updating the token flags accordingly.

/// Returns `true` if it matches.

fn try_double_char_prefix(&mut self, value: [char; 2]) -> bool {

match value {

['r', 'f' | 'F'] | ['f' | 'F', 'r'] => {

self.current_flags |= TokenFlags::F_STRING | TokenFlags::RAW_STRING_LOWERCASE;

}

['R', 'f' | 'F'] | ['f' | 'F', 'R'] => {

self.current_flags |= TokenFlags::F_STRING | TokenFlags::RAW_STRING_UPPERCASE;

}

['r', 't' | 'T'] | ['t' | 'T', 'r'] => {

self.current_flags |= TokenFlags::T_STRING | TokenFlags::RAW_STRING_LOWERCASE;

}

['R', 't' | 'T'] | ['t' | 'T', 'R'] => {

self.current_flags |= TokenFlags::T_STRING | TokenFlags::RAW_STRING_UPPERCASE;

}

['r', 'b' | 'B'] | ['b' | 'B', 'r'] => {

self.current_flags |= TokenFlags::BYTE_STRING | TokenFlags::RAW_STRING_LOWERCASE;

}

['R', 'b' | 'B'] | ['b' | 'B', 'R'] => {

self.current_flags |= TokenFlags::BYTE_STRING | TokenFlags::RAW_STRING_UPPERCASE;

}

_ => return false,

}

true

}

/// Lex a f-string or t-string start token if positioned at the start of an f-string or t-string.

fn lex_interpolated_string_start(&mut self, quote: char) -> Option<TokenKind> {

#[cfg(debug_assertions)]

debug_assert_eq!(self.cursor.previous(), quote);

if quote == '"' {

self.current_flags |= TokenFlags::DOUBLE_QUOTES;

}

if self.cursor.eat_char2(quote, quote) {

self.current_flags |= TokenFlags::TRIPLE_QUOTED_STRING;

}

let ftcontext = InterpolatedStringContext::new(

self.current_flags,

self.nesting,

self.token_range().start(),

)?;

let kind = ftcontext.kind();

self.interpolated_strings.push(ftcontext);

Some(kind.start_token())

}

/// Lex an f-string or t-string middle or end token.

fn lex_interpolated_string_middle_or_end(&mut self) -> Option<TokenKind> {

// SAFETY: Safe because the function is only called when `self.fstrings` is not empty.

let interpolated_string = self.interpolated_strings.current().unwrap();

let string_kind = interpolated_string.kind();

let interpolated_flags = interpolated_string.flags();

// Check if we're at the end of the f-string.

if interpolated_string.is_triple_quoted() {

let quote_char = interpolated_string.quote_char();

if self.cursor.eat_char3(quote_char, quote_char, quote_char) {

self.current_flags = interpolated_string.flags();

return Some(string_kind.end_token());

}

} else if self.cursor.eat_char(interpolated_string.quote_char()) {

self.current_flags = interpolated_string.flags();

return Some(string_kind.end_token());

}

// This isn't going to change for the duration of the loop.

let in_format_spec = interpolated_string.is_in_format_spec(self.nesting);

let mut in_named_unicode = false;

loop {

match self.cursor.first() {

// The condition is to differentiate between the `NUL` (`\0`) character

// in the source code and the one returned by `self.cursor.first()` when

// we reach the end of the source code.

EOF_CHAR if self.cursor.is_eof() => {

let detected_line = self.detected_line();

let range = TextRange::empty(interpolated_string.start());

let error = if interpolated_string.is_triple_quoted() {

InterpolatedStringErrorType::UnterminatedTripleQuotedString {

detected_line,

}

} else {

InterpolatedStringErrorType::UnterminatedString { detected_line }

};

self.reset_nesting(interpolated_string.nesting());

self.interpolated_strings.pop();

self.current_flags |= TokenFlags::UNCLOSED_STRING;

self.push_error(LexicalError::new(

LexicalErrorType::from_interpolated_string_error(error, string_kind),

range,

));

break;

}

'\n' | '\r' if !interpolated_string.is_triple_quoted() => {