Skip to content

Commit 73b6806

Browse files
youknowdotyouknowdot
authored
Limit single-input parser warning emission to the first logical input (#9304)
Keep later numeric warnings available after parser warning escalation. Preserve compound tokenizer warning emission and legacy malformed-source fallback. Assisted-by: OpenAI Codex:gpt-6-astra
1 parent e5554e2 commit 73b6806

2 files changed

Lines changed: 91 additions & 10 deletions

File tree

‎crates/compiler/src/lib.rs‎

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1279,6 +1279,35 @@ fn first_parse_error(
12791279
Err(error)
12801280
}
12811281

1282+
/// The first logical input accepted by the single-input grammar.
1283+
#[derive(Clone, Copy)]
1284+
pub enum SingleModeStatementBoundary {
1285+
Simple { newline_index: usize, end: TextSize },
1286+
Compound { end: TextSize },
1287+
}
1288+
1289+
/// Classify a logical input without changing parser diagnostics or recovery.
1290+
#[must_use]
1291+
pub fn single_mode_statement_boundary(
1292+
parsed: &parser::Parsed<ast::Mod>,
1293+
) -> Option<SingleModeStatementBoundary> {
1294+
let ast::Mod::Module(module) = parsed.syntax() else {
1295+
return None;
1296+
};
1297+
let first = module.body.first()?;
1298+
if is_compound_stmt(first) {
1299+
return Some(SingleModeStatementBoundary::Compound { end: first.end() });
1300+
}
1301+
let newline_index = parsed
1302+
.tokens()
1303+
.iter()
1304+
.position(|token| token.kind() == TokenKind::Newline)?;
1305+
Some(SingleModeStatementBoundary::Simple {
1306+
newline_index,
1307+
end: parsed.tokens()[newline_index].end(),
1308+
})
1309+
}
1310+
12821311
/// Single mode reads only the first statement when it is a simple statement, so any code after
12831312
/// its line is reported as multiple statements, ahead of errors the rest of the source has.
12841313
#[must_use]
@@ -1289,8 +1318,8 @@ pub fn single_mode_multiple_statements_error(
12891318
let ast::Mod::Module(module) = parsed.syntax() else {
12901319
return None;
12911320
};
1292-
let first = module.body.first()?;
1293-
if is_compound_stmt(first) {
1321+
let boundary = single_mode_statement_boundary(parsed)?;
1322+
if matches!(boundary, SingleModeStatementBoundary::Compound { .. }) {
12941323
// The interactive grammar ends a compound statement with a NEWLINE,
12951324
// so the first token of a following statement is invalid syntax.
12961325
let next = module.body.get(1)?;
@@ -1315,9 +1344,9 @@ pub fn single_mode_multiple_statements_error(
13151344
));
13161345
}
13171346
let tokens = parsed.tokens();
1318-
let newline_index = tokens
1319-
.iter()
1320-
.position(|token| token.kind() == TokenKind::Newline)?;
1347+
let SingleModeStatementBoundary::Simple { newline_index, .. } = boundary else {
1348+
unreachable!()
1349+
};
13211350
let newline = &tokens[newline_index];
13221351
if parsed
13231352
.errors()

‎crates/vm/src/vm/compile.rs‎

Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1738,16 +1738,19 @@ mod escape_warnings {
17381738
warnings: Vec<ParserWarning>,
17391739
tokenizer_recovery: bool,
17401740
has_valid_syntax: bool,
1741+
single_boundary: Option<compiler::SingleModeStatementBoundary>,
17411742
}
17421743

17431744
fn collect_parser_warnings(
17441745
source: &str,
17451746
filename: &str,
17461747
mode: Option<ruff_python_parser::Mode>,
17471748
inherited_barry_as_flufl: bool,
1749+
single_mode: bool,
17481750
) -> CollectedParserWarnings {
17491751
let mut ast_tokenizer_recovery = true;
17501752
let mut has_valid_syntax = false;
1753+
let mut single_boundary = None;
17511754
let mut warnings = Vec::new();
17521755
collect_numeric_literal_warnings(source, &mut warnings);
17531756
// The compile that follows rejects this source; parsing it here
@@ -1765,6 +1768,9 @@ mod escape_warnings {
17651768
let parsed = ruff_python_parser::parse_unchecked(barry_source.source(), options);
17661769
if parsed.has_valid_syntax() {
17671770
has_valid_syntax = true;
1771+
if single_mode {
1772+
single_boundary = compiler::single_mode_statement_boundary(&parsed);
1773+
}
17681774
let mut visitor = EscapeWarningVisitor {
17691775
source,
17701776
warnings: &mut warnings,
@@ -1837,6 +1843,7 @@ mod escape_warnings {
18371843
warnings,
18381844
tokenizer_recovery: ast_tokenizer_recovery,
18391845
has_valid_syntax,
1846+
single_boundary,
18401847
}
18411848
}
18421849

@@ -1861,14 +1868,20 @@ mod escape_warnings {
18611868
inherited_barry_as_flufl: bool,
18621869
emitted_escapes: &[usize],
18631870
) -> Result<(), CompileWarningError> {
1871+
let is_single = matches!(mode, compiler::Mode::Single);
18641872
let mode = match mode {
18651873
compiler::Mode::Eval => ruff_python_parser::Mode::Expression,
18661874
compiler::Mode::Exec | compiler::Mode::Single | compiler::Mode::BlockExpr => {
18671875
ruff_python_parser::Mode::Module
18681876
}
18691877
};
1870-
let collected =
1871-
collect_parser_warnings(source, filename, Some(mode), inherited_barry_as_flufl);
1878+
let collected = collect_parser_warnings(
1879+
source,
1880+
filename,
1881+
Some(mode),
1882+
inherited_barry_as_flufl,
1883+
is_single,
1884+
);
18721885
if !collected.has_valid_syntax || !collected.tokenizer_recovery {
18731886
// Invalid source and interpolated tokenizer warnings need a
18741887
// separate recovery contract. Preserve their existing behavior.
@@ -1880,9 +1893,10 @@ mod escape_warnings {
18801893
filename,
18811894
module,
18821895
collected.warnings,
1896+
collected.single_boundary,
18831897
WarningEmission::Standard {
18841898
emitted_escapes,
1885-
tokenizer_recovery: collected.tokenizer_recovery,
1899+
tokenizer_recovery: true,
18861900
},
18871901
)
18881902
.map(|_| ())
@@ -1902,6 +1916,7 @@ mod escape_warnings {
19021916
filename,
19031917
module,
19041918
warnings,
1919+
None,
19051920
WarningEmission::Standard {
19061921
emitted_escapes: &[],
19071922
tokenizer_recovery: false,
@@ -1954,6 +1969,7 @@ mod escape_warnings {
19541969
filename,
19551970
module,
19561971
warnings,
1972+
None,
19571973
WarningEmission::Standard {
19581974
emitted_escapes,
19591975
tokenizer_recovery: false,
@@ -1975,13 +1991,26 @@ mod escape_warnings {
19751991
CompileStart::Eval => Some(ruff_python_parser::Mode::Expression),
19761992
CompileStart::FuncType => None,
19771993
};
1978-
let collected =
1979-
collect_parser_warnings(source, filename, mode, inherited_barry_as_flufl);
1994+
let collected = collect_parser_warnings(
1995+
source,
1996+
filename,
1997+
mode,
1998+
inherited_barry_as_flufl,
1999+
matches!(start, CompileStart::Single),
2000+
);
19802001
self.emit_parser_warning_batch(
19812002
source,
19822003
filename,
19832004
module.map(AsObject::as_object),
19842005
collected.warnings,
2006+
if matches!(start, CompileStart::Single)
2007+
&& collected.has_valid_syntax
2008+
&& collected.tokenizer_recovery
2009+
{
2010+
collected.single_boundary
2011+
} else {
2012+
None
2013+
},
19852014
WarningEmission::Ast {
19862015
// Unlike an interactive prompt, Single compile input recovers
19872016
// after a parser warning becomes an error. Later numeric
@@ -2010,12 +2039,14 @@ mod escape_warnings {
20102039
filename,
20112040
Some(ruff_python_parser::Mode::Module),
20122041
inherited_barry_as_flufl,
2042+
false,
20132043
);
20142044
self.emit_parser_warning_batch(
20152045
source,
20162046
filename,
20172047
module.map(AsObject::as_object),
20182048
collected.warnings,
2049+
None,
20192050
WarningEmission::Interactive(emitted),
20202051
)
20212052
.map(|_| ())
@@ -2027,6 +2058,7 @@ mod escape_warnings {
20272058
filename: &str,
20282059
module: Option<&PyObject>,
20292060
warnings: Vec<ParserWarning>,
2061+
single_boundary: Option<compiler::SingleModeStatementBoundary>,
20302062
emission: WarningEmission<'_>,
20312063
) -> Result<Vec<usize>, CompileWarningError> {
20322064
let (emitted_escapes, ast_tokenizer_recovery, mut interactive) = match emission {
@@ -2040,6 +2072,26 @@ mod escape_warnings {
20402072
let mut considered_escapes = Vec::new();
20412073
let mut parser_error = None;
20422074
for warning in warnings {
2075+
// A successful single input does not parse later statements.
2076+
// Keep those numeric events for recovery after an earlier
2077+
// parser warning escalates, rather than truncating the source.
2078+
if parser_error.is_none()
2079+
&& let Some(boundary) = single_boundary
2080+
{
2081+
let end = match (boundary, &warning) {
2082+
(compiler::SingleModeStatementBoundary::Simple { end, .. }, _) => Some(end),
2083+
(
2084+
compiler::SingleModeStatementBoundary::Compound { end },
2085+
ParserWarning::Escape(_, EscapeOrigin::Parser, _),
2086+
) => Some(end),
2087+
// Compound input requests another token and can emit
2088+
// tokenizer warnings while rejecting the next statement.
2089+
_ => None,
2090+
};
2091+
if end.is_some_and(|end| warning.offset() >= end.to_usize()) {
2092+
continue;
2093+
}
2094+
}
20432095
// CPython resumes tokenization after a parser-origin SyntaxError.
20442096
// Preserve ordinary warning order and never replay prior warnings.
20452097
if parser_error.is_some() && !matches!(warning, ParserWarning::Numeric { .. }) {

0 commit comments

Comments
 (0)