Skip to content

Commit 1cd1b5d

Browse files
authored
Preserve tokenizer-only compiler warning order (#9316)
Use the collected lexical warning order for valid Exec/Eval inputs without parser-origin escape events. Keep tokenizer errors terminal and retain mixed-origin, invalid-source, and Single fallback behavior. Assisted-by: OpenAI Codex:model-version-unavailable
1 parent 6dbcb95 commit 1cd1b5d

1 file changed

Lines changed: 17 additions & 4 deletions

File tree

‎crates/vm/src/vm/compile.rs‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1868,6 +1868,7 @@ mod escape_warnings {
18681868
inherited_barry_as_flufl: bool,
18691869
emitted_escapes: &[usize],
18701870
) -> Result<(), CompileWarningError> {
1871+
let tokenizer_only_mode = matches!(mode, compiler::Mode::Exec | compiler::Mode::Eval);
18711872
let is_single = matches!(mode, compiler::Mode::Single);
18721873
let mode = match mode {
18731874
compiler::Mode::Eval => ruff_python_parser::Mode::Expression,
@@ -1882,9 +1883,21 @@ mod escape_warnings {
18821883
inherited_barry_as_flufl,
18831884
is_single,
18841885
);
1885-
if !collected.has_valid_syntax || !collected.tokenizer_recovery {
1886-
// Invalid source and interpolated tokenizer warnings need a
1887-
// separate recovery contract. Preserve their existing behavior.
1886+
// Tokenizer-only batches can preserve lexical warning order without
1887+
// parser-error recovery. Mixed parser/tokenizer events still need
1888+
// their separate fallback contract, as does Single's input boundary.
1889+
let tokenizer_only = tokenizer_only_mode
1890+
&& collected.warnings.iter().any(|warning| {
1891+
matches!(
1892+
warning,
1893+
ParserWarning::Escape(_, EscapeOrigin::Tokenizer, _)
1894+
)
1895+
})
1896+
&& collected.warnings.iter().all(|warning| {
1897+
!matches!(warning, ParserWarning::Escape(_, EscapeOrigin::Parser, _))
1898+
});
1899+
if !collected.has_valid_syntax || (!collected.tokenizer_recovery && !tokenizer_only) {
1900+
// Preserve invalid-source and mixed-origin fallback behavior.
18881901
self.emit_tokenizer_syntax_warnings(source, filename, module)?;
18891902
return self.emit_string_escape_warnings(source, filename, module, emitted_escapes);
18901903
}
@@ -1896,7 +1909,7 @@ mod escape_warnings {
18961909
collected.single_boundary,
18971910
WarningEmission::Standard {
18981911
emitted_escapes,
1899-
tokenizer_recovery: true,
1912+
tokenizer_recovery: !tokenizer_only,
19001913
},
19011914
)
19021915
.map(|_| ())

0 commit comments

Comments
 (0)