Skip to content

Commit eb79fef

Browse files
committed
Order parser warnings for valid source while preserving recovery boundaries
Reuse the parser warning collector only for valid, recovery-eligible source. Preserve malformed-source and tokenizer-origin interpolation fallback. Assisted-by: OpenAI Codex:model-version-unavailable
1 parent 1ba1107 commit eb79fef

1 file changed

Lines changed: 82 additions & 17 deletions

File tree

‎crates/vm/src/vm/compile.rs‎

Lines changed: 82 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -645,10 +645,16 @@ impl VirtualMachine {
645645
let _ = (module, emitted_escapes);
646646
#[cfg(feature = "parser")]
647647
{
648-
self.emit_tokenizer_syntax_warnings(source, &source_path, module)
649-
.map_err(VmCompileError::Warning)?;
650-
self.emit_string_escape_warnings(source, &source_path, module, emitted_escapes)
651-
.map_err(VmCompileError::Warning)?;
648+
self.emit_standard_parser_warnings(
649+
source,
650+
&source_path,
651+
module,
652+
mode,
653+
opts.future_features
654+
.contains(crate::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL),
655+
emitted_escapes,
656+
)
657+
.map_err(VmCompileError::Warning)?;
652658
}
653659
self.compile_after_parser_warnings_object(source, mode, source_path, opts, module)
654660
}
@@ -1728,13 +1734,20 @@ mod escape_warnings {
17281734
}
17291735
}
17301736

1737+
struct CollectedParserWarnings {
1738+
warnings: Vec<ParserWarning>,
1739+
tokenizer_recovery: bool,
1740+
has_valid_syntax: bool,
1741+
}
1742+
17311743
fn collect_parser_warnings(
17321744
source: &str,
17331745
filename: &str,
17341746
mode: Option<ruff_python_parser::Mode>,
17351747
inherited_barry_as_flufl: bool,
1736-
) -> (Vec<ParserWarning>, bool) {
1748+
) -> CollectedParserWarnings {
17371749
let mut ast_tokenizer_recovery = true;
1750+
let mut has_valid_syntax = false;
17381751
let mut warnings = Vec::new();
17391752
collect_numeric_literal_warnings(source, &mut warnings);
17401753
// The compile that follows rejects this source; parsing it here
@@ -1751,6 +1764,7 @@ mod escape_warnings {
17511764
);
17521765
let parsed = ruff_python_parser::parse_unchecked(barry_source.source(), options);
17531766
if parsed.has_valid_syntax() {
1767+
has_valid_syntax = true;
17541768
let mut visitor = EscapeWarningVisitor {
17551769
source,
17561770
warnings: &mut warnings,
@@ -1819,18 +1833,62 @@ mod escape_warnings {
18191833
}) {
18201834
ast_tokenizer_recovery = false;
18211835
}
1822-
(warnings, ast_tokenizer_recovery)
1836+
CollectedParserWarnings {
1837+
warnings,
1838+
tokenizer_recovery: ast_tokenizer_recovery,
1839+
has_valid_syntax,
1840+
}
18231841
}
18241842

18251843
enum WarningEmission<'a> {
1826-
Standard { emitted_escapes: &'a [usize] },
1827-
Ast { tokenizer_recovery: bool },
1844+
Standard {
1845+
emitted_escapes: &'a [usize],
1846+
tokenizer_recovery: bool,
1847+
},
1848+
Ast {
1849+
tokenizer_recovery: bool,
1850+
},
18281851
Interactive(&'a mut super::InteractiveWarnings),
18291852
}
18301853

18311854
impl VirtualMachine {
1832-
// Keep ordinary source compilation's existing warning order and
1833-
// recovery surface; interactive records use the bounded parser below.
1855+
pub(super) fn emit_standard_parser_warnings(
1856+
&self,
1857+
source: &str,
1858+
filename: &str,
1859+
module: Option<&PyObject>,
1860+
mode: compiler::Mode,
1861+
inherited_barry_as_flufl: bool,
1862+
emitted_escapes: &[usize],
1863+
) -> Result<(), CompileWarningError> {
1864+
let mode = match mode {
1865+
compiler::Mode::Eval => ruff_python_parser::Mode::Expression,
1866+
compiler::Mode::Exec | compiler::Mode::Single | compiler::Mode::BlockExpr => {
1867+
ruff_python_parser::Mode::Module
1868+
}
1869+
};
1870+
let collected =
1871+
collect_parser_warnings(source, filename, Some(mode), inherited_barry_as_flufl);
1872+
if !collected.has_valid_syntax || !collected.tokenizer_recovery {
1873+
// Invalid source and interpolated tokenizer warnings need a
1874+
// separate recovery contract. Preserve their existing behavior.
1875+
self.emit_tokenizer_syntax_warnings(source, filename, module)?;
1876+
return self.emit_string_escape_warnings(source, filename, module, emitted_escapes);
1877+
}
1878+
self.emit_parser_warning_batch(
1879+
source,
1880+
filename,
1881+
module,
1882+
collected.warnings,
1883+
WarningEmission::Standard {
1884+
emitted_escapes,
1885+
tokenizer_recovery: collected.tokenizer_recovery,
1886+
},
1887+
)
1888+
.map(|_| ())
1889+
}
1890+
1891+
// Preserve the existing invalid-source warning order and recovery surface.
18341892
pub(super) fn emit_tokenizer_syntax_warnings(
18351893
&self,
18361894
source: &str,
@@ -1846,6 +1904,7 @@ mod escape_warnings {
18461904
warnings,
18471905
WarningEmission::Standard {
18481906
emitted_escapes: &[],
1907+
tokenizer_recovery: false,
18491908
},
18501909
)
18511910
.map(|_| ())
@@ -1895,7 +1954,10 @@ mod escape_warnings {
18951954
filename,
18961955
module,
18971956
warnings,
1898-
WarningEmission::Standard { emitted_escapes },
1957+
WarningEmission::Standard {
1958+
emitted_escapes,
1959+
tokenizer_recovery: false,
1960+
},
18991961
)
19001962
.map(|_| ())
19011963
}
@@ -1913,15 +1975,15 @@ mod escape_warnings {
19131975
CompileStart::Eval => Some(ruff_python_parser::Mode::Expression),
19141976
CompileStart::FuncType => None,
19151977
};
1916-
let (warnings, recovery) =
1978+
let collected =
19171979
collect_parser_warnings(source, filename, mode, inherited_barry_as_flufl);
19181980
self.emit_parser_warning_batch(
19191981
source,
19201982
filename,
19211983
module.map(AsObject::as_object),
1922-
warnings,
1984+
collected.warnings,
19231985
WarningEmission::Ast {
1924-
tokenizer_recovery: recovery
1986+
tokenizer_recovery: collected.tokenizer_recovery
19251987
&& matches!(start, CompileStart::File | CompileStart::Eval),
19261988
},
19271989
)
@@ -1936,7 +1998,7 @@ mod escape_warnings {
19361998
inherited_barry_as_flufl: bool,
19371999
emitted: &mut super::InteractiveWarnings,
19382000
) -> Result<(), CompileWarningError> {
1939-
let (warnings, _) = collect_parser_warnings(
2001+
let collected = collect_parser_warnings(
19402002
source,
19412003
filename,
19422004
Some(ruff_python_parser::Mode::Module),
@@ -1946,7 +2008,7 @@ mod escape_warnings {
19462008
source,
19472009
filename,
19482010
module.map(AsObject::as_object),
1949-
warnings,
2011+
collected.warnings,
19502012
WarningEmission::Interactive(emitted),
19512013
)
19522014
.map(|_| ())
@@ -1961,7 +2023,10 @@ mod escape_warnings {
19612023
emission: WarningEmission<'_>,
19622024
) -> Result<Vec<usize>, CompileWarningError> {
19632025
let (emitted_escapes, ast_tokenizer_recovery, mut interactive) = match emission {
1964-
WarningEmission::Standard { emitted_escapes } => (emitted_escapes, false, None),
2026+
WarningEmission::Standard {
2027+
emitted_escapes,
2028+
tokenizer_recovery,
2029+
} => (emitted_escapes, tokenizer_recovery, None),
19652030
WarningEmission::Ast { tokenizer_recovery } => (&[][..], tokenizer_recovery, None),
19662031
WarningEmission::Interactive(emitted) => (&[][..], false, Some(emitted)),
19672032
};

0 commit comments

Comments
 (0)