youknowone · Nov 11, 2026 · Nov 10, 2026 · Nov 10, 2026
diff --git a/crates/compiler-core/src/lib.rs b/crates/compiler-core/src/lib.rs
@@ -9,10 +9,12 @@ pub mod bytecode;
 pub mod frozen;
 pub mod marshal;
 mod mode;
+mod source;
 mod source_location;
 pub mod varint;
 
 pub use mode::Mode;
+pub use source::normalize_newlines;
 pub use source_location::{AstLocationError, AstLocations, SourceLocationRange};
 
 pub use ruff_source_file::{

diff --git a/crates/compiler-core/src/source.rs b/crates/compiler-core/src/source.rs
@@ -0,0 +1,43 @@
+use alloc::{borrow::Cow, string::String};
+
+/// Translate physical source line endings before tokenization, including those
+/// inside literals, as CPython's `_PyTokenizer_translate_newlines` does.
+#[must_use]
+pub fn normalize_newlines(source: &str) -> Cow<'_, str> {
+    let Some(first_cr) = source.find('\r') else {
+        return Cow::Borrowed(source);
+    };
+    let mut normalized = String::with_capacity(source.len());
+    normalized.push_str(&source[..first_cr]);
+    let mut rest = &source[first_cr..];
+    while let Some(cr) = rest.find('\r') {
+        normalized.push_str(&rest[..cr]);
+        normalized.push('\n');
+        rest = &rest[cr + 1..];
+        if let Some(after_lf) = rest.strip_prefix('\n') {
+            rest = after_lf;
+        }
+    }
+    normalized.push_str(rest);
+    Cow::Owned(normalized)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn borrow_lf_source() {
+        let source = "print('한글')\n";
+        assert!(matches!(normalize_newlines(source), Cow::Borrowed(s) if s == source));
+    }
+
+    #[test]
+    fn physical_line_endings() {
+        assert_eq!(
+            normalize_newlines("a\r\nb\rc\n\rd\r\r\n"),
+            "a\nb\nc\n\nd\n\n"
+        );
+        assert_eq!(normalize_newlines("'\\r'\r"), "'\\r'\n");
+    }
+}
diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs
@@ -2111,13 +2111,7 @@ pub fn compile(
     source_path: &str,
     opts: CompileOpts,
 ) -> Result<CodeObject, CompileError> {
-    // TODO: do this less hacky; ruff's parser should translate a CRLF line
-    //       break in a multiline string into just an LF in the parsed value
-    #[cfg(windows)]
-    let source = source.replace("\r\n", "\n");
-    #[cfg(windows)]
-    let source = source.as_str();
-
+    let source = core::normalize_newlines(source);
     let source_file = SourceFileBuilder::new(source_path, source).finish();
     _compile(source_file, mode, opts)
     // let index = LineIndex::from_source_text(source);
@@ -2448,9 +2442,7 @@ pub fn compile_with_syntax_warning_handler<'a>(
     opts: CompileOpts,
     syntax_warning_handler: &'a mut compile::SyntaxWarningHandler<'a>,
 ) -> Result<CodeObject, CompileError> {
-    let source = source.replace("\r\n", "\n");
-    #[cfg(windows)]
-    let source = source.as_str();
+    let source = core::normalize_newlines(source);
 
     let source_file = SourceFileBuilder::new(source_path, source).finish();
     _compile_with_syntax_warning_handler(source_file, mode, opts, Some(syntax_warning_handler))
@@ -2461,6 +2453,7 @@ pub fn compile_symtable(
     mode: Mode,
     source_path: &str,
 ) -> Result<symboltable::SymbolTable, CompileError> {
+    let source = core::normalize_newlines(source);
     let source_file = SourceFileBuilder::new(source_path, source).finish();
     _compile_symtable(source_file, mode)
 }

diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs
@@ -81,6 +81,8 @@ impl VmCompileError {
 
 impl CompileWarningError {
     fn into_pyexception(self, vm: &VirtualMachine, source: Option<&str>) -> PyBaseExceptionRef {
+        let source = source.map(compiler::core::normalize_newlines);
+        let source = source.as_deref();
         if !self
             .exception
             .fast_isinstance(vm.ctx.exceptions.syntax_warning)
@@ -463,10 +465,9 @@ impl VirtualMachine {
         };
         // Normalize the source before parsing so literal values, warning ranges,
         // and diagnostic text all refer to the same universal-newline input.
-        Ok(if decoded.contains('\r') {
-            decoded.replace("\r\n", "\n").replace('\r', "\n")
-        } else {
-            decoded
+        Ok(match compiler::core::normalize_newlines(&decoded) {
+            alloc::borrow::Cow::Borrowed(_) => decoded,
+            alloc::borrow::Cow::Owned(normalized) => normalized,
         })
     }
 
@@ -722,6 +723,8 @@ impl VirtualMachine {
         original_filename: &Py<PyStr>,
         module: Option<&Py<PyStr>>,
     ) -> Result<compiler::codegen::symboltable::SymbolTable, VmCompileError> {
+        let source = compiler::core::normalize_newlines(source);
+        let source = source.as_ref();
         #[cfg(feature = "parser")]
         {
             let start = match mode {
@@ -815,6 +818,8 @@ impl VirtualMachine {
     ) -> Result<PyRef<PyCode>, VmCompileError> {
         #[cfg(not(feature = "parser"))]
         let _ = (module, emitted_escapes);
+        let source = compiler::core::normalize_newlines(source);
+        let source = source.as_ref();
         #[cfg(feature = "parser")]
         {
             self.emit_standard_parser_warnings(

diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs
@@ -510,6 +510,9 @@ impl VirtualMachine {
             return self.new_recursion_error(error.to_string());
         }
 
+        let source = source.map(crate::compiler::core::normalize_newlines);
+        let source = source.as_deref();
+
         let incomplete_or_syntax = |allow| -> &'static Py<crate::builtins::PyType> {
             if allow {
                 self.ctx.exceptions.incomplete_input_error

diff --git a/extra_tests/snippets/builtin_compile.py b/extra_tests/snippets/builtin_compile.py
@@ -250,3 +250,61 @@ def _expect_syntax(source, mode):
             raise AssertionError("initial eval indentation was accepted")
 
 assert eval(compile("(\n  1\n)", "<test>", "eval")) == 1
+
+# Physical CR and CRLF are translated before lexing, even inside literals.
+# Keep explicit backslash-r escapes distinct from physical line endings.
+for ending in ("\r", "\r\n", "\n\r", "\r\r\n"):
+    normalized = ending.replace("\r\n", "\n").replace("\r", "\n")
+    for prefix in ("", "r", "b", "f"):
+        source = prefix + '"""a' + ending + 'b"""'
+        expected = "a" + normalized + "b"
+        if prefix == "b":
+            expected = expected.encode()
+        assert eval(source) == expected
+        assert eval(source.encode()) == expected
+        assert eval(compile(source, "<newlines>", "eval")) == expected
+        tree = ast.parse(source, mode="eval")
+        assert eval(compile(tree, "<newlines>", "eval")) == expected
+        namespace = {}
+        exec("value = " + source, namespace)
+        assert namespace["value"] == expected
+    assert eval('t"""a' + ending + 'b"""').strings == ("a" + normalized + "b",)
+    namespace = {}
+    exec('def f():\r    """a' + ending + '    b"""\r    pass', namespace)
+    assert namespace["f"].__doc__ == "a" + normalized + "b"
+    assert eval('"a\\' + ending[:1] + 'b"') == "ab"
+assert eval(r'"a\rb"') == "a\rb"
+assert eval(r'r"a\rb"') == r"a\rb"
+
+# Warnings and SyntaxError text must use the same source as the parser.
+import warnings
+
+for source in ('"""a\r\\q"""', '"""a\r\\q""" + )', "1\r+", '"a\rb"'):
+    results = []
+    for text in (source, source.replace("\r", "\n")):
+        with warnings.catch_warnings(record=True) as captured:
+            warnings.simplefilter("always")
+            try:
+                eval(text)
+            except SyntaxError as exc:
+                result = (exc.msg, exc.lineno, exc.offset, exc.text)
+            else:
+                result = None
+            results.append((result, [(str(w.message), w.lineno) for w in captured]))
+    assert results[0] == results[1], (source, results)
+
+# Audit hooks see the caller's bytes, not the parser's translated buffer.
+import sys
+
+newline_audit_events = []
+newline_audit_source = '"""audit\rsource"""'
+
+
+def record_newline_compile(event, args):
+    if event == "compile" and args[0] == newline_audit_source.encode():
+        newline_audit_events.append(args[0])
+
+
+sys.addaudithook(record_newline_compile)
+assert eval(newline_audit_source) == "audit\nsource"
+assert newline_audit_events == [newline_audit_source.encode()]