Skip to content

Commit 7f4d308

Browse files
youknowoneyouknowone
authored
Implement maybe_pyc_file and .pyc file execution (RustPython#6539)
* Implement maybe_pyc_file and .pyc file execution * unmark failing tests
1 parent 4a6e8fb commit 7f4d308

8 files changed

Lines changed: 69 additions & 159 deletions

File tree

‎Lib/test/test_cmd_line_script.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -247,8 +247,6 @@ def test_script_abspath(self):
247247
script_dir, None,
248248
importlib.machinery.SourceFileLoader)
249249

250-
# TODO: RUSTPYTHON
251-
@unittest.expectedFailure
252250
def test_script_compiled(self):
253251
with os_helper.temp_dir() as script_dir:
254252
script_name = _make_test_script(script_dir, 'script')

‎Lib/test/test_compileall.py‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -325,8 +325,6 @@ def _test_ddir_only(self, *, ddir, parallel=True):
325325
self.assertEqual(mod_code_obj.co_filename, expected_in)
326326
self.assertIn(f'"{expected_in}"', os.fsdecode(err))
327327

328-
# TODO: RUSTPYTHON
329-
@unittest.expectedFailure
330328
def test_ddir_only_one_worker(self):
331329
"""Recursive compile_dir ddir= contains package paths; bpo39769."""
332330
return self._test_ddir_only(ddir="<a prefix>", parallel=False)
@@ -336,8 +334,6 @@ def test_ddir_multiple_workers(self):
336334
"""Recursive compile_dir ddir= contains package paths; bpo39769."""
337335
return self._test_ddir_only(ddir="<a prefix>", parallel=True)
338336

339-
# TODO: RUSTPYTHON
340-
@unittest.expectedFailure
341337
def test_ddir_empty_only_one_worker(self):
342338
"""Recursive compile_dir ddir='' contains package paths; bpo39769."""
343339
return self._test_ddir_only(ddir="", parallel=False)
@@ -347,8 +343,6 @@ def test_ddir_empty_multiple_workers(self):
347343
"""Recursive compile_dir ddir='' contains package paths; bpo39769."""
348344
return self._test_ddir_only(ddir="", parallel=True)
349345

350-
# TODO: RUSTPYTHON
351-
@unittest.expectedFailure
352346
def test_strip_only(self):
353347
fullpath = ["test", "build", "real", "path"]
354348
path = os.path.join(self.directory, *fullpath)
@@ -408,8 +402,6 @@ def test_prepend_only(self):
408402
str(err, encoding=sys.getdefaultencoding())
409403
)
410404

411-
# TODO: RUSTPYTHON
412-
@unittest.expectedFailure
413405
def test_strip_and_prepend(self):
414406
fullpath = ["test", "build", "real", "path"]
415407
path = os.path.join(self.directory, *fullpath)
@@ -887,8 +879,6 @@ def test_workers_available_cores(self, compile_dir):
887879
self.assertTrue(compile_dir.called)
888880
self.assertEqual(compile_dir.call_args[-1]['workers'], 0)
889881

890-
# TODO: RUSTPYTHON
891-
@unittest.expectedFailure
892882
def test_strip_and_prepend(self):
893883
fullpath = ["test", "build", "real", "path"]
894884
path = os.path.join(self.directory, *fullpath)

‎Lib/test/test_importlib/source/test_file_loader.py‎

Lines changed: 0 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -359,24 +359,6 @@ def test_overridden_unchecked_hash_based_pyc(self):
359359
) = util.test_both(SimpleTest, importlib=importlib, machinery=machinery,
360360
abc=importlib_abc, util=importlib_util)
361361

362-
# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed
363-
class Source_SimpleTest(Source_SimpleTest):
364-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
365-
@unittest.expectedFailure
366-
def test_checked_hash_based_pyc(self):
367-
super().test_checked_hash_based_pyc()
368-
369-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
370-
@unittest.expectedFailure
371-
def test_unchecked_hash_based_pyc(self):
372-
super().test_unchecked_hash_based_pyc()
373-
374-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
375-
@unittest.expectedFailure
376-
def test_overridden_unchecked_hash_based_pyc(self):
377-
super().test_overridden_unchecked_hash_based_pyc()
378-
379-
380362
class SourceDateEpochTestMeta(SourceDateEpochTestMeta,
381363
type(Source_SimpleTest)):
382364
pass
@@ -697,24 +679,6 @@ class SourceLoaderBadBytecodeTestPEP451(
697679
machinery=machinery, abc=importlib_abc,
698680
util=importlib_util)
699681

700-
# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed
701-
class Source_SourceBadBytecodePEP451(Source_SourceBadBytecodePEP451):
702-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
703-
@unittest.expectedFailure
704-
def test_bad_marshal(self):
705-
super().test_bad_marshal()
706-
707-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
708-
@unittest.expectedFailure
709-
def test_no_marshal(self):
710-
super().test_no_marshal()
711-
712-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
713-
@unittest.expectedFailure
714-
def test_non_code_marshal(self):
715-
super().test_non_code_marshal()
716-
717-
718682
class SourceLoaderBadBytecodeTestPEP302(
719683
SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302):
720684
pass
@@ -726,24 +690,6 @@ class SourceLoaderBadBytecodeTestPEP302(
726690
machinery=machinery, abc=importlib_abc,
727691
util=importlib_util)
728692

729-
# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed
730-
class Source_SourceBadBytecodePEP302(Source_SourceBadBytecodePEP302):
731-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
732-
@unittest.expectedFailure
733-
def test_bad_marshal(self):
734-
super().test_bad_marshal()
735-
736-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
737-
@unittest.expectedFailure
738-
def test_no_marshal(self):
739-
super().test_no_marshal()
740-
741-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
742-
@unittest.expectedFailure
743-
def test_non_code_marshal(self):
744-
super().test_non_code_marshal()
745-
746-
747693
class SourcelessLoaderBadBytecodeTest:
748694

749695
@classmethod
@@ -829,39 +775,6 @@ class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest,
829775
machinery=machinery, abc=importlib_abc,
830776
util=importlib_util)
831777

832-
# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed
833-
class Source_SourcelessBadBytecodePEP451(Source_SourcelessBadBytecodePEP451):
834-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
835-
@unittest.expectedFailure
836-
def test_magic_only(self):
837-
super().test_magic_only()
838-
839-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
840-
@unittest.expectedFailure
841-
def test_no_marshal(self):
842-
super().test_no_marshal()
843-
844-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
845-
@unittest.expectedFailure
846-
def test_partial_flags(self):
847-
super().test_partial_flags()
848-
849-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
850-
@unittest.expectedFailure
851-
def test_partial_hash(self):
852-
super().test_partial_hash()
853-
854-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
855-
@unittest.expectedFailure
856-
def test_partial_size(self):
857-
super().test_partial_size()
858-
859-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
860-
@unittest.expectedFailure
861-
def test_partial_timestamp(self):
862-
super().test_partial_timestamp()
863-
864-
865778
class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest,
866779
BadBytecodeTestPEP302):
867780
pass
@@ -873,38 +786,5 @@ class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest,
873786
machinery=machinery, abc=importlib_abc,
874787
util=importlib_util)
875788

876-
# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed
877-
class Source_SourcelessBadBytecodePEP302(Source_SourcelessBadBytecodePEP302):
878-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
879-
@unittest.expectedFailure
880-
def test_magic_only(self):
881-
super().test_magic_only()
882-
883-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
884-
@unittest.expectedFailure
885-
def test_no_marshal(self):
886-
super().test_no_marshal()
887-
888-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
889-
@unittest.expectedFailure
890-
def test_partial_flags(self):
891-
super().test_partial_flags()
892-
893-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
894-
@unittest.expectedFailure
895-
def test_partial_hash(self):
896-
super().test_partial_hash()
897-
898-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
899-
@unittest.expectedFailure
900-
def test_partial_size(self):
901-
super().test_partial_size()
902-
903-
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
904-
@unittest.expectedFailure
905-
def test_partial_timestamp(self):
906-
super().test_partial_timestamp()
907-
908-
909789
if __name__ == '__main__':
910790
unittest.main()

‎Lib/test/test_importlib/test_util.py‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -327,15 +327,6 @@ def test_incorporates_rn(self):
327327
) = util.test_both(MagicNumberTests, util=importlib_util)
328328

329329

330-
# TODO: RUSTPYTHON
331-
@unittest.expectedFailure
332-
def test_incorporates_rn_MONKEYPATCH(self):
333-
self.assertTrue(self.util.MAGIC_NUMBER.endswith(b'\r\n'))
334-
335-
# TODO: RUSTPYTHON
336-
Frozen_MagicNumberTests.test_incorporates_rn = test_incorporates_rn_MONKEYPATCH
337-
338-
339330
class PEP3147Tests:
340331

341332
"""Tests of PEP 3147-related functions: cache_from_source and source_from_cache."""

‎crates/vm/src/import.rs‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ use crate::{
55
builtins::{PyCode, list, traceback::PyTraceback},
66
exceptions::types::PyBaseException,
77
scope::Scope,
8-
version::get_git_revision,
98
vm::{VirtualMachine, thread},
109
};
1110

@@ -44,16 +43,6 @@ pub(crate) fn init_importlib_package(vm: &VirtualMachine, importlib: PyObjectRef
4443

4544
let install_external = importlib.get_attr("_install_external_importers", vm)?;
4645
install_external.call((), vm)?;
47-
// Set pyc magic number to commit hash. Should be changed when bytecode will be more stable.
48-
let importlib_external = vm.import("_frozen_importlib_external", 0)?;
49-
let mut magic = get_git_revision().into_bytes();
50-
magic.truncate(4);
51-
if magic.len() != 4 {
52-
// os_random is expensive, but this is only ever called once
53-
magic = rustpython_common::rand::os_random::<4>().to_vec();
54-
}
55-
let magic: PyObjectRef = vm.ctx.new_bytes(magic).into();
56-
importlib_external.set_attr("MAGIC_NUMBER", magic, vm)?;
5746
let zipimport_res = (|| -> PyResult<()> {
5847
let zipimport = vm.import("zipimport", 0)?;
5948
let zipimporter = zipimport.get_attr("zipimporter", vm)?;

‎crates/vm/src/stdlib/imp.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ mod _imp {
8585
PyObjectRef, PyRef, PyResult, VirtualMachine,
8686
builtins::{PyBytesRef, PyCode, PyMemoryView, PyModule, PyStrRef},
8787
function::OptionalArg,
88-
import,
88+
import, version,
8989
};
9090

9191
#[pyattr]
@@ -94,6 +94,9 @@ mod _imp {
9494
.new_str(vm.state.config.settings.check_hash_pycs_mode.to_string())
9595
}
9696

97+
#[pyattr(name = "pyc_magic_number_token")]
98+
use version::PYC_MAGIC_NUMBER_TOKEN;
99+
97100
#[pyfunction]
98101
const fn extension_suffixes() -> PyResult<Vec<PyObjectRef>> {
99102
Ok(Vec::new())

‎crates/vm/src/version.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,3 +126,14 @@ pub fn get_git_datetime() -> String {
126126

127127
format!("{date} {time}")
128128
}
129+
130+
// Must be aligned to Lib/importlib/_bootstrap_external.py
131+
pub const PYC_MAGIC_NUMBER: u16 = 3531;
132+
133+
// CPython format: magic_number | ('\r' << 16) | ('\n' << 24)
134+
// This protects against text-mode file reads
135+
pub const PYC_MAGIC_NUMBER_TOKEN: u32 =
136+
(PYC_MAGIC_NUMBER as u32) | ((b'\r' as u32) << 16) | ((b'\n' as u32) << 24);
137+
138+
/// Magic number as little-endian bytes for .pyc files
139+
pub const PYC_MAGIC_NUMBER_BYTES: [u8; 4] = PYC_MAGIC_NUMBER_TOKEN.to_le_bytes();

‎crates/vm/src/vm/compile.rs‎

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -75,13 +75,20 @@ impl VirtualMachine {
7575
// Consider to use enum to distinguish `path`
7676
// https://github.com/RustPython/RustPython/pull/6276#discussion_r2529849479
7777

78-
// TODO: check .pyc here
79-
let pyc = false;
78+
let pyc = maybe_pyc_file(path);
8079
if pyc {
81-
todo!("running pyc is not implemented yet");
80+
// pyc file execution
81+
set_main_loader(&module_dict, path, "SourcelessFileLoader", self)?;
82+
let loader = module_dict.get_item("__loader__", self)?;
83+
let get_code = loader.get_attr("get_code", self)?;
84+
let code_obj = get_code.call((identifier!(self, __main__).to_owned(),), self)?;
85+
let code = code_obj
86+
.downcast::<PyCode>()
87+
.map_err(|_| self.new_runtime_error("Bad code object in .pyc file".to_owned()))?;
88+
self.run_code_obj(code, scope)?;
8289
} else {
8390
if path != "<stdin>" {
84-
set_main_loader(&module_dict, path, self)?;
91+
set_main_loader(&module_dict, path, "SourceFileLoader", self)?;
8592
}
8693
// TODO: replace to something equivalent to py_run_file
8794
match std::fs::read_to_string(path) {
@@ -125,16 +132,57 @@ impl VirtualMachine {
125132
}
126133
}
127134

128-
fn set_main_loader(module_dict: &PyDictRef, filename: &str, vm: &VirtualMachine) -> PyResult<()> {
135+
fn set_main_loader(
136+
module_dict: &PyDictRef,
137+
filename: &str,
138+
loader_name: &str,
139+
vm: &VirtualMachine,
140+
) -> PyResult<()> {
129141
vm.import("importlib.machinery", 0)?;
130142
let sys_modules = vm.sys_module.get_attr(identifier!(vm, modules), vm)?;
131143
let machinery = sys_modules.get_item("importlib.machinery", vm)?;
132-
let loader_class = machinery.get_attr("SourceFileLoader", vm)?;
144+
let loader_name = vm.ctx.new_str(loader_name);
145+
let loader_class = machinery.get_attr(&loader_name, vm)?;
133146
let loader = loader_class.call((identifier!(vm, __main__).to_owned(), filename), vm)?;
134147
module_dict.set_item("__loader__", loader, vm)?;
135148
Ok(())
136149
}
137150

151+
/// Check whether a file is maybe a pyc file.
152+
///
153+
/// Detection is performed by:
154+
/// 1. Checking if the filename ends with ".pyc"
155+
/// 2. If not, reading the first 2 bytes and comparing with the magic number
156+
fn maybe_pyc_file(path: &str) -> bool {
157+
// 1. Check if filename ends with ".pyc"
158+
if path.ends_with(".pyc") {
159+
return true;
160+
}
161+
maybe_pyc_file_with_magic(path, &crate::version::PYC_MAGIC_NUMBER_BYTES).unwrap_or(false)
162+
}
163+
164+
fn maybe_pyc_file_with_magic(path: &str, magic_number: &[u8]) -> std::io::Result<bool> {
165+
// part of maybe_pyc_file
166+
// For non-.pyc extension, check magic number
167+
let path_obj = std::path::Path::new(path);
168+
if !path_obj.is_file() {
169+
return Ok(false);
170+
}
171+
172+
let mut file = std::fs::File::open(path)?;
173+
let mut buf = [0u8; 2];
174+
175+
use std::io::Read;
176+
if file.read(&mut buf)? != 2 || magic_number.len() < 2 {
177+
return Ok(false);
178+
}
179+
180+
// Read only two bytes of the magic. If the file was opened in
181+
// text mode, the bytes 3 and 4 of the magic (\r\n) might not
182+
// be read as they are on disk.
183+
Ok(buf == magic_number[..2])
184+
}
185+
138186
fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> {
139187
let path_importer_cache = vm.sys_module.get_attr("path_importer_cache", vm)?;
140188
let path_importer_cache = PyDictRef::try_from_object(vm, path_importer_cache)?;

0 commit comments

Comments
 (0)