diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index d773674feef..187d21a182e 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -247,8 +247,6 @@ def test_script_abspath(self): script_dir, None, importlib.machinery.SourceFileLoader) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_script_compiled(self): with os_helper.temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script') diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index f84e67e9cf6..9fa3dbc47e5 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -325,8 +325,6 @@ def _test_ddir_only(self, *, ddir, parallel=True): self.assertEqual(mod_code_obj.co_filename, expected_in) self.assertIn(f'"{expected_in}"', os.fsdecode(err)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ddir_only_one_worker(self): """Recursive compile_dir ddir= contains package paths; bpo39769.""" return self._test_ddir_only(ddir="", parallel=False) @@ -336,8 +334,6 @@ def test_ddir_multiple_workers(self): """Recursive compile_dir ddir= contains package paths; bpo39769.""" return self._test_ddir_only(ddir="", parallel=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ddir_empty_only_one_worker(self): """Recursive compile_dir ddir='' contains package paths; bpo39769.""" return self._test_ddir_only(ddir="", parallel=False) @@ -347,8 +343,6 @@ def test_ddir_empty_multiple_workers(self): """Recursive compile_dir ddir='' contains package paths; bpo39769.""" return self._test_ddir_only(ddir="", parallel=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strip_only(self): fullpath = ["test", "build", "real", "path"] path = os.path.join(self.directory, *fullpath) @@ -408,8 +402,6 @@ def test_prepend_only(self): str(err, encoding=sys.getdefaultencoding()) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strip_and_prepend(self): fullpath = ["test", "build", "real", "path"] path = os.path.join(self.directory, *fullpath) @@ -887,8 +879,6 @@ def test_workers_available_cores(self, compile_dir): self.assertTrue(compile_dir.called) self.assertEqual(compile_dir.call_args[-1]['workers'], 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strip_and_prepend(self): fullpath = ["test", "build", "real", "path"] path = os.path.join(self.directory, *fullpath) diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index d487fc9b82d..fa0efee8da6 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -359,24 +359,6 @@ def test_overridden_unchecked_hash_based_pyc(self): ) = util.test_both(SimpleTest, importlib=importlib, machinery=machinery, abc=importlib_abc, util=importlib_util) -# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed -class Source_SimpleTest(Source_SimpleTest): - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_checked_hash_based_pyc(self): - super().test_checked_hash_based_pyc() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_unchecked_hash_based_pyc(self): - super().test_unchecked_hash_based_pyc() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_overridden_unchecked_hash_based_pyc(self): - super().test_overridden_unchecked_hash_based_pyc() - - class SourceDateEpochTestMeta(SourceDateEpochTestMeta, type(Source_SimpleTest)): pass @@ -697,24 +679,6 @@ class SourceLoaderBadBytecodeTestPEP451( machinery=machinery, abc=importlib_abc, util=importlib_util) -# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed -class Source_SourceBadBytecodePEP451(Source_SourceBadBytecodePEP451): - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_bad_marshal(self): - super().test_bad_marshal() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_no_marshal(self): - super().test_no_marshal() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_non_code_marshal(self): - super().test_non_code_marshal() - - class SourceLoaderBadBytecodeTestPEP302( SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): pass @@ -726,24 +690,6 @@ class SourceLoaderBadBytecodeTestPEP302( machinery=machinery, abc=importlib_abc, util=importlib_util) -# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed -class Source_SourceBadBytecodePEP302(Source_SourceBadBytecodePEP302): - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_bad_marshal(self): - super().test_bad_marshal() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_no_marshal(self): - super().test_no_marshal() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_non_code_marshal(self): - super().test_non_code_marshal() - - class SourcelessLoaderBadBytecodeTest: @classmethod @@ -829,39 +775,6 @@ class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, machinery=machinery, abc=importlib_abc, util=importlib_util) -# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed -class Source_SourcelessBadBytecodePEP451(Source_SourcelessBadBytecodePEP451): - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_magic_only(self): - super().test_magic_only() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_no_marshal(self): - super().test_no_marshal() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_flags(self): - super().test_partial_flags() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_hash(self): - super().test_partial_hash() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_size(self): - super().test_partial_size() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_timestamp(self): - super().test_partial_timestamp() - - class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, BadBytecodeTestPEP302): pass @@ -873,38 +786,5 @@ class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, machinery=machinery, abc=importlib_abc, util=importlib_util) -# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed -class Source_SourcelessBadBytecodePEP302(Source_SourcelessBadBytecodePEP302): - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_magic_only(self): - super().test_magic_only() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_no_marshal(self): - super().test_no_marshal() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_flags(self): - super().test_partial_flags() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_hash(self): - super().test_partial_hash() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_size(self): - super().test_partial_size() - - # TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed - @unittest.expectedFailure - def test_partial_timestamp(self): - super().test_partial_timestamp() - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 201f5069114..87b7de9889b 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -327,15 +327,6 @@ def test_incorporates_rn(self): ) = util.test_both(MagicNumberTests, util=importlib_util) -# TODO: RUSTPYTHON -@unittest.expectedFailure -def test_incorporates_rn_MONKEYPATCH(self): - self.assertTrue(self.util.MAGIC_NUMBER.endswith(b'\r\n')) - -# TODO: RUSTPYTHON -Frozen_MagicNumberTests.test_incorporates_rn = test_incorporates_rn_MONKEYPATCH - - class PEP3147Tests: """Tests of PEP 3147-related functions: cache_from_source and source_from_cache.""" diff --git a/crates/vm/src/import.rs b/crates/vm/src/import.rs index 39748655e0f..f970a335aa5 100644 --- a/crates/vm/src/import.rs +++ b/crates/vm/src/import.rs @@ -5,7 +5,6 @@ use crate::{ builtins::{PyCode, list, traceback::PyTraceback}, exceptions::types::PyBaseException, scope::Scope, - version::get_git_revision, vm::{VirtualMachine, thread}, }; @@ -44,16 +43,6 @@ pub(crate) fn init_importlib_package(vm: &VirtualMachine, importlib: PyObjectRef let install_external = importlib.get_attr("_install_external_importers", vm)?; install_external.call((), vm)?; - // Set pyc magic number to commit hash. Should be changed when bytecode will be more stable. - let importlib_external = vm.import("_frozen_importlib_external", 0)?; - let mut magic = get_git_revision().into_bytes(); - magic.truncate(4); - if magic.len() != 4 { - // os_random is expensive, but this is only ever called once - magic = rustpython_common::rand::os_random::<4>().to_vec(); - } - let magic: PyObjectRef = vm.ctx.new_bytes(magic).into(); - importlib_external.set_attr("MAGIC_NUMBER", magic, vm)?; let zipimport_res = (|| -> PyResult<()> { let zipimport = vm.import("zipimport", 0)?; let zipimporter = zipimport.get_attr("zipimporter", vm)?; diff --git a/crates/vm/src/stdlib/imp.rs b/crates/vm/src/stdlib/imp.rs index 76b3bfd124c..df32b0c0068 100644 --- a/crates/vm/src/stdlib/imp.rs +++ b/crates/vm/src/stdlib/imp.rs @@ -85,7 +85,7 @@ mod _imp { PyObjectRef, PyRef, PyResult, VirtualMachine, builtins::{PyBytesRef, PyCode, PyMemoryView, PyModule, PyStrRef}, function::OptionalArg, - import, + import, version, }; #[pyattr] @@ -94,6 +94,9 @@ mod _imp { .new_str(vm.state.config.settings.check_hash_pycs_mode.to_string()) } + #[pyattr(name = "pyc_magic_number_token")] + use version::PYC_MAGIC_NUMBER_TOKEN; + #[pyfunction] const fn extension_suffixes() -> PyResult> { Ok(Vec::new()) diff --git a/crates/vm/src/version.rs b/crates/vm/src/version.rs index deb3dccd535..0a598842a56 100644 --- a/crates/vm/src/version.rs +++ b/crates/vm/src/version.rs @@ -126,3 +126,14 @@ pub fn get_git_datetime() -> String { format!("{date} {time}") } + +// Must be aligned to Lib/importlib/_bootstrap_external.py +pub const PYC_MAGIC_NUMBER: u16 = 3531; + +// CPython format: magic_number | ('\r' << 16) | ('\n' << 24) +// This protects against text-mode file reads +pub const PYC_MAGIC_NUMBER_TOKEN: u32 = + (PYC_MAGIC_NUMBER as u32) | ((b'\r' as u32) << 16) | ((b'\n' as u32) << 24); + +/// Magic number as little-endian bytes for .pyc files +pub const PYC_MAGIC_NUMBER_BYTES: [u8; 4] = PYC_MAGIC_NUMBER_TOKEN.to_le_bytes(); diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index a7e31cf0377..b5f10e47aa1 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -75,13 +75,20 @@ impl VirtualMachine { // Consider to use enum to distinguish `path` // https://github.com/RustPython/RustPython/pull/6276#discussion_r2529849479 - // TODO: check .pyc here - let pyc = false; + let pyc = maybe_pyc_file(path); if pyc { - todo!("running pyc is not implemented yet"); + // pyc file execution + set_main_loader(&module_dict, path, "SourcelessFileLoader", self)?; + let loader = module_dict.get_item("__loader__", self)?; + let get_code = loader.get_attr("get_code", self)?; + let code_obj = get_code.call((identifier!(self, __main__).to_owned(),), self)?; + let code = code_obj + .downcast::() + .map_err(|_| self.new_runtime_error("Bad code object in .pyc file".to_owned()))?; + self.run_code_obj(code, scope)?; } else { if path != "" { - set_main_loader(&module_dict, path, self)?; + set_main_loader(&module_dict, path, "SourceFileLoader", self)?; } // TODO: replace to something equivalent to py_run_file match std::fs::read_to_string(path) { @@ -125,16 +132,57 @@ impl VirtualMachine { } } -fn set_main_loader(module_dict: &PyDictRef, filename: &str, vm: &VirtualMachine) -> PyResult<()> { +fn set_main_loader( + module_dict: &PyDictRef, + filename: &str, + loader_name: &str, + vm: &VirtualMachine, +) -> PyResult<()> { vm.import("importlib.machinery", 0)?; let sys_modules = vm.sys_module.get_attr(identifier!(vm, modules), vm)?; let machinery = sys_modules.get_item("importlib.machinery", vm)?; - let loader_class = machinery.get_attr("SourceFileLoader", vm)?; + let loader_name = vm.ctx.new_str(loader_name); + let loader_class = machinery.get_attr(&loader_name, vm)?; let loader = loader_class.call((identifier!(vm, __main__).to_owned(), filename), vm)?; module_dict.set_item("__loader__", loader, vm)?; Ok(()) } +/// Check whether a file is maybe a pyc file. +/// +/// Detection is performed by: +/// 1. Checking if the filename ends with ".pyc" +/// 2. If not, reading the first 2 bytes and comparing with the magic number +fn maybe_pyc_file(path: &str) -> bool { + // 1. Check if filename ends with ".pyc" + if path.ends_with(".pyc") { + return true; + } + maybe_pyc_file_with_magic(path, &crate::version::PYC_MAGIC_NUMBER_BYTES).unwrap_or(false) +} + +fn maybe_pyc_file_with_magic(path: &str, magic_number: &[u8]) -> std::io::Result { + // part of maybe_pyc_file + // For non-.pyc extension, check magic number + let path_obj = std::path::Path::new(path); + if !path_obj.is_file() { + return Ok(false); + } + + let mut file = std::fs::File::open(path)?; + let mut buf = [0u8; 2]; + + use std::io::Read; + if file.read(&mut buf)? != 2 || magic_number.len() < 2 { + return Ok(false); + } + + // Read only two bytes of the magic. If the file was opened in + // text mode, the bytes 3 and 4 of the magic (\r\n) might not + // be read as they are on disk. + Ok(buf == magic_number[..2]) +} + fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult> { let path_importer_cache = vm.sys_module.get_attr("path_importer_cache", vm)?; let path_importer_cache = PyDictRef::try_from_object(vm, path_importer_cache)?;