Skip to content

Commit 91cf882

Browse files
brettcannonbrettcannon
committed
Refactor source and bytecode file loaders in importlib so that there
are source-only and source/bytecode loaders.
1 parent 0515619 commit 91cf882

File tree

4 files changed

+164
-150
lines changed

4 files changed

+164
-150
lines changed

‎Lib/importlib/NOTES‎

Lines changed: 2 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,6 @@
11
to do
22
/////
33

4-
* Refactor source/bytecode finder/loader code such that bytecode support is a
5-
subclass of source support (makes it nicer for VMs that don't use CPython
6-
bytecode).
7-
8-
+ PyLoader (for ABC)
9-
10-
- load_module for source only
11-
- get_code for source only
12-
13-
+ PyFileLoader(PyLoader)
14-
15-
- get_data
16-
- source_mtime
17-
- source_path
18-
19-
+PyPycLoader (PyLoader, for ABC)
20-
21-
- load_module for source and bytecode
22-
- get_code for source and bytecode
23-
24-
+ PyPycFileLoader(PyPycLoader, PyFileLoader)
25-
26-
- bytecode_path
27-
- write_bytecode
28-
294
* Implement PEP 302 protocol for loaders (should just be a matter of testing).
305

316
+ Source/bytecode.
@@ -42,7 +17,6 @@ to do
4217

4318
* load_module
4419

45-
- (?) Importer(Finder, Loader)
4620
- ResourceLoader(Loader)
4721

4822
* get_data
@@ -89,6 +63,8 @@ to do
8963
* Add leading underscores to all objects in importlib._bootstrap that are not
9064
publicly exposed.
9165

66+
* Reorder importlib/_bootstrap.py so definitions are not in inverted order.
67+
9268
* Make sure that there is documentation *somewhere* fully explaining the
9369
semantics of import that can be referenced from the package's documentation
9470
(even if it is in the package documentation itself, although it might be best

‎Lib/importlib/_bootstrap.py‎

Lines changed: 149 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -315,17 +315,124 @@ def decorated(self, fullname):
315315
return decorated
316316

317317

318-
class _PyFileLoader:
319-
# XXX Still smart to have this as a separate class? Or would it work
320-
# better to integrate with PyFileFinder? Could cache _is_pkg info.
321-
# FileFinder can be changed to return self instead of a specific loader
322-
# call. Otherwise _base_path can be calculated on the fly without issue if
323-
# it is known whether a module should be treated as a path or package to
324-
# minimize stat calls. Could even go as far as to stat the directory the
325-
# importer is in to detect changes and then cache all the info about what
326-
# files were found (if stating directories is platform-dependent).
327-
328-
"""Load a Python source or bytecode file."""
318+
class PyLoader:
319+
320+
"""Loader base class for Python source.
321+
322+
Requires implementing the optional PEP 302 protocols as well as
323+
source_mtime and source_path.
324+
325+
"""
326+
327+
@module_for_loader
328+
def load_module(self, module):
329+
"""Load a source module."""
330+
return _load_module(module)
331+
332+
def _load_module(self, module):
333+
"""Initialize a module from source."""
334+
name = module.__name__
335+
source_path = self.source_path(name)
336+
code_object = self.get_code(module.__name__)
337+
if not hasattr(module, '__file__'):
338+
module.__file__ = source_path
339+
if self.is_package(name):
340+
module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]]
341+
module.__package__ = module.__name__
342+
if not hasattr(module, '__path__'):
343+
module.__package__ = module.__package__.rpartition('.')[0]
344+
exec(code_object, module.__dict__)
345+
return module
346+
347+
def get_code(self, fullname):
348+
"""Get a code object from source."""
349+
source_path = self.source_path(fullname)
350+
source = self.get_data(source_path)
351+
# Convert to universal newlines.
352+
line_endings = b'\n'
353+
for index, c in enumerate(source):
354+
if c == ord(b'\n'):
355+
break
356+
elif c == ord(b'\r'):
357+
line_endings = b'\r'
358+
try:
359+
if source[index+1] == ord(b'\n'):
360+
line_endings += b'\n'
361+
except IndexError:
362+
pass
363+
break
364+
if line_endings != b'\n':
365+
source = source.replace(line_endings, b'\n')
366+
return compile(source, source_path, 'exec', dont_inherit=True)
367+
368+
369+
class PyPycLoader(PyLoader):
370+
371+
"""Loader base class for Python source and bytecode.
372+
373+
Requires implementing the methods needed for PyLoader as well as
374+
bytecode_path and write_bytecode.
375+
376+
"""
377+
378+
@module_for_loader
379+
def load_module(self, module):
380+
"""Load a module from source or bytecode."""
381+
name = module.__name__
382+
source_path = self.source_path(name)
383+
bytecode_path = self.bytecode_path(name)
384+
module.__file__ = source_path if source_path else bytecode_path
385+
return self._load_module(module)
386+
387+
def get_code(self, fullname):
388+
"""Get a code object from source or bytecode."""
389+
# XXX Care enough to make sure this call does not happen if the magic
390+
# number is bad?
391+
source_timestamp = self.source_mtime(fullname)
392+
# Try to use bytecode if it is available.
393+
bytecode_path = self.bytecode_path(fullname)
394+
if bytecode_path:
395+
data = self.get_data(bytecode_path)
396+
magic = data[:4]
397+
pyc_timestamp = marshal._r_long(data[4:8])
398+
bytecode = data[8:]
399+
try:
400+
# Verify that the magic number is valid.
401+
if imp.get_magic() != magic:
402+
raise ImportError("bad magic number")
403+
# Verify that the bytecode is not stale (only matters when
404+
# there is source to fall back on.
405+
if source_timestamp:
406+
if pyc_timestamp < source_timestamp:
407+
raise ImportError("bytecode is stale")
408+
except ImportError:
409+
# If source is available give it a shot.
410+
if source_timestamp is not None:
411+
pass
412+
else:
413+
raise
414+
else:
415+
# Bytecode seems fine, so try to use it.
416+
# XXX If the bytecode is ill-formed, would it be beneficial to
417+
# try for using source if available and issue a warning?
418+
return marshal.loads(bytecode)
419+
elif source_timestamp is None:
420+
raise ImportError("no source or bytecode available to create code "
421+
"object for {0!r}".format(fullname))
422+
# Use the source.
423+
code_object = super().get_code(fullname)
424+
# Generate bytecode and write it out.
425+
if not sys.dont_write_bytecode:
426+
data = bytearray(imp.get_magic())
427+
data.extend(marshal._w_long(source_timestamp))
428+
data.extend(marshal.dumps(code_object))
429+
self.write_bytecode(fullname, data)
430+
return code_object
431+
432+
433+
class PyFileLoader(PyLoader):
434+
435+
"""Load a Python source file."""
329436

330437
def __init__(self, name, path, is_pkg):
331438
self._name = name
@@ -354,29 +461,6 @@ def source_path(self, fullname):
354461
# Not a property so that it is easy to override.
355462
return self._find_path(imp.PY_SOURCE)
356463

357-
@check_name
358-
def bytecode_path(self, fullname):
359-
"""Return the path to a bytecode file, or None if one does not
360-
exist."""
361-
# Not a property for easy overriding.
362-
return self._find_path(imp.PY_COMPILED)
363-
364-
@module_for_loader
365-
def load_module(self, module):
366-
"""Load a Python source or bytecode module."""
367-
name = module.__name__
368-
source_path = self.source_path(name)
369-
bytecode_path = self.bytecode_path(name)
370-
code_object = self.get_code(module.__name__)
371-
module.__file__ = source_path if source_path else bytecode_path
372-
module.__loader__ = self
373-
if self.is_package(name):
374-
module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]]
375-
module.__package__ = module.__name__
376-
if not hasattr(module, '__path__'):
377-
module.__package__ = module.__package__.rpartition('.')[0]
378-
exec(code_object, module.__dict__)
379-
return module
380464

381465
@check_name
382466
def source_mtime(self, name):
@@ -405,6 +489,34 @@ def get_source(self, fullname):
405489
# anything other than UTF-8.
406490
return open(source_path, encoding=encoding).read()
407491

492+
493+
def get_data(self, path):
494+
"""Return the data from path as raw bytes."""
495+
return _fileio._FileIO(path, 'r').read()
496+
497+
@check_name
498+
def is_package(self, fullname):
499+
"""Return a boolean based on whether the module is a package.
500+
501+
Raises ImportError (like get_source) if the loader cannot handle the
502+
package.
503+
504+
"""
505+
return self._is_pkg
506+
507+
508+
# XXX Rename _PyFileLoader throughout
509+
class PyPycFileLoader(PyPycLoader, PyFileLoader):
510+
511+
"""Load a module from a source or bytecode file."""
512+
513+
@check_name
514+
def bytecode_path(self, fullname):
515+
"""Return the path to a bytecode file, or None if one does not
516+
exist."""
517+
# Not a property for easy overriding.
518+
return self._find_path(imp.PY_COMPILED)
519+
408520
@check_name
409521
def write_bytecode(self, name, data):
410522
"""Write out 'data' for the specified module, returning a boolean
@@ -428,82 +540,6 @@ def write_bytecode(self, name, data):
428540
else:
429541
raise
430542

431-
def get_code(self, name):
432-
"""Return the code object for the module."""
433-
# XXX Care enough to make sure this call does not happen if the magic
434-
# number is bad?
435-
source_timestamp = self.source_mtime(name)
436-
# Try to use bytecode if it is available.
437-
bytecode_path = self.bytecode_path(name)
438-
if bytecode_path:
439-
data = self.get_data(bytecode_path)
440-
magic = data[:4]
441-
pyc_timestamp = marshal._r_long(data[4:8])
442-
bytecode = data[8:]
443-
try:
444-
# Verify that the magic number is valid.
445-
if imp.get_magic() != magic:
446-
raise ImportError("bad magic number")
447-
# Verify that the bytecode is not stale (only matters when
448-
# there is source to fall back on.
449-
if source_timestamp:
450-
if pyc_timestamp < source_timestamp:
451-
raise ImportError("bytcode is stale")
452-
except ImportError:
453-
# If source is available give it a shot.
454-
if source_timestamp is not None:
455-
pass
456-
else:
457-
raise
458-
else:
459-
# Bytecode seems fine, so try to use it.
460-
# XXX If the bytecode is ill-formed, would it be beneficial to
461-
# try for using source if available and issue a warning?
462-
return marshal.loads(bytecode)
463-
elif source_timestamp is None:
464-
raise ImportError("no source or bytecode available to create code "
465-
"object for {0!r}".format(name))
466-
# Use the source.
467-
source_path = self.source_path(name)
468-
source = self.get_data(source_path)
469-
# Convert to universal newlines.
470-
line_endings = b'\n'
471-
for index, c in enumerate(source):
472-
if c == ord(b'\n'):
473-
break
474-
elif c == ord(b'\r'):
475-
line_endings = b'\r'
476-
try:
477-
if source[index+1] == ord(b'\n'):
478-
line_endings += b'\n'
479-
except IndexError:
480-
pass
481-
break
482-
if line_endings != b'\n':
483-
source = source.replace(line_endings, b'\n')
484-
code_object = compile(source, source_path, 'exec', dont_inherit=True)
485-
# Generate bytecode and write it out.
486-
if not sys.dont_write_bytecode:
487-
data = bytearray(imp.get_magic())
488-
data.extend(marshal._w_long(source_timestamp))
489-
data.extend(marshal.dumps(code_object))
490-
self.write_bytecode(name, data)
491-
return code_object
492-
493-
def get_data(self, path):
494-
"""Return the data from path as raw bytes."""
495-
return _fileio._FileIO(path, 'r').read()
496-
497-
@check_name
498-
def is_package(self, fullname):
499-
"""Return a boolean based on whether the module is a package.
500-
501-
Raises ImportError (like get_source) if the loader cannot handle the
502-
package.
503-
504-
"""
505-
return self._is_pkg
506-
507543

508544
class FileFinder:
509545

@@ -583,7 +619,7 @@ class PyFileFinder(FileFinder):
583619
"""Importer for source/bytecode files."""
584620

585621
_possible_package = True
586-
_loader = _PyFileLoader
622+
_loader = PyFileLoader
587623

588624
def __init__(self, path_entry):
589625
# Lack of imp during class creation means _suffixes is set here.
@@ -597,6 +633,8 @@ class PyPycFileFinder(PyFileFinder):
597633

598634
"""Finder for source and bytecode files."""
599635

636+
_loader = PyPycFileLoader
637+
600638
def __init__(self, path_entry):
601639
super().__init__(path_entry)
602640
self._suffixes += suffix_list(imp.PY_COMPILED)

0 commit comments

Comments
 (0)