Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Doc/whatsnew/3.16.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1106,7 +1106,7 @@ Optimizations
Startup performance is tracked on the `Python Speed Center
<https://speed.python.org/timeline/?exe=12&base=&ben=python_startup.legacy&env=1%2C7&revs=50&equid=off&quarts=on&extr=on>`__.
(Contributed by Pablo Galindo Salgado, Cody Maloney, and Pieter Eendebak in
:gh:`159137`, :gh:`160858`, :gh:`160913`, and :gh:`87613`.)
:gh:`159137`, :gh:`160858`, :gh:`160913`, :gh:`87613`, and :gh:`161941`.)

* Speed up iteration over asynchronous generators: values yielded to
``async for`` no longer create and discard a :exc:`StopIteration` exception
Expand Down
26 changes: 22 additions & 4 deletions Lib/importlib/_bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -969,6 +969,7 @@ def _fix_up_module(cls, module):
__path__ = spec.submodule_search_locations
if ispkg:
assert __path__ == [], __path__
pkgdir = cls._package_dir(spec.name, pkgdir)
if pkgdir:
spec.submodule_search_locations.insert(0, pkgdir)
else:
Expand Down Expand Up @@ -997,10 +998,6 @@ def _fix_up_module(cls, module):
(__file__, pkgdir,
) = cls._resolve_filename(state.origname, spec.name, ispkg)
assert state.filename == __file__, (state.filename, __file__)
if pkgdir:
assert __path__ == [pkgdir], (__path__, pkgdir)
else:
assert __path__ == ([] if ispkg else None), __path__
else:
__file__ = None
assert state.filename is None, state.filename
Expand Down Expand Up @@ -1043,6 +1040,25 @@ def _resolve_filename(cls, fullname, alias=None, ispkg=False):
filename = f'{sys._stdlib_dir}{sep}{relfile}.py'
return filename, pkgdir

@classmethod
def _package_dir(cls, fullname, pkgdir):
"""Return the directory of the frozen package 'fullname'.

Normally this is 'pkgdir', the package's directory in the standard
library, so that submodules which were not frozen are found there.
When that directory does not exist (e.g. the standard library is
zipped), a standard library package is looked up on sys.path
instead.
"""
if (_bootstrap_external is None
or fullname not in sys.stdlib_module_names
or (pkgdir and _bootstrap_external._path_isdir(pkgdir))):
return pkgdir
spec = _bootstrap_external.PathFinder.find_spec(fullname)
if spec is not None and spec.submodule_search_locations:
return spec.submodule_search_locations[0]
return pkgdir

@classmethod
def find_spec(cls, fullname, path=None, target=None):
info = _call_with_frames_removed(_imp.find_frozen, fullname)
Expand All @@ -1069,6 +1085,8 @@ def find_spec(cls, fullname, path=None, target=None):
filename=filename,
origname=origname,
)
if ispkg:
pkgdir = cls._package_dir(fullname, pkgdir)
if pkgdir:
spec.submodule_search_locations.insert(0, pkgdir)
return spec
Expand Down
68 changes: 67 additions & 1 deletion Lib/test/test_importlib/frozen/test_finder.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,16 @@
machinery = util.import_importlib('importlib.machinery')

import os.path
import sys
import tempfile
import textwrap
import unittest
import zipfile
from unittest import mock

from test.support import import_helper, REPO_ROOT, STDLIB_DIR
from test import support
from test.support import (import_helper, os_helper, script_helper,
REPO_ROOT, STDLIB_DIR)


def resolve_stdlib_file(name, ispkg=False):
Expand Down Expand Up @@ -179,5 +186,64 @@ def test_not_using_frozen(self):
) = util.test_both(FindSpecTests, machinery=machinery)


class PackagePathTests(unittest.TestCase):

"""Test __path__ of frozen packages."""

def setUp(self):
self.tmpdir = tempfile.mkdtemp()
self.addCleanup(os_helper.rmtree, self.tmpdir)
self.missing_dir = os.path.join(self.tmpdir, 'does-not-exist')

def import_phello(self):
with (mock.patch.dict(sys.modules),
import_helper.frozen_modules(),
support.captured_stdout()):
sys.modules.pop('__phello__', None)
import __phello__
self.assertEqual(__phello__.__spec__.origin, 'frozen')
return __phello__

def test_not_in_stdlib(self):
# A frozen package which is not a standard library package
# is not looked up on sys.path.
os.mkdir(os.path.join(self.tmpdir, '__phello__'))
with open(os.path.join(self.tmpdir, '__phello__', '__init__.py'), 'wb'):
pass
for stdlib_dir, expected in [
(self.missing_dir, [os.path.join(self.missing_dir, '__phello__')]),
(None, []),
]:
with (self.subTest(stdlib_dir=stdlib_dir),
support.swap_attr(sys, 'path', [self.tmpdir, *sys.path]),
support.swap_attr(sys, '_stdlib_dir', stdlib_dir)):
module = self.import_phello()
self.assertEqual(module.__path__, expected)

@support.requires_subprocess()
def test_zipped_stdlib(self):
# gh-148750: The codecs which are not frozen can be imported
# if the standard library is not available as a directory.
zipname = os.path.join(self.tmpdir, 'stdlib.zip')
with zipfile.ZipFile(zipname, 'w') as zf:
for name in ('__init__', 'cp437'):
zf.write(os.path.join(STDLIB_DIR, 'encodings', f'{name}.py'),
f'encodings/{name}.py')
code = textwrap.dedent("""
import encodings
print(encodings.__spec__.origin)
print(encodings.__path__[0])
import encodings.cp437
print(encodings.cp437.__spec__.origin)
""")
# PYTHONHOME is a directory without a standard library.
proc = script_helper.assert_python_ok(
'-S', '-X', 'frozen_modules=on', '-c', code,
PYTHONHOME=self.tmpdir, PYTHONPATH=zipname)
pkgdir = os.path.join(zipname, 'encodings')
self.assertEqual(proc.out.decode().splitlines(),
['frozen', pkgdir, os.path.join(pkgdir, 'cp437.py')])


if __name__ == '__main__':
unittest.main()
25 changes: 25 additions & 0 deletions Makefile.pre.in
Original file line number Diff line number Diff line change
Expand Up @@ -1729,6 +1729,11 @@ FROZEN_FILES_IN = \
Lib/_pybuiltins.py \
Lib/abc.py \
Lib/codecs.py \
Lib/encodings/__init__.py \
Lib/encodings/aliases.py \
Lib/encodings/utf_8.py \
Lib/encodings/_win_cp_codecs.py \
Lib/encodings/_iconv_codecs.py \
Lib/io.py \
Lib/_collections_abc.py \
Lib/_sitebuiltins.py \
Expand Down Expand Up @@ -1756,6 +1761,11 @@ FROZEN_FILES_OUT = \
Python/frozen_modules/builtins.h \
Python/frozen_modules/abc.h \
Python/frozen_modules/codecs.h \
Python/frozen_modules/encodings.h \
Python/frozen_modules/encodings.aliases.h \
Python/frozen_modules/encodings.utf_8.h \
Python/frozen_modules/encodings._win_cp_codecs.h \
Python/frozen_modules/encodings._iconv_codecs.h \
Python/frozen_modules/io.h \
Python/frozen_modules/_collections_abc.h \
Python/frozen_modules/_sitebuiltins.h \
Expand Down Expand Up @@ -1808,6 +1818,21 @@ Python/frozen_modules/abc.h: Lib/abc.py $(FREEZE_MODULE_DEPS)
Python/frozen_modules/codecs.h: Lib/codecs.py $(FREEZE_MODULE_DEPS)
$(FREEZE_MODULE) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h

Python/frozen_modules/encodings.h: Lib/encodings/__init__.py $(FREEZE_MODULE_DEPS)
$(FREEZE_MODULE) encodings $(srcdir)/Lib/encodings/__init__.py Python/frozen_modules/encodings.h

Python/frozen_modules/encodings.aliases.h: Lib/encodings/aliases.py $(FREEZE_MODULE_DEPS)
$(FREEZE_MODULE) encodings.aliases $(srcdir)/Lib/encodings/aliases.py Python/frozen_modules/encodings.aliases.h

Python/frozen_modules/encodings.utf_8.h: Lib/encodings/utf_8.py $(FREEZE_MODULE_DEPS)
$(FREEZE_MODULE) encodings.utf_8 $(srcdir)/Lib/encodings/utf_8.py Python/frozen_modules/encodings.utf_8.h

Python/frozen_modules/encodings._win_cp_codecs.h: Lib/encodings/_win_cp_codecs.py $(FREEZE_MODULE_DEPS)
$(FREEZE_MODULE) encodings._win_cp_codecs $(srcdir)/Lib/encodings/_win_cp_codecs.py Python/frozen_modules/encodings._win_cp_codecs.h

Python/frozen_modules/encodings._iconv_codecs.h: Lib/encodings/_iconv_codecs.py $(FREEZE_MODULE_DEPS)
$(FREEZE_MODULE) encodings._iconv_codecs $(srcdir)/Lib/encodings/_iconv_codecs.py Python/frozen_modules/encodings._iconv_codecs.h

Python/frozen_modules/io.h: Lib/io.py $(FREEZE_MODULE_DEPS)
$(FREEZE_MODULE) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
The ``encodings`` package and the codecs that every interpreter imports at startup (``encodings.aliases``, ``encodings.utf_8`` and the platform codecs) are frozen again, which makes interpreter startup faster.
25 changes: 25 additions & 0 deletions PCbuild/_freeze_module.vcxproj
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,31 @@
<IntFile>$(IntDir)codecs.g.h</IntFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\codecs.h</OutFile>
</None>
<None Include="..\Lib\encodings\__init__.py">
<ModName>encodings</ModName>
<IntFile>$(IntDir)encodings.g.h</IntFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\encodings.h</OutFile>
</None>
<None Include="..\Lib\encodings\aliases.py">
<ModName>encodings.aliases</ModName>
<IntFile>$(IntDir)encodings.aliases.g.h</IntFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\encodings.aliases.h</OutFile>
</None>
<None Include="..\Lib\encodings\utf_8.py">
<ModName>encodings.utf_8</ModName>
<IntFile>$(IntDir)encodings.utf_8.g.h</IntFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\encodings.utf_8.h</OutFile>
</None>
<None Include="..\Lib\encodings\_win_cp_codecs.py">
<ModName>encodings._win_cp_codecs</ModName>
<IntFile>$(IntDir)encodings._win_cp_codecs.g.h</IntFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\encodings._win_cp_codecs.h</OutFile>
</None>
<None Include="..\Lib\encodings\_iconv_codecs.py">
<ModName>encodings._iconv_codecs</ModName>
<IntFile>$(IntDir)encodings._iconv_codecs.g.h</IntFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\encodings._iconv_codecs.h</OutFile>
</None>
<None Include="..\Lib\io.py">
<ModName>io</ModName>
<IntFile>$(IntDir)io.g.h</IntFile>
Expand Down
15 changes: 15 additions & 0 deletions PCbuild/_freeze_module.vcxproj.filters
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,21 @@
<None Include="..\Lib\codecs.py">
<Filter>Python Files</Filter>
</None>
<None Include="..\Lib\encodings\__init__.py">
<Filter>Python Files</Filter>
</None>
<None Include="..\Lib\encodings\aliases.py">
<Filter>Python Files</Filter>
</None>
<None Include="..\Lib\encodings\utf_8.py">
<Filter>Python Files</Filter>
</None>
<None Include="..\Lib\encodings\_win_cp_codecs.py">
<Filter>Python Files</Filter>
</None>
<None Include="..\Lib\encodings\_iconv_codecs.py">
<Filter>Python Files</Filter>
</None>
<None Include="..\Lib\io.py">
<Filter>Python Files</Filter>
</None>
Expand Down
10 changes: 10 additions & 0 deletions Python/frozen.c
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@
#include "frozen_modules/builtins.h"
#include "frozen_modules/abc.h"
#include "frozen_modules/codecs.h"
#include "frozen_modules/encodings.h"
#include "frozen_modules/encodings.aliases.h"
#include "frozen_modules/encodings.utf_8.h"
#include "frozen_modules/encodings._win_cp_codecs.h"
#include "frozen_modules/encodings._iconv_codecs.h"
#include "frozen_modules/io.h"
#include "frozen_modules/_collections_abc.h"
#include "frozen_modules/_sitebuiltins.h"
Expand Down Expand Up @@ -79,6 +84,11 @@ static const struct _frozen stdlib_modules[] = {
/* stdlib - startup, without site (python -S) */
{"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), false},
{"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), false},
{"encodings", _Py_M__encodings, (int)sizeof(_Py_M__encodings), true},
{"encodings.aliases", _Py_M__encodings_aliases, (int)sizeof(_Py_M__encodings_aliases), false},
{"encodings.utf_8", _Py_M__encodings_utf_8, (int)sizeof(_Py_M__encodings_utf_8), false},
{"encodings._win_cp_codecs", _Py_M__encodings__win_cp_codecs, (int)sizeof(_Py_M__encodings__win_cp_codecs), false},
{"encodings._iconv_codecs", _Py_M__encodings__iconv_codecs, (int)sizeof(_Py_M__encodings__iconv_codecs), false},
{"io", _Py_M__io, (int)sizeof(_Py_M__io), false},

/* stdlib - startup, with site */
Expand Down
17 changes: 10 additions & 7 deletions Tools/build/freeze_modules.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,16 @@
('stdlib - startup, without site (python -S)', [
'abc',
'codecs',
# gh-148750: Partially freezing encodings causes import errors of
# submodules when using a zipped standard library.
#'<encodings>',
#'encodings.aliases',
#'encodings.utf_8',
#'encodings._win_cp_codecs',
#'encodings._iconv_codecs',
# Only the parts of the encodings package that every interpreter
# imports at startup are frozen. The other codecs are found via
# encodings.__path__, which FrozenImporter resolves through the
# path-based finders when the stdlib directory is not available
# (e.g. a zipped stdlib).
'<encodings>',
'encodings.aliases',
'encodings.utf_8',
'encodings._win_cp_codecs',
'encodings._iconv_codecs',
'io',
]),
('stdlib - startup, with site', [
Expand Down
Loading