Skip to content

Commit aaab6d2

Browse files
jaracojaraco
committed
gh-123987: Fix issues in importlib.resources.
Also addresses gh-123085.
1 parent 4ed7d1d commit aaab6d2

File tree

8 files changed

+134
-21
lines changed

8 files changed

+134
-21
lines changed

‎Lib/importlib/resources/__init__.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,11 @@
1-
"""Read resources contained within a package."""
1+
"""
2+
Read resources contained within a package.
3+
4+
This codebase is shared between importlib.resources in the stdlib
5+
and importlib_resources in PyPI. See
6+
https://github.com/python/importlib_metadata/wiki/Development-Methodology
7+
for more detail.
8+
"""
29

310
from ._common import (
411
as_file,

‎Lib/importlib/resources/_common.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,10 @@ def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]:
6666
# zipimport.zipimporter does not support weak references, resulting in a
6767
# TypeError. That seems terrible.
6868
spec = package.__spec__
69-
reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore
69+
reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore[union-attr]
7070
if reader is None:
7171
return None
72-
return reader(spec.name) # type: ignore
72+
return reader(spec.name) # type: ignore[union-attr]
7373

7474

7575
@functools.singledispatch
@@ -93,12 +93,13 @@ def _infer_caller():
9393
"""
9494

9595
def is_this_file(frame_info):
96-
return frame_info.filename == __file__
96+
return frame_info.filename == stack[0].filename
9797

9898
def is_wrapper(frame_info):
9999
return frame_info.function == 'wrapper'
100100

101-
not_this_file = itertools.filterfalse(is_this_file, inspect.stack())
101+
stack = inspect.stack()
102+
not_this_file = itertools.filterfalse(is_this_file, stack)
102103
# also exclude 'wrapper' due to singledispatch in the call stack
103104
callers = itertools.filterfalse(is_wrapper, not_this_file)
104105
return next(callers).frame

‎Lib/importlib/resources/readers.py‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
from __future__ import annotations
2+
13
import collections
24
import contextlib
35
import itertools
@@ -6,6 +8,7 @@
68
import re
79
import warnings
810
import zipfile
11+
from collections.abc import Iterator
912

1013
from . import abc
1114

@@ -135,27 +138,31 @@ class NamespaceReader(abc.TraversableResources):
135138
def __init__(self, namespace_path):
136139
if 'NamespacePath' not in str(namespace_path):
137140
raise ValueError('Invalid path')
138-
self.path = MultiplexedPath(*map(self._resolve, namespace_path))
141+
self.path = MultiplexedPath(*filter(bool, map(self._resolve, namespace_path)))
139142

140143
@classmethod
141-
def _resolve(cls, path_str) -> abc.Traversable:
144+
def _resolve(cls, path_str) -> abc.Traversable | None:
142145
r"""
143146
Given an item from a namespace path, resolve it to a Traversable.
144147
145148
path_str might be a directory on the filesystem or a path to a
146149
zipfile plus the path within the zipfile, e.g. ``/foo/bar`` or
147150
``/foo/baz.zip/inner_dir`` or ``foo\baz.zip\inner_dir\sub``.
151+
152+
path_str might also be a sentinel used by editable packages to
153+
trigger other behaviors (see python/importlib_resources#311).
154+
In that case, return None.
148155
"""
149-
(dir,) = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir())
150-
return dir
156+
dirs = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir())
157+
return next(dirs, None)
151158

152159
@classmethod
153-
def _candidate_paths(cls, path_str):
160+
def _candidate_paths(cls, path_str: str) -> Iterator[abc.Traversable]:
154161
yield pathlib.Path(path_str)
155162
yield from cls._resolve_zip_path(path_str)
156163

157164
@staticmethod
158-
def _resolve_zip_path(path_str):
165+
def _resolve_zip_path(path_str: str):
159166
for match in reversed(list(re.finditer(r'[\\/]', path_str))):
160167
with contextlib.suppress(
161168
FileNotFoundError,

‎Lib/importlib/resources/simple.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ class ResourceHandle(Traversable):
7777

7878
def __init__(self, parent: ResourceContainer, name: str):
7979
self.parent = parent
80-
self.name = name # type: ignore
80+
self.name = name # type: ignore[misc]
8181

8282
def is_file(self):
8383
return True

‎Lib/test/test_importlib/resources/_path.py‎

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,44 @@
22
import functools
33

44
from typing import Dict, Union
5+
from typing import runtime_checkable
6+
from typing import Protocol
57

68

79
####
8-
# from jaraco.path 3.4.1
10+
# from jaraco.path 3.7.1
911

10-
FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']] # type: ignore
1112

13+
class Symlink(str):
14+
"""
15+
A string indicating the target of a symlink.
16+
"""
17+
18+
19+
FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']]
20+
21+
22+
@runtime_checkable
23+
class TreeMaker(Protocol):
24+
def __truediv__(self, *args, **kwargs): ... # pragma: no cover
25+
26+
def mkdir(self, **kwargs): ... # pragma: no cover
27+
28+
def write_text(self, content, **kwargs): ... # pragma: no cover
29+
30+
def write_bytes(self, content): ... # pragma: no cover
1231

13-
def build(spec: FilesSpec, prefix=pathlib.Path()):
32+
def symlink_to(self, target): ... # pragma: no cover
33+
34+
35+
def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker:
36+
return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore[return-value]
37+
38+
39+
def build(
40+
spec: FilesSpec,
41+
prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore[assignment]
42+
):
1443
"""
1544
Build a set of files/directories, as described by the spec.
1645
@@ -25,21 +54,25 @@ def build(spec: FilesSpec, prefix=pathlib.Path()):
2554
... "__init__.py": "",
2655
... },
2756
... "baz.py": "# Some code",
28-
... }
57+
... "bar.py": Symlink("baz.py"),
58+
... },
59+
... "bing": Symlink("foo"),
2960
... }
3061
>>> target = getfixture('tmp_path')
3162
>>> build(spec, target)
3263
>>> target.joinpath('foo/baz.py').read_text(encoding='utf-8')
3364
'# Some code'
65+
>>> target.joinpath('bing/bar.py').read_text(encoding='utf-8')
66+
'# Some code'
3467
"""
3568
for name, contents in spec.items():
36-
create(contents, pathlib.Path(prefix) / name)
69+
create(contents, _ensure_tree_maker(prefix) / name)
3770

3871

3972
@functools.singledispatch
4073
def create(content: Union[str, bytes, FilesSpec], path):
4174
path.mkdir(exist_ok=True)
42-
build(content, prefix=path) # type: ignore
75+
build(content, prefix=path) # type: ignore[arg-type]
4376

4477

4578
@create.register
@@ -52,5 +85,10 @@ def _(content: str, path):
5285
path.write_text(content, encoding='utf-8')
5386

5487

88+
@create.register
89+
def _(content: Symlink, path):
90+
path.symlink_to(content)
91+
92+
5593
# end from jaraco.path
5694
####

‎Lib/test/test_importlib/resources/test_files.py‎

Lines changed: 58 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
import os
2+
import pathlib
3+
import py_compile
4+
import shutil
15
import textwrap
26
import unittest
37
import warnings
@@ -7,6 +11,7 @@
711
from importlib import resources
812
from importlib.resources.abc import Traversable
913
from . import util
14+
from test.support import os_helper, import_helper
1015

1116

1217
@contextlib.contextmanager
@@ -55,6 +60,26 @@ class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase):
5560
class OpenNamespaceTests(FilesTests, util.DiskSetup, unittest.TestCase):
5661
MODULE = 'namespacedata01'
5762

63+
def test_non_paths_in_dunder_path(self):
64+
"""
65+
Non-path items in a namespace package's ``__path__`` are ignored.
66+
67+
As reported in python/importlib_resources#311, some tools
68+
like Setuptools, when creating editable packages, will inject
69+
non-paths into a namespace package's ``__path__``, a
70+
sentinel like
71+
``__editable__.sample_namespace-1.0.finder.__path_hook__``
72+
to cause the ``PathEntryFinder`` to be called when searching
73+
for packages. In that case, resources should still be loadable.
74+
"""
75+
import namespacedata01
76+
77+
namespacedata01.__path__.append(
78+
'__editable__.sample_namespace-1.0.finder.__path_hook__'
79+
)
80+
81+
resources.files(namespacedata01)
82+
5883

5984
class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase):
6085
ZIP_MODULE = 'namespacedata01'
@@ -81,7 +106,7 @@ def test_module_resources(self):
81106
"""
82107
A module can have resources found adjacent to the module.
83108
"""
84-
import mod
109+
import mod # type: ignore[import-not-found]
85110

86111
actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8')
87112
assert actual == self.spec['res.txt']
@@ -97,8 +122,8 @@ class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.Test
97122

98123
class ImplicitContextFiles:
99124
set_val = textwrap.dedent(
100-
"""
101-
import importlib.resources as res
125+
f"""
126+
import {resources.__name__} as res
102127
val = res.files().joinpath('res.txt').read_text(encoding='utf-8')
103128
"""
104129
)
@@ -108,6 +133,10 @@ class ImplicitContextFiles:
108133
'submod.py': set_val,
109134
'res.txt': 'resources are the best',
110135
},
136+
'frozenpkg': {
137+
'__init__.py': set_val.replace(resources.__name__, 'c_resources'),
138+
'res.txt': 'resources are the best',
139+
},
111140
}
112141

113142
def test_implicit_files_package(self):
@@ -122,6 +151,32 @@ def test_implicit_files_submodule(self):
122151
"""
123152
assert importlib.import_module('somepkg.submod').val == 'resources are the best'
124153

154+
def _compile_importlib(self):
155+
"""
156+
Make a compiled-only copy of the importlib resources package.
157+
"""
158+
bin_site = self.fixtures.enter_context(os_helper.temp_dir())
159+
c_resources = pathlib.Path(bin_site, 'c_resources')
160+
sources = pathlib.Path(resources.__file__).parent
161+
shutil.copytree(sources, c_resources, ignore=lambda *_: ['__pycache__'])
162+
163+
for dirpath, _, filenames in os.walk(c_resources):
164+
for filename in filenames:
165+
source_path = pathlib.Path(dirpath) / filename
166+
cfile = source_path.with_suffix('.pyc')
167+
py_compile.compile(source_path, cfile)
168+
pathlib.Path.unlink(source_path)
169+
self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site))
170+
171+
def test_implicit_files_with_compiled_importlib(self):
172+
"""
173+
Caller detection works for compiled-only resources module.
174+
175+
python/cpython#123085
176+
"""
177+
self._compile_importlib()
178+
assert importlib.import_module('frozenpkg').val == 'resources are the best'
179+
125180

126181
class ImplicitContextFilesDiskTests(
127182
DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase

‎Misc/NEWS.d/next/Library/2024-09-12-14-24-25.gh-issue-123987.7_OD1p.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Fixed issue in NamespaceReader where a non-path item in a namespace path,
2+
such as a sentinel added by an editable installer, would break resource
3+
loading.

‎Misc/NEWS.d/next/Library/2024-09-12-14-26-05.gh-issue-123085.Naekav.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fixed issue in bare ``importlib.resources.files()`` when the library is
2+
available as compiled only (no source).

0 commit comments

Comments
 (0)