forked from cool-RR/python_toolbox
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathimport_tools.py
More file actions
289 lines (223 loc) · 9.04 KB
/
import_tools.py
File metadata and controls
289 lines (223 loc) · 9.04 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
# Copyright 2009-2015 Ram Rachum.
# This program is distributed under the MIT license.
'''Defines various tools related to importing.'''
import sys
import os.path
import imp
import zipimport
try:
import pathlib
except:
from python_toolbox.third_party import pathlib
from python_toolbox import package_finder
from python_toolbox import caching
def import_all(package, exclude='__init__', silent_fail=False):
'''
Import all the modules and packages that live inside the given package.
This is not recursive. Modules and packages defined inside a subpackage
will not be imported (of course, that subpackage itself may import them
anyway.)
You may specify a module/package to exclude, which is by default
`__init__`.
Returns a list with all the imported modules and packages.
todo: only tested with __init__ passed in
'''
paths = package_finder.get_packages_and_modules_filenames(package)
names = {}
for path in paths:
name = path.stem
if name == exclude:
continue
full_name = package.__name__ + '.' + name
names[path] = full_name
d = {}
for (path, name) in names.items():
try:
d[name] = normal_import(name)
except Exception:
if not silent_fail:
raise
return d
def normal_import(module_name):
'''
Import a module.
This function has several advantages over `__import__`:
1. It avoids the weird `fromlist=['']` that you need to give `__import__`
in order for it to return the specific module you requested instead of
the outermost package, and
2. It avoids a weird bug in Linux, where importing using `__import__` can
lead to a `module.__name__` containing two consecutive dots.
'''
if '.' in module_name:
package_name, submodule_name = module_name.rsplit('.', 1)
package = __import__(module_name)
return reduce(getattr, [package] + module_name.split('.')[1:])
else:
return __import__(module_name)
@caching.cache() # todo: clear cache if `sys.path` changes
def import_if_exists(module_name, silent_fail=False):
'''
Import module by name and return it, only if it exists.
If `silent_fail` is `True`, will return `None` if the module doesn't exist.
If `silent_fail` is False, will raise `ImportError`.
`silent_fail` applies only to whether the module exists or not; if it does
exist, but there's an error importing it... *release the hounds.*
I mean, we just raise the error.
'''
if '.' in module_name:
package_name, submodule_name = module_name.rsplit('.', 1)
package = import_if_exists(package_name, silent_fail=silent_fail)
if not package:
assert silent_fail is True
return None
package_path = package.__path__
if not exists(submodule_name, package_path):
if silent_fail is True:
return None
else: # silent_fail is False
raise ImportError("Can't find %s." % module_name)
else: # '.' not in module_name
if not exists(module_name):
if silent_fail is True:
return None
else: # silent_fail is False
raise ImportError("Can't find %s." % module_name)
return normal_import(module_name)
def exists(module_name, path=None):
'''
Return whether a module by the name `module_name` exists.
This seems to be the best way to carefully import a module.
Currently implemented for top-level packages only. (i.e. no dots.)
Supports modules imported from a zip file.
'''
if '.' in module_name:
raise NotImplementedError
module_file = None
try:
module_file, _, _ = find_module(module_name, path=path,
legacy_output=True)
except ImportError:
return False
else:
return True
finally:
if hasattr(module_file, 'close'):
module_file.close()
def _import_by_path_from_zip(path):
'''Import a module from a path inside a zip file.'''
assert '.zip' in path
parent_path, child_name = path.rsplit(os.path.sep, 1)
zip_importer = zipimport.zipimporter(parent_path)
module = zip_importer.load_module(child_name)
return module
def import_by_path(path, name=None, keep_in_sys_modules=True):
'''
Import module/package by path.
You may specify a name: This is helpful only if it's an hierarchical name,
i.e. a name with dots like "orange.claw.hammer". This will become the
imported module's __name__ attribute. Otherwise only the short name,
"hammer", will be used, which might cause problems in some cases. (Like
when using multiprocessing.)
'''
path = pathlib.Path(path)
if '.zip' in path:
if name is not None:
raise NotImplementedError
module = _import_by_path_from_zip(path)
else: # '.zip' not in path
short_name = path.stem
if name is None: name = short_name
my_file = None
try:
(my_file, pathname, description) = \
imp.find_module(short_name, [path.parent])
module = imp.load_module(name, my_file, pathname, description)
finally:
if my_file is not None:
my_file.close()
if not keep_in_sys_modules:
del sys.modules[module.__name__]
return module
def find_module(module_name, path=None, look_in_zip=True, legacy_output=False):
'''
Search for a module by name and return its filename.
When `path=None`, search for a built-in, frozen or special module and
continue search in `sys.path`.
When `legacy_output=True`, instead of returning the module's filename,
returns a tuple `(file, filename, (suffix, mode, type))`.
When `look_in_zip=True`, also looks in zipmodules.
todo: Gives funky output when `legacy_output=True and look_in_zip=True`.
'''
# todo: test
if look_in_zip:
try:
result = _find_module_in_some_zip_path(module_name, path)
except ImportError:
pass
else:
return (None, result, None) if legacy_output else result
if '.' in module_name:
parent_name, child_name = module_name.rsplit('.', 1)
parent_path = find_module(parent_name, path)
result = imp.find_module(child_name, [parent_path])
else:
result = imp.find_module(module_name, path)
if legacy_output:
return result
else: # legacy_output is False
file_, path_, description_ = result
if file_ is not None:
file_.close()
return path_
def _find_module_in_some_zip_path(module_name, path=None):
'''
If a module called `module_name` exists in a zip archive, get its path.
If the module is not found, raises `ImportError`.
'''
original_path_argument = path
if path is not None:
zip_paths = path
else:
zip_paths = [path for path in sys.path if '.zip' in path]
# todo: Find better way to filter zip paths.
for zip_path in zip_paths:
# Trying to create a zip importer:
try:
zip_importer = zipimport.zipimporter(zip_path)
except zipimport.ZipImportError:
continue
# Excepted `ZipImportError` because we may have zip paths in
# `sys.path` that don't really exist, which causes `zipimport` to
# raise `ZipImportError`.
#
# todo: should find smarter way of catching this, excepting
# `ZipImportError` is not a good idea.
result = zip_importer.find_module(
# Python's zip importer stupidly needs us to replace dots with path
# separators:
_module_address_to_partial_path(module_name)
)
if result is None:
continue
else:
assert result is zip_importer
#if '.' in module_name:
#parent_package_name, child_module_name = \
#module_name.rsplit('.')
#leading_path = \
#_module_address_to_partial_path(parent_package_name)
#else:
#leading_path = ''
return pathlib.Path(str(zip_path)) / \
_module_address_to_partial_path(module_name)
if original_path_argument is not None:
raise ImportError('Module not found in the given zip path.')
else:
raise ImportError('Module not found in any of the zip paths.')
def _module_address_to_partial_path(module_address):
'''
Convert a dot-seperated address to a path-seperated address.
For example, on Linux, `'python_toolbox.caching.cached_property'` would be
converted to `'python_toolbox/caching/cached_property'`.
'''
return os.path.sep.join(module_address.split('.'))