Repository navigation
Expand file tree
/
Copy pathiptest.py
More file actions
346 lines (287 loc) · 12.5 KB
/
Copy pathiptest.py
File metadata and controls
346 lines (287 loc) · 12.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
# -*- coding: utf-8 -*-
"""IPython Test Suite Runner.
This module provides a main entry point to a user script to test IPython
itself from the command line. There are two ways of running this script:
1. With the syntax `iptest all`. This runs our entire test suite by
calling this script (with different arguments) or trial recursively. This
causes modules and package to be tested in different processes, using nose
or trial where appropriate.
2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
the script simply calls nose, but with special command line flags and
plugins loaded.
For now, this script requires that both nose and twisted are installed. This
will change in the future.
"""
#-----------------------------------------------------------------------------
# Module imports
#-----------------------------------------------------------------------------
import os
import os.path as path
import sys
import subprocess
import tempfile
import time
import warnings
import nose.plugins.builtin
from nose.core import TestProgram
from IPython.platutils import find_cmd
from IPython.testing.plugin.ipdoctest import IPythonDoctest
pjoin = path.join
#-----------------------------------------------------------------------------
# Logic for skipping doctests
#-----------------------------------------------------------------------------
def test_for(mod):
"""Test to see if mod is importable."""
try:
__import__(mod)
except ImportError:
return False
else:
return True
have_curses = test_for('_curses')
have_wx = test_for('wx')
have_wx_aui = test_for('wx.aui')
have_zi = test_for('zope.interface')
have_twisted = test_for('twisted')
have_foolscap = test_for('foolscap')
have_objc = test_for('objc')
have_pexpect = test_for('pexpect')
# For the IPythonDoctest plugin, we need to exclude certain patterns that cause
# testing problems. We should strive to minimize the number of skipped
# modules, since this means untested code. As the testing machinery
# solidifies, this list should eventually become empty.
EXCLUDE = [pjoin('IPython', 'external'),
pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
pjoin('IPython_doctest_plugin'),
pjoin('IPython', 'Gnuplot'),
pjoin('IPython', 'Extensions', 'ipy_'),
pjoin('IPython', 'Extensions', 'PhysicalQInput'),
pjoin('IPython', 'Extensions', 'PhysicalQInteractive'),
pjoin('IPython', 'Extensions', 'InterpreterPasteInput'),
pjoin('IPython', 'Extensions', 'scitedirector'),
pjoin('IPython', 'Extensions', 'numeric_formats'),
pjoin('IPython', 'testing', 'attic'),
pjoin('IPython', 'testing', 'tutils'),
pjoin('IPython', 'testing', 'tools'),
pjoin('IPython', 'testing', 'mkdoctests'),
pjoin('IPython', 'gui'),
]
if not have_wx:
EXCLUDE.append(pjoin('IPython', 'Extensions', 'igrid'))
EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
if not have_wx_aui:
EXCLUDE.append(pjoin('IPython', 'gui', 'wx', 'wxIPython'))
if not have_objc:
EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
if not have_curses:
EXCLUDE.append(pjoin('IPython', 'Extensions', 'ibrowse'))
if not sys.platform == 'win32':
EXCLUDE.append(pjoin('IPython', 'platutils_win32'))
# These have to be skipped on win32 because the use echo, rm, cd, etc.
# See ticket https://bugs.launchpad.net/bugs/366982
if sys.platform == 'win32':
EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
if not os.name == 'posix':
EXCLUDE.append(pjoin('IPython', 'platutils_posix'))
if not have_pexpect:
EXCLUDE.append(pjoin('IPython', 'irunner'))
if not have_twisted:
EXCLUDE.append(pjoin('IPython', 'frontend', 'asyncfrontendbase'))
EXCLUDE.append(pjoin('IPython', 'frontend', 'prefilterfrontend'))
EXCLUDE.append(pjoin('IPython', 'frontend', 'frontendbase'))
EXCLUDE.append(pjoin('IPython', 'frontend', 'linefrontendbase'))
EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', 'test_linefrontend'))
EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', 'test_frontendbase'))
EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
'test_prefilterfrontend'))
EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
'test_asyncfrontendbase'))
EXCLUDE.append(pjoin('IPython', 'kernel', 'error'))
EXCLUDE.append(pjoin('IPython', 'testing', 'parametric'))
EXCLUDE.append(pjoin('IPython', 'testing', 'util'))
EXCLUDE.append(pjoin('IPython', 'testing', 'tests', 'test_decorators_trial'))
# This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
if sys.platform == 'win32':
EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
#-----------------------------------------------------------------------------
# Functions and classes
#-----------------------------------------------------------------------------
def run_iptest():
"""Run the IPython test suite using nose.
This function is called when this script is **not** called with the form
`iptest all`. It simply calls nose with appropriate command line flags
and accepts all of the standard nose arguments.
"""
warnings.filterwarnings('ignore',
'This will be removed soon. Use IPython.testing.util instead')
argv = sys.argv + [
# Loading ipdoctest causes problems with Twisted.
# I am removing this as a temporary fix to get the
# test suite back into working shape. Our nose
# plugin needs to be gone through with a fine
# toothed comb to find what is causing the problem.
'--with-ipdoctest',
'--ipdoctest-tests','--ipdoctest-extension=txt',
'--detailed-errors',
# We add --exe because of setuptools' imbecility (it
# blindly does chmod +x on ALL files). Nose does the
# right thing and it tries to avoid executables,
# setuptools unfortunately forces our hand here. This
# has been discussed on the distutils list and the
# setuptools devs refuse to fix this problem!
'--exe',
]
# Detect if any tests were required by explicitly calling an IPython
# submodule or giving a specific path
has_tests = False
for arg in sys.argv:
if 'IPython' in arg or arg.endswith('.py') or \
(':' in arg and '.py' in arg):
has_tests = True
break
# If nothing was specifically requested, test full IPython
if not has_tests:
argv.append('IPython')
# Construct list of plugins, omitting the existing doctest plugin, which
# ours replaces (and extends).
plugins = [IPythonDoctest(EXCLUDE)]
for p in nose.plugins.builtin.plugins:
plug = p()
if plug.name == 'doctest':
continue
#print '*** adding plugin:',plug.name # dbg
plugins.append(plug)
TestProgram(argv=argv,plugins=plugins)
class IPTester(object):
"""Call that calls iptest or trial in a subprocess.
"""
def __init__(self,runner='iptest',params=None):
""" """
if runner == 'iptest':
self.runner = ['iptest','-v']
else:
self.runner = [find_cmd('trial')]
if params is None:
params = []
if isinstance(params,str):
params = [params]
self.params = params
# Assemble call
self.call_args = self.runner+self.params
if sys.platform == 'win32':
def run(self):
"""Run the stored commands"""
# On Windows, cd to temporary directory to run tests. Otherwise,
# Twisted's trial may not be able to execute 'trial IPython', since
# it will confuse the IPython module name with the ipython
# execution scripts, because the windows file system isn't case
# sensitive.
# We also use os.system instead of subprocess.call, because I was
# having problems with subprocess and I just don't know enough
# about win32 to debug this reliably. Os.system may be the 'old
# fashioned' way to do it, but it works just fine. If someone
# later can clean this up that's fine, as long as the tests run
# reliably in win32.
curdir = os.getcwd()
os.chdir(tempfile.gettempdir())
stat = os.system(' '.join(self.call_args))
os.chdir(curdir)
return stat
else:
def run(self):
"""Run the stored commands"""
return subprocess.call(self.call_args)
def make_runners():
"""Define the modules and packages that need to be tested.
"""
# This omits additional top-level modules that should not be doctested.
# XXX: Shell.py is also ommited because of a bug in the skip_doctest
# decorator. See ticket https://bugs.launchpad.net/bugs/366209
top_mod = \
['background_jobs.py', 'ColorANSI.py', 'completer.py', 'ConfigLoader.py',
'CrashHandler.py', 'Debugger.py', 'deep_reload.py', 'demo.py',
'DPyGetOpt.py', 'dtutils.py', 'excolors.py', 'FakeModule.py',
'generics.py', 'genutils.py', 'history.py', 'hooks.py', 'ipapi.py',
'iplib.py', 'ipmaker.py', 'ipstruct.py', 'Itpl.py',
'Logger.py', 'macro.py', 'Magic.py', 'OInspect.py',
'OutputTrap.py', 'platutils.py', 'prefilter.py', 'Prompts.py',
'PyColorize.py', 'Release.py', 'rlineimpl.py', 'shadowns.py',
'shellglobals.py', 'strdispatch.py',
'ultraTB.py', 'upgrade_dir.py', 'usage.py', 'wildcard.py',
# See note above for why this is skipped
# 'Shell.py',
'winconsole.py']
if have_twisted:
top_mod.append('twshell.py')
if have_pexpect:
top_mod.append('irunner.py')
if sys.platform == 'win32':
top_mod.append('platutils_win32.py')
elif os.name == 'posix':
top_mod.append('platutils_posix.py')
else:
top_mod.append('platutils_dummy.py')
# These are tested by nose, so skip IPython.kernel
top_pack = ['config','Extensions','frontend',
'testing','tests','tools','UserConfig']
if have_wx:
top_pack.append('gui')
modules = ['IPython.%s' % m[:-3] for m in top_mod ]
packages = ['IPython.%s' % m for m in top_pack ]
# Make runners
runners = dict(zip(top_pack, [IPTester(params=v) for v in packages]))
# Test IPython.kernel using trial if twisted is installed
if have_zi and have_twisted and have_foolscap:
runners['trial'] = IPTester('trial',['IPython'])
runners['modules'] = IPTester(params=modules)
return runners
def run_iptestall():
"""Run the entire IPython test suite by calling nose and trial.
This function constructs :class:`IPTester` instances for all IPython
modules and package and then runs each of them. This causes the modules
and packages of IPython to be tested each in their own subprocess using
nose or twisted.trial appropriately.
"""
runners = make_runners()
# Run all test runners, tracking execution time
failed = {}
t_start = time.time()
for name,runner in runners.iteritems():
print '*'*77
print 'IPython test group:',name
res = runner.run()
if res:
failed[name] = res
t_end = time.time()
t_tests = t_end - t_start
nrunners = len(runners)
nfail = len(failed)
# summarize results
print
print '*'*77
print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
print
if not failed:
print 'OK'
else:
# If anything went wrong, point out what command to rerun manually to
# see the actual errors and individual summary
print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
for name in failed:
failed_runner = runners[name]
print '-'*40
print 'Runner failed:',name
print 'You may wish to rerun this one individually, with:'
print ' '.join(failed_runner.call_args)
print
def main():
if len(sys.argv) == 1:
run_iptestall()
else:
if sys.argv[1] == 'all':
run_iptestall()
else:
run_iptest()
if __name__ == '__main__':
main()