Skip to content

Commit fea98bf

Browse files
vstinnervstinner
authored
[2.7] bpo-30523, bpo-30764, bpo-30776: Sync regrtest from master (#2444)
* bpo-30523: regrtest --list-cases --match (#2401) * regrtest --list-cases now supports --match and --match-file options. Example: ./python -m test --list-cases -m FileTests test_os * --list-cases now also sets support.verbose to False to prevent messages to stdout when loading test modules. * Add support._match_test() private function. (cherry picked from commit ace56d5) (cherry picked from commit 36946c0) * bpo-30764: regrtest: add --fail-env-changed option (#2402) * bpo-30764: regrtest: change exit code on failure * Exit code 2 if failed tests ("bad") * Exit code 3 if interrupted * bpo-30764: regrtest: add --fail-env-changed option If the option is set, mark a test as failed if it alters the environment, for example if it creates a file without removing it. (cherry picked from commit 63f54c6) (cherry picked from commit 1f33857) * bpo-30776: reduce regrtest -R false positives (#2422) * Change the regrtest --huntrleaks checker to decide if a test file leaks or not. Require that each run leaks at least 1 reference. * Warmup runs are now completely ignored: ignored in the checker test and not used anymore to compute the sum. * Add an unit test for a reference leak. Example of reference differences previously considered a failure (leak) and now considered as success (success, no leak): [3, 0, 0] [0, 1, 0] [8, -8, 1] (cherry picked from commit 48b5c42) (cherry picked from commit e0f8b43)
1 parent 2d77558 commit fea98bf

File tree

3 files changed

+148
-42
lines changed

3 files changed

+148
-42
lines changed

‎Lib/test/regrtest.py‎

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
don't execute them
6868
--list-cases -- only write the name of test cases that will be run,
6969
don't execute them
70+
--fail-env-changed -- if a test file alters the environment, mark the test
71+
as failed
7072
7173
7274
Additional Option Details:
@@ -327,7 +329,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
327329
'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=',
328330
'multiprocess=', 'slaveargs=', 'forever', 'header', 'pgo',
329331
'failfast', 'match=', 'testdir=', 'list-tests', 'list-cases',
330-
'coverage', 'matchfile='])
332+
'coverage', 'matchfile=', 'fail-env-changed'])
331333
except getopt.error, msg:
332334
usage(2, msg)
333335

@@ -339,6 +341,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
339341
slaveargs = None
340342
list_tests = False
341343
list_cases_opt = False
344+
fail_env_changed = False
342345
for o, a in opts:
343346
if o in ('-h', '--help'):
344347
usage(0)
@@ -439,6 +442,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
439442
list_tests = True
440443
elif o == '--list-cases':
441444
list_cases_opt = True
445+
elif o == '--fail-env-changed':
446+
fail_env_changed = True
442447
else:
443448
print >>sys.stderr, ("No handler for option {}. Please "
444449
"report this as a bug at http://bugs.python.org.").format(o)
@@ -558,7 +563,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
558563
sys.exit(0)
559564

560565
if list_cases_opt:
561-
list_cases(testdir, selected)
566+
list_cases(testdir, selected, match_tests)
562567
sys.exit(0)
563568

564569
if trace:
@@ -908,11 +913,19 @@ def local_runtest():
908913
result = "FAILURE"
909914
elif interrupted:
910915
result = "INTERRUPTED"
916+
elif environment_changed and fail_env_changed:
917+
result = "ENV CHANGED"
911918
else:
912919
result = "SUCCESS"
913920
print("Tests result: %s" % result)
914921

915-
sys.exit(len(bad) > 0 or interrupted)
922+
if bad:
923+
sys.exit(2)
924+
if interrupted:
925+
sys.exit(130)
926+
if fail_env_changed and environment_changed:
927+
sys.exit(3)
928+
sys.exit(0)
916929

917930

918931
STDTESTS = [
@@ -1310,7 +1323,18 @@ def run_the_test():
13101323
if i >= nwarmup:
13111324
deltas.append(rc_after - rc_before)
13121325
print >> sys.stderr
1313-
if any(deltas):
1326+
1327+
# bpo-30776: Try to ignore false positives:
1328+
#
1329+
# [3, 0, 0]
1330+
# [0, 1, 0]
1331+
# [8, -8, 1]
1332+
#
1333+
# Expected leaks:
1334+
#
1335+
# [5, 5, 6]
1336+
# [10, 1, 1]
1337+
if all(delta >= 1 for delta in deltas):
13141338
msg = '%s leaked %s references, sum=%s' % (test, deltas, sum(deltas))
13151339
print >> sys.stderr, msg
13161340
with open(fname, "a") as refrep:
@@ -1501,9 +1525,13 @@ def _list_cases(suite):
15011525
if isinstance(test, unittest.TestSuite):
15021526
_list_cases(test)
15031527
elif isinstance(test, unittest.TestCase):
1504-
print(test.id())
1528+
if test_support._match_test(test):
1529+
print(test.id())
1530+
1531+
def list_cases(testdir, selected, match_tests):
1532+
test_support.verbose = False
1533+
test_support.match_tests = match_tests
15051534

1506-
def list_cases(testdir, selected):
15071535
skipped = []
15081536
for test in selected:
15091537
abstest = get_abs_module(testdir, test)

‎Lib/test/support/__init__.py‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1542,6 +1542,23 @@ def _run_suite(suite):
15421542
raise TestFailed(err)
15431543

15441544

1545+
def _match_test(test):
1546+
global match_tests
1547+
1548+
if match_tests is None:
1549+
return True
1550+
test_id = test.id()
1551+
1552+
for match_test in match_tests:
1553+
if fnmatch.fnmatchcase(test_id, match_test):
1554+
return True
1555+
1556+
for name in test_id.split("."):
1557+
if fnmatch.fnmatchcase(name, match_test):
1558+
return True
1559+
return False
1560+
1561+
15451562
def run_unittest(*classes):
15461563
"""Run tests from unittest.TestCase-derived classes."""
15471564
valid_types = (unittest.TestSuite, unittest.TestCase)
@@ -1556,20 +1573,7 @@ def run_unittest(*classes):
15561573
suite.addTest(cls)
15571574
else:
15581575
suite.addTest(unittest.makeSuite(cls))
1559-
def case_pred(test):
1560-
if match_tests is None:
1561-
return True
1562-
test_id = test.id()
1563-
1564-
for match_test in match_tests:
1565-
if fnmatch.fnmatchcase(test_id, match_test):
1566-
return True
1567-
1568-
for name in test_id.split("."):
1569-
if fnmatch.fnmatchcase(name, match_test):
1570-
return True
1571-
return False
1572-
_filter_suite(suite, case_pred)
1576+
_filter_suite(suite, _match_test)
15731577
_run_suite(suite)
15741578

15751579
#=======================================================================

‎Lib/test/test_regrtest.py‎

Lines changed: 96 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -91,19 +91,19 @@ def parse_executed_tests(self, output):
9191
return list(match.group(1) for match in parser)
9292

9393
def check_executed_tests(self, output, tests, skipped=(), failed=(),
94-
omitted=(), randomize=False, interrupted=False):
94+
env_changed=(), omitted=(),
95+
randomize=False, interrupted=False,
96+
fail_env_changed=False):
9597
if isinstance(tests, str):
9698
tests = [tests]
9799
if isinstance(skipped, str):
98100
skipped = [skipped]
99101
if isinstance(failed, str):
100102
failed = [failed]
103+
if isinstance(env_changed, str):
104+
env_changed = [env_changed]
101105
if isinstance(omitted, str):
102106
omitted = [omitted]
103-
ntest = len(tests)
104-
nskipped = len(skipped)
105-
nfailed = len(failed)
106-
nomitted = len(omitted)
107107

108108
executed = self.parse_executed_tests(output)
109109
if randomize:
@@ -129,11 +129,17 @@ def list_regex(line_format, tests):
129129
regex = list_regex('%s test%s failed', failed)
130130
self.check_line(output, regex)
131131

132+
if env_changed:
133+
regex = list_regex('%s test%s altered the execution environment',
134+
env_changed)
135+
self.check_line(output, regex)
136+
132137
if omitted:
133138
regex = list_regex('%s test%s omitted', omitted)
134139
self.check_line(output, regex)
135140

136-
good = ntest - nskipped - nfailed - nomitted
141+
good = (len(tests) - len(skipped) - len(failed)
142+
- len(omitted) - len(env_changed))
137143
if good:
138144
regex = r'%s test%s OK\.$' % (good, plural(good))
139145
if not skipped and not failed and good > 1:
@@ -143,10 +149,12 @@ def list_regex(line_format, tests):
143149
if interrupted:
144150
self.check_line(output, 'Test suite interrupted by signal SIGINT.')
145151

146-
if nfailed:
152+
if failed:
147153
result = 'FAILURE'
148154
elif interrupted:
149155
result = 'INTERRUPTED'
156+
elif fail_env_changed and env_changed:
157+
result = 'ENV CHANGED'
150158
else:
151159
result = 'SUCCESS'
152160
self.check_line(output, 'Tests result: %s' % result)
@@ -325,7 +333,7 @@ def test_main():
325333
test_failing = self.create_test('failing', code=code)
326334
tests = [test_ok, test_failing]
327335

328-
output = self.run_tests(*tests, exitcode=1)
336+
output = self.run_tests(*tests, exitcode=2)
329337
self.check_executed_tests(output, tests, failed=test_failing)
330338

331339
def test_resources(self):
@@ -394,7 +402,7 @@ def test_fromfile(self):
394402
def test_interrupted(self):
395403
code = TEST_INTERRUPTED
396404
test = self.create_test('sigint', code=code)
397-
output = self.run_tests(test, exitcode=1)
405+
output = self.run_tests(test, exitcode=130)
398406
self.check_executed_tests(output, test, omitted=test,
399407
interrupted=True)
400408

@@ -423,7 +431,7 @@ def test_slow_interrupted(self):
423431
args = ("--slowest", "-j2", test)
424432
else:
425433
args = ("--slowest", test)
426-
output = self.run_tests(*args, exitcode=1)
434+
output = self.run_tests(*args, exitcode=130)
427435
self.check_executed_tests(output, test,
428436
omitted=test, interrupted=True)
429437

@@ -461,24 +469,88 @@ def test_main():
461469
support.run_unittest(ForeverTester)
462470
""")
463471
test = self.create_test('forever', code=code)
464-
output = self.run_tests('--forever', test, exitcode=1)
472+
output = self.run_tests('--forever', test, exitcode=2)
465473
self.check_executed_tests(output, [test]*3, failed=test)
466474

475+
def check_leak(self, code, what):
476+
test = self.create_test('huntrleaks', code=code)
477+
478+
filename = 'reflog.txt'
479+
self.addCleanup(support.unlink, filename)
480+
output = self.run_tests('--huntrleaks', '3:3:', test,
481+
exitcode=2,
482+
stderr=subprocess.STDOUT)
483+
self.check_executed_tests(output, [test], failed=test)
484+
485+
line = 'beginning 6 repetitions\n123456\n......\n'
486+
self.check_line(output, re.escape(line))
487+
488+
line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what)
489+
self.assertIn(line2, output)
490+
491+
with open(filename) as fp:
492+
reflog = fp.read()
493+
self.assertIn(line2, reflog)
494+
495+
@unittest.skipUnless(Py_DEBUG, 'need a debug build')
496+
def test_huntrleaks(self):
497+
# test --huntrleaks
498+
code = textwrap.dedent("""
499+
import unittest
500+
from test import support
501+
502+
GLOBAL_LIST = []
503+
504+
class RefLeakTest(unittest.TestCase):
505+
def test_leak(self):
506+
GLOBAL_LIST.append(object())
507+
508+
def test_main():
509+
support.run_unittest(RefLeakTest)
510+
""")
511+
self.check_leak(code, 'references')
512+
467513
def test_list_tests(self):
468514
# test --list-tests
469515
tests = [self.create_test() for i in range(5)]
470516
output = self.run_tests('--list-tests', *tests)
471517
self.assertEqual(output.rstrip().splitlines(),
472518
tests)
473519

520+
def test_list_cases(self):
521+
# test --list-cases
522+
code = textwrap.dedent("""
523+
import unittest
524+
525+
class Tests(unittest.TestCase):
526+
def test_method1(self):
527+
pass
528+
def test_method2(self):
529+
pass
530+
""")
531+
testname = self.create_test(code=code)
532+
533+
# Test --list-cases
534+
all_methods = ['%s.Tests.test_method1' % testname,
535+
'%s.Tests.test_method2' % testname]
536+
output = self.run_tests('--list-cases', testname)
537+
self.assertEqual(output.splitlines(), all_methods)
538+
539+
# Test --list-cases with --match
540+
all_methods = ['%s.Tests.test_method1' % testname]
541+
output = self.run_tests('--list-cases',
542+
'-m', 'test_method1',
543+
testname)
544+
self.assertEqual(output.splitlines(), all_methods)
545+
474546
def test_crashed(self):
475547
# Any code which causes a crash
476548
code = 'import test.support; test.support._crash_python()'
477549
crash_test = self.create_test(name="crash", code=code)
478550
ok_test = self.create_test(name="ok")
479551

480552
tests = [crash_test, ok_test]
481-
output = self.run_tests("-j2", *tests, exitcode=1)
553+
output = self.run_tests("-j2", *tests, exitcode=2)
482554
self.check_executed_tests(output, tests, failed=crash_test,
483555
randomize=True)
484556

@@ -532,26 +604,28 @@ def test_main():
532604
subset = ['test_method1', 'test_method3']
533605
self.assertEqual(methods, subset)
534606

535-
def test_list_cases(self):
536-
# test --list-cases
607+
def test_env_changed(self):
537608
code = textwrap.dedent("""
538609
import unittest
539610
from test import support
540611
541612
class Tests(unittest.TestCase):
542-
def test_method1(self):
543-
pass
544-
def test_method2(self):
545-
pass
613+
def test_env_changed(self):
614+
open("env_changed", "w").close()
546615
547616
def test_main():
548617
support.run_unittest(Tests)
549618
""")
550619
testname = self.create_test(code=code)
551-
all_methods = ['%s.Tests.test_method1' % testname,
552-
'%s.Tests.test_method2' % testname]
553-
output = self.run_tests('--list-cases', testname)
554-
self.assertEqual(output.splitlines(), all_methods)
620+
621+
# don't fail by default
622+
output = self.run_tests(testname)
623+
self.check_executed_tests(output, [testname], env_changed=testname)
624+
625+
# fail with --fail-env-changed
626+
output = self.run_tests("--fail-env-changed", testname, exitcode=3)
627+
self.check_executed_tests(output, [testname], env_changed=testname,
628+
fail_env_changed=True)
555629

556630

557631
def test_main():

0 commit comments

Comments
 (0)