Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 185 additions & 0 deletions Lib/test/bisect_cmd.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
#!/usr/bin/env python3

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Keep new harness logic out of Lib/test

This new executable helper is test-harness logic, and the same commit also changes runner selection, cleanup, and test data in other Lib/test files. Repository rules forbid modifying test logic or data and limit acceptable test-file changes to adding or removing expected-failure markers, so this synchronization must be reworked without these test-file logic edits.

AGENTS.md reference: AGENTS.md:L278-L282

Useful? React with 👍 / 👎.

"""
Command line tool to bisect failing CPython tests.
Find the test_os test method which alters the environment:
./python -m test.bisect_cmd --fail-env-changed test_os
Find a reference leak in "test_os", write the list of failing tests into the
"bisect" file:
./python -m test.bisect_cmd -o bisect -R 3:3 test_os
Load an existing list of tests from a file using -i option:
./python -m test --list-cases -m FileTests test_os > tests
./python -m test.bisect_cmd -i tests test_os
"""

import argparse
import datetime
import os.path
import math
import random
import subprocess
import sys
import tempfile
import time


def write_tests(filename, tests):
with open(filename, "w") as fp:
for name in tests:
print(name, file=fp)
fp.flush()


def write_output(filename, tests):
if not filename:
return
print("Writing %s tests into %s" % (len(tests), filename))
write_tests(filename, tests)
return filename


def format_shell_args(args):
return ' '.join(args)


def python_cmd():
cmd = [sys.executable]
cmd.extend(subprocess._args_from_interpreter_flags())
cmd.extend(subprocess._optim_args_from_interpreter_flags())
Comment on lines +52 to +53

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Forward the optimization flag only once

When bisection starts under -O, the repository's _args_from_interpreter_flags() already initializes its result from _optim_args_from_interpreter_flags() (Lib/subprocess.py:421-435), and the next line appends that optimization flag again. RustPython's CLI help distinguishes -O from -OO because the latter also discards docstrings (src/settings.rs:79-82), while the parser increments the level for every occurrence (src/settings.rs:145), so the list/test children run at -OO instead of the parent's -O and can stop reproducing docstring-sensitive failures; remove the second append.

Useful? React with 👍 / 👎.

cmd.extend(('-X', 'faulthandler'))
return cmd


def list_cases(args):
cmd = python_cmd()
cmd.extend(['-m', 'test', '--list-cases'])
cmd.extend(args.test_args)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve --python semantics during bisection

In a -j ... --python COMMAND --bisect run, the checked -m test --help contract describes --python as the test-subprocess command (Lib/test/libregrtest/cmdline.py:256-257) and --bisect as invoking this module (Lib/test/libregrtest/cmdline.py:394-395). RunTests.bisect_cmd_args() forwards --python but not -j, and appending it here creates python -m test --list-cases --python COMMAND; _parse_args() rejects that combination at Lib/test/libregrtest/cmdline.py:524-526 with -p requires -j!, so custom-interpreter bisection aborts before listing cases. Carry the interpreter command separately instead of forwarding this invalid regrtest combination.

Useful? React with 👍 / 👎.

proc = subprocess.run(cmd,
stdout=subprocess.PIPE,
universal_newlines=True)
exitcode = proc.returncode
if exitcode:
cmd = format_shell_args(cmd)
print("Failed to list tests: %s failed with exit code %s"
% (cmd, exitcode))
sys.exit(exitcode)
tests = proc.stdout.splitlines()
return tests


def run_tests(args, tests, huntrleaks=None):
tmp = tempfile.mktemp()
try:
write_tests(tmp, tests)

cmd = python_cmd()
cmd.extend(['-u', '-m', 'test', '--matchfile', tmp])
cmd.extend(args.test_args)
print("+ %s" % format_shell_args(cmd))

sys.stdout.flush()
sys.stderr.flush()

proc = subprocess.run(cmd)
return proc.returncode
finally:
if os.path.exists(tmp):
os.unlink(tmp)


def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--input',
help='Test names produced by --list-tests written '
'into a file. If not set, run --list-tests')
parser.add_argument('-o', '--output',
help='Result of the bisection')
parser.add_argument('-n', '--max-tests', type=int, default=1,
help='Maximum number of tests to stop the bisection '
'(default: 1)')
parser.add_argument('-N', '--max-iter', type=int, default=100,
help='Maximum number of bisection iterations '
'(default: 100)')
# FIXME: document that following arguments are test arguments

args, test_args = parser.parse_known_args()
args.test_args = test_args
return args


def main():
args = parse_args()
for opt in ('-w', '--rerun', '--verbose2'):
if opt in args.test_args:
print(f"WARNING: {opt} option should not be used to bisect!")
print()

if args.input:
with open(args.input) as fp:
tests = [line.strip() for line in fp]
else:
tests = list_cases(args)

print("Start bisection with %s tests" % len(tests))
print("Test arguments: %s" % format_shell_args(args.test_args))
print("Bisection will stop when getting %s or less tests "
"(-n/--max-tests option), or after %s iterations "
"(-N/--max-iter option)"
% (args.max_tests, args.max_iter))
output = write_output(args.output, tests)
print()

start_time = time.monotonic()
iteration = 1
try:
while len(tests) > args.max_tests and iteration <= args.max_iter:
ntest = len(tests)
ntest = max(ntest // 2, 1)
subtests = random.sample(tests, ntest)

print(f"[+] Iteration {iteration}/{args.max_iter}: "
f"run {len(subtests)} tests/{len(tests)}")
print()

exitcode = run_tests(args, subtests)

print("ran %s tests/%s" % (ntest, len(tests)))
print("exit", exitcode)
if exitcode:
print("Tests failed: continuing with this subtest")
tests = subtests
output = write_output(args.output, tests)
else:
print("Tests succeeded: skipping this subtest, trying a new subset")
print()
iteration += 1
except KeyboardInterrupt:
print()
print("Bisection interrupted!")
print()

print("Tests (%s):" % len(tests))
for test in tests:
print("* %s" % test)
print()

if output:
print("Output written into %s" % output)

dt = math.ceil(time.monotonic() - start_time)
if len(tests) <= args.max_tests:
print("Bisection completed in %s iterations and %s"
% (iteration, datetime.timedelta(seconds=dt)))
else:
print("Bisection failed after %s iterations and %s"
% (iteration, datetime.timedelta(seconds=dt)))
sys.exit(1)


if __name__ == "__main__":
main()
48 changes: 48 additions & 0 deletions Lib/test/cov.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
"""A minimal hook for gathering line coverage of the standard library.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Add the required Assisted-by trailer

The commit explicitly states that Codex assisted with implementation and validation, but its parsed trailers contain no Assisted-by entry. Repository policy requires every AI-assisted contribution to use an Assisted-by: AGENT_NAME:MODEL_VERSION trailer, so the required trailer must be supplied before merge.

AGENTS.md reference: AGENTS.md:L20-L23

Useful? React with 👍 / 👎.

Designed to be used with -Xpresite=test.cov:enable which means:
* it can't import anything besides `sys` to avoid tainting gathered coverage
* filenames are not normalized
To get gathered coverage back, look for 'test.cov' in `sys.modules`
instead of importing directly. That way you can determine if the module
was already in use.
If you need to disable the hook, call the `disable()` function.
"""

import sys

mon = sys.monitoring

FileName = str
LineNo = int
Location = tuple[FileName, LineNo]

coverage: set[Location] = set()


# `types` and `typing` aren't imported to avoid invalid coverage
def add_line(
code: "types.CodeType",
lineno: int,
) -> "typing.Literal[sys.monitoring.DISABLE]":
coverage.add((code.co_filename, lineno))
return mon.DISABLE


def enable():
mon.use_tool_id(mon.COVERAGE_ID, "regrtest coverage")
mon.register_callback(mon.COVERAGE_ID, mon.events.LINE, add_line)
mon.set_events(mon.COVERAGE_ID, mon.events.LINE)


def disable():
mon.set_events(mon.COVERAGE_ID, 0)
mon.register_callback(mon.COVERAGE_ID, mon.events.LINE, None)
mon.free_tool_id(mon.COVERAGE_ID)


if __name__ == "__main__":
enable()
9 changes: 9 additions & 0 deletions Lib/test/libregrtest/refleak.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,13 @@ def runtest_refleak(test_name, test_func,
for obj in abc.__subclasses__() + [abc]:
abcs[obj] = _get_dump(obj)[0]

# `ByteString` is not included in `collections.abc.__all__`
with warnings.catch_warnings(action='ignore', category=DeprecationWarning):
ByteString = collections.abc.ByteString
# Mypy doesn't even think `ByteString` is a class, hence the `type: ignore`
for obj in ByteString.__subclasses__() + [ByteString]: # type: ignore[attr-defined]
abcs[obj] = _get_dump(obj)[0]

# bpo-31217: Integer pool to get a single integer object for the same
# value. The pool is used to prevent false alarm when checking for memory
# block leaks. Fill the pool with values in -1000..1000 which are the most
Expand Down Expand Up @@ -254,6 +261,8 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs, linecache_data):

# Clear ABC registries, restoring previously saved ABC registries.
abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__]
with warnings.catch_warnings(action='ignore', category=DeprecationWarning):
abs_classes.append(collections.abc.ByteString)
abs_classes = filter(isabstract, abs_classes)
for abc in abs_classes:
for obj in abc.__subclasses__() + [abc]:
Expand Down
9 changes: 6 additions & 3 deletions Lib/test/libregrtest/run_workers.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,9 +147,12 @@ def _kill(self) -> None:

use_killpg = USE_PROCESS_GROUP
if use_killpg:
parent_sid = os.getsid(0)
sid = os.getsid(popen.pid)
use_killpg = (sid != parent_sid)
try:

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Run the mandatory validation suites

The commit's validation section explicitly says the full workspace/CAPI suites were not run and that the changed modules were not executed or imported, despite adding platform- and process-sensitive runner behavior here. Repository validation requires the workspace suite, the CAPI suite, and relevant RustPython module tests for Lib/ changes before completion, so those checks are mandatory before merge.

AGENTS.md reference: AGENTS.md:L31-L35

Useful? React with 👍 / 👎.

use_killpg = (os.getsid(popen.pid) != os.getsid(0))
except PermissionError:
# On OpenBSD getsid() is only allowed for a process in the
# same session, so the failure means that it is not.
use_killpg = True

if use_killpg:
what = f"{self} process group"
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/libregrtest/runtests.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ def create_python_cmd(self) -> list[str]:
if '-u' not in python_opts:
cmd.append('-u') # Unbuffered stdout and stderr
if self.coverage:
cmd.append("-Xpresite=test.cov")
cmd.append("-Xpresite=test.cov:enable")
return cmd

def bisect_cmd_args(self) -> list[str]:
Expand Down
8 changes: 8 additions & 0 deletions Lib/test/libregrtest/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,14 @@ def setup_process() -> None:
for signum in signals:
faulthandler.register(signum, chain=True, file=stderr_fd)

# Restore the default SIGINT handler if there is no Python-level
# handler. Python inherits the SIG_IGN disposition when the test
# suite runs as a shell background job; then asyncio.Runner does not
# install its own handler and _thread.interrupt_main() is a no-op,
# which makes some tests in test_asyncio and test_threading hang.
if signal.getsignal(signal.SIGINT) in (signal.SIG_IGN, signal.SIG_DFL):
signal.signal(signal.SIGINT, signal.default_int_handler)

adjust_rlimit_nofile()

support.record_original_stdout(sys.stdout)
Expand Down
8 changes: 4 additions & 4 deletions Lib/test/libregrtest/tsan.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,12 @@

TSAN_TESTS = [
'test_asyncio',
# TODO: enable more of test_capi once bugs are fixed (GH-116908, GH-116909).
'test_capi.test_mem',
'test_capi.test_pyatomic',
'test_capi',
'test_code',
'test_ctypes',
# 'test_concurrent_futures', # gh-130605: too many data races
'test_concurrent_futures',
'test_enum',
'test_enumerate',
'test_functools',
'test_httpservers',
'test_imaplib',
Expand All @@ -31,6 +30,7 @@
'test_threadsignals',
'test_weakref',
'test_free_threading',
'test_embed',
]

# Tests that should be run with `--parallel-threads=N` under TSAN. These tests
Expand Down
4 changes: 2 additions & 2 deletions Lib/test/subprocessdata/fd_status.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
file descriptors on stdout.
Usage:
fd_stats.py: check all file descriptors
fd_status.py: check all file descriptors (up to 255)
fd_status.py fd1 fd2 ...: check only specified file descriptors
"""

Expand All @@ -18,7 +18,7 @@
_MAXFD = os.sysconf("SC_OPEN_MAX")
except:
_MAXFD = 256
test_fds = range(0, _MAXFD)
test_fds = range(0, min(_MAXFD, 256))
else:
test_fds = map(int, sys.argv[1:])
for fd in test_fds:
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/support/strace_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ def sections(self):
def _filter_memory_call(call):
# mmap can operate on a fd or "MAP_ANONYMOUS" which gives a block of memory.
# Ignore "MAP_ANONYMOUS + the "MAP_ANON" alias.
if call.syscall == "mmap" and "MAP_ANON" in call.args[3]:
if call.syscall in ("mmap", "mmap2") and "MAP_ANON" in call.args[3]:
return True

if call.syscall in ("munmap", "mprotect"):
Expand Down
Loading