Skip to content

Add CPython 3.10's test/support/threading_helper #3553

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 3 commits into from
Feb 14, 2022
Merged

Add CPython 3.10's test/support/threading_helper #3553

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e41f43d
Add CPython 3.10's test/support/threading_helper
fanninpm Feb 13, 2022
ef10487
Migrate threading functions and class to helper
fanninpm Feb 13, 2022
c8748bc
Fix test.support imports
fanninpm Feb 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions Lib/test/lock_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import weakref

from test import support
from test.support import threading_helper


def _wait():
Expand All @@ -31,7 +32,7 @@ def __init__(self, f, n, wait_before_exit=False):
self.started = []
self.finished = []
self._can_exit = not wait_before_exit
self.wait_thread = support.wait_threads_exit()
self.wait_thread = threading_helper.wait_threads_exit()
self.wait_thread.__enter__()

def task():
Expand Down Expand Up @@ -67,10 +68,10 @@ def do_finish(self):

class BaseTestCase(unittest.TestCase):
def setUp(self):
self._threads = support.threading_setup()
self._threads = threading_helper.threading_setup()

def tearDown(self):
support.threading_cleanup(*self._threads)
threading_helper.threading_cleanup(*self._threads)
support.reap_children()

def assertTimeout(self, actual, expected):
Expand Down Expand Up @@ -235,7 +236,7 @@ def f():
lock.acquire()
phase.append(None)

with support.wait_threads_exit():
with threading_helper.wait_threads_exit():
start_new_thread(f, ())
while len(phase) == 0:
_wait()
Expand Down
5 changes: 3 additions & 2 deletions Lib/test/pickletester.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,11 @@
from test.support import os_helper
from test.support import (
TestFailed, run_with_locale, no_tracing,
_2G, _4G, bigmemtest, reap_threads
_2G, _4G, bigmemtest
)
from test.support.import_helper import forget
from test.support.os_helper import TESTFN
from test.support import threading_helper
from test.support.warnings_helper import save_restore_warnings_filters

from pickle import bytes_types
Expand Down Expand Up @@ -1378,7 +1379,7 @@ def test_truncated_data(self):
for p in badpickles:
self.check_unpickling_error(self.truncated_errors, p)

@reap_threads
@threading_helper.reap_threads
def test_unpickle_module_race(self):
# https://bugs.python.org/issue34572
locker_module = dedent("""
Expand Down
193 changes: 0 additions & 193 deletions Lib/test/support/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,6 @@
'temp_umask', "reap_children",
# logging
"TestHandler",
# threads
"threading_setup", "threading_cleanup", "reap_threads", "start_threads",
# miscellaneous
"check_warnings", "check_no_resource_warning", "check_no_warnings",
"EnvironmentVarGuard",
Expand Down Expand Up @@ -1541,104 +1539,6 @@ def print_warning(msg):
# to cleanup threads.
environment_altered = False

# NOTE: we use thread._count() rather than threading.enumerate() (or the
# moral equivalent thereof) because a threading.Thread object is still alive
# until its __bootstrap() method has returned, even after it has been
# unregistered from the threading module.
# thread._count(), on the other hand, only gets decremented *after* the
# __bootstrap() method has returned, which gives us reliable reference counts
# at the end of a test run.

def threading_setup():
return _thread._count(), threading._dangling.copy()

def threading_cleanup(*original_values):
global environment_altered

_MAX_COUNT = 100

for count in range(_MAX_COUNT):
values = _thread._count(), threading._dangling
if values == original_values:
break

if not count: # Display a warning at the first iteration
environment_altered = True
dangling_threads = values[1]
print_warning(f"threading_cleanup() failed to cleanup "
f"{values[0] - original_values[0]} threads "
f"(count: {values[0]}, "
f"dangling: {len(dangling_threads)})")
for thread in dangling_threads:
print_warning(f"Dangling thread: {thread!r}")

# Don't hold references to threads
dangling_threads = None
values = None

time.sleep(0.01)
gc_collect()


def reap_threads(func):
"""Use this function when threads are being used. This will
ensure that the threads are cleaned up even when the test fails.
"""
@functools.wraps(func)
def decorator(*args):
key = threading_setup()
try:
return func(*args)
finally:
threading_cleanup(*key)
return decorator


@contextlib.contextmanager
def wait_threads_exit(timeout=60.0):
"""
bpo-31234: Context manager to wait until all threads created in the with
statement exit.

Use _thread.count() to check if threads exited. Indirectly, wait until
threads exit the internal t_bootstrap() C function of the _thread module.

threading_setup() and threading_cleanup() are designed to emit a warning
if a test leaves running threads in the background. This context manager
is designed to cleanup threads started by the _thread.start_new_thread()
which doesn't allow to wait for thread exit, whereas thread.Thread has a
join() method.
"""
old_count = _thread._count()
try:
yield
finally:
start_time = time.monotonic()
deadline = start_time + timeout
while True:
count = _thread._count()
if count <= old_count:
break
if time.monotonic() > deadline:
dt = time.monotonic() - start_time
msg = (f"wait_threads() failed to cleanup {count - old_count} "
f"threads after {dt:.1f} seconds "
f"(count: {count}, old count: {old_count})")
raise AssertionError(msg)
time.sleep(0.010)
gc_collect()


def join_thread(thread, timeout=30.0):
"""Join a thread. Raise an AssertionError if the thread is still alive
after timeout seconds.
"""
thread.join(timeout)
if thread.is_alive():
msg = f"failed to join the thread in {timeout:.1f} seconds"
raise AssertionError(msg)


def reap_children():
"""Use this function at the end of test_main() whenever sub-processes
are started. This will help ensure that no extra children (zombies)
Expand Down Expand Up @@ -1667,42 +1567,6 @@ def reap_children():
environment_altered = True


@contextlib.contextmanager
def start_threads(threads, unlock=None):
threads = list(threads)
started = []
try:
try:
for t in threads:
t.start()
started.append(t)
except:
if verbose:
print("Can't start %d threads, only %d threads started" %
(len(threads), len(started)))
raise
yield
finally:
try:
if unlock:
unlock()
endtime = starttime = time.monotonic()
for timeout in range(1, 16):
endtime += 60
for t in started:
t.join(max(endtime - time.monotonic(), 0.01))
started = [t for t in started if t.is_alive()]
if not started:
break
if verbose:
print('Unable to join %d threads during a period of '
'%d minutes' % (len(started), timeout))
finally:
started = [t for t in started if t.is_alive()]
if started:
faulthandler.dump_traceback(sys.stdout)
raise AssertionError('Unable to join %d threads' % len(started))

@contextlib.contextmanager
def swap_attr(obj, attr, new_val):
"""Temporary swap out an attribute with a new object.
Expand Down Expand Up @@ -2507,63 +2371,6 @@ def __exit__(self, *exc_info):
del self.unraisable


class catch_threading_exception:
"""
Context manager catching threading.Thread exception using
threading.excepthook.

Attributes set when an exception is catched:

* exc_type
* exc_value
* exc_traceback
* thread

See threading.excepthook() documentation for these attributes.

These attributes are deleted at the context manager exit.

Usage:

with support.catch_threading_exception() as cm:
# code spawning a thread which raises an exception
...

# check the thread exception, use cm attributes:
# exc_type, exc_value, exc_traceback, thread
...

# exc_type, exc_value, exc_traceback, thread attributes of cm no longer
# exists at this point
# (to avoid reference cycles)
"""

def __init__(self):
self.exc_type = None
self.exc_value = None
self.exc_traceback = None
self.thread = None
self._old_hook = None

def _hook(self, args):
self.exc_type = args.exc_type
self.exc_value = args.exc_value
self.exc_traceback = args.exc_traceback
self.thread = args.thread

def __enter__(self):
self._old_hook = threading.excepthook
threading.excepthook = self._hook
return self

def __exit__(self, *exc_info):
threading.excepthook = self._old_hook
del self.exc_type
del self.exc_value
del self.exc_traceback
del self.thread


def wait_process(pid, *, exitcode, timeout=None):
"""
Wait until process pid completes and check that the process exit code is
Expand Down
Loading