Skip to content

Commit 0122b4d

Browse files
ericsnowcurrentlyericsnowcurrently
authored
[3.12] gh-105716: Support Background Threads in Subinterpreters Consistently (gh-109921) (gh-110707)
The existence of background threads running on a subinterpreter was preventing interpreters from getting properly destroyed, as well as impacting the ability to run the interpreter again. It also affected how we wait for non-daemon threads to finish. We add PyInterpreterState.threads.main, with some internal C-API functions. (cherry-picked from commit 1dd9dee)
1 parent 82ae5a6 commit 0122b4d

File tree

11 files changed

+385
-150
lines changed

11 files changed

+385
-150
lines changed

‎Doc/data/python3.12.abi‎

Lines changed: 123 additions & 105 deletions
Large diffs are not rendered by default.

‎Include/internal/pycore_interp.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,8 @@ struct _is {
194194
struct _Py_interp_cached_objects cached_objects;
195195
struct _Py_interp_static_objects static_objects;
196196

197+
/* The thread currently executing in the __main__ module, if any. */
198+
PyThreadState *threads_main;
197199
/* The ID of the OS thread in which we are finalizing.
198200
We use _Py_atomic_address instead of adding a new _Py_atomic_ulong. */
199201
_Py_atomic_address _finalizing_id;

‎Include/internal/pycore_pystate.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp)
4444
interp == &_PyRuntime._main_interpreter);
4545
}
4646

47+
// Export for _xxsubinterpreters module.
48+
PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *);
49+
PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *);
50+
PyAPI_FUNC(int) _PyInterpreterState_IsRunningMain(PyInterpreterState *);
51+
4752

4853
static inline const PyConfig *
4954
_Py_GetMainConfig(void)

‎Lib/test/test_interpreters.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,16 @@ def test_subinterpreter(self):
261261
self.assertTrue(interp.is_running())
262262
self.assertFalse(interp.is_running())
263263

264+
def test_finished(self):
265+
r, w = os.pipe()
266+
interp = interpreters.create()
267+
interp.run(f"""if True:
268+
import os
269+
os.write({w}, b'x')
270+
""")
271+
self.assertFalse(interp.is_running())
272+
self.assertEqual(os.read(r, 1), b'x')
273+
264274
def test_from_subinterpreter(self):
265275
interp = interpreters.create()
266276
out = _run_output(interp, dedent(f"""
@@ -288,6 +298,31 @@ def test_bad_id(self):
288298
with self.assertRaises(ValueError):
289299
interp.is_running()
290300

301+
def test_with_only_background_threads(self):
302+
r_interp, w_interp = os.pipe()
303+
r_thread, w_thread = os.pipe()
304+
305+
DONE = b'D'
306+
FINISHED = b'F'
307+
308+
interp = interpreters.create()
309+
interp.run(f"""if True:
310+
import os
311+
import threading
312+
313+
def task():
314+
v = os.read({r_thread}, 1)
315+
assert v == {DONE!r}
316+
os.write({w_interp}, {FINISHED!r})
317+
t = threading.Thread(target=task)
318+
t.start()
319+
""")
320+
self.assertFalse(interp.is_running())
321+
322+
os.write(w_thread, DONE)
323+
interp.run('t.join()')
324+
self.assertEqual(os.read(r_interp, 1), FINISHED)
325+
291326

292327
class TestInterpreterClose(TestBase):
293328

@@ -389,6 +424,37 @@ def test_still_running(self):
389424
interp.close()
390425
self.assertTrue(interp.is_running())
391426

427+
def test_subthreads_still_running(self):
428+
r_interp, w_interp = os.pipe()
429+
r_thread, w_thread = os.pipe()
430+
431+
FINISHED = b'F'
432+
433+
interp = interpreters.create()
434+
interp.run(f"""if True:
435+
import os
436+
import threading
437+
import time
438+
439+
done = False
440+
441+
def notify_fini():
442+
global done
443+
done = True
444+
t.join()
445+
threading._register_atexit(notify_fini)
446+
447+
def task():
448+
while not done:
449+
time.sleep(0.1)
450+
os.write({w_interp}, {FINISHED!r})
451+
t = threading.Thread(target=task)
452+
t.start()
453+
""")
454+
interp.close()
455+
456+
self.assertEqual(os.read(r_interp, 1), FINISHED)
457+
392458

393459
class TestInterpreterRun(TestBase):
394460

@@ -465,6 +531,37 @@ def test_bytes_for_script(self):
465531
with self.assertRaises(TypeError):
466532
interp.run(b'print("spam")')
467533

534+
def test_with_background_threads_still_running(self):
535+
r_interp, w_interp = os.pipe()
536+
r_thread, w_thread = os.pipe()
537+
538+
RAN = b'R'
539+
DONE = b'D'
540+
FINISHED = b'F'
541+
542+
interp = interpreters.create()
543+
interp.run(f"""if True:
544+
import os
545+
import threading
546+
547+
def task():
548+
v = os.read({r_thread}, 1)
549+
assert v == {DONE!r}
550+
os.write({w_interp}, {FINISHED!r})
551+
t = threading.Thread(target=task)
552+
t.start()
553+
os.write({w_interp}, {RAN!r})
554+
""")
555+
interp.run(f"""if True:
556+
os.write({w_interp}, {RAN!r})
557+
""")
558+
559+
os.write(w_thread, DONE)
560+
interp.run('t.join()')
561+
self.assertEqual(os.read(r_interp, 1), RAN)
562+
self.assertEqual(os.read(r_interp, 1), RAN)
563+
self.assertEqual(os.read(r_interp, 1), FINISHED)
564+
468565
# test_xxsubinterpreters covers the remaining Interpreter.run() behavior.
469566

470567

‎Lib/test/test_threading.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,11 @@
2626
from test import lock_tests
2727
from test import support
2828

29+
try:
30+
from test.support import interpreters
31+
except ModuleNotFoundError:
32+
interpreters = None
33+
2934
threading_helper.requires_working_threading(module=True)
3035

3136
# Between fork() and exec(), only async-safe functions are allowed (issues
@@ -45,6 +50,12 @@ def skip_unless_reliable_fork(test):
4550
return test
4651

4752

53+
def requires_subinterpreters(meth):
54+
"""Decorator to skip a test if subinterpreters are not supported."""
55+
return unittest.skipIf(interpreters is None,
56+
'subinterpreters required')(meth)
57+
58+
4859
def restore_default_excepthook(testcase):
4960
testcase.addCleanup(setattr, threading, 'excepthook', threading.excepthook)
5061
threading.excepthook = threading.__excepthook__
@@ -1296,6 +1307,44 @@ def f():
12961307
# The thread was joined properly.
12971308
self.assertEqual(os.read(r, 1), b"x")
12981309

1310+
@requires_subinterpreters
1311+
def test_threads_join_with_no_main(self):
1312+
r_interp, w_interp = self.pipe()
1313+
1314+
INTERP = b'I'
1315+
FINI = b'F'
1316+
DONE = b'D'
1317+
1318+
interp = interpreters.create()
1319+
interp.run(f"""if True:
1320+
import os
1321+
import threading
1322+
import time
1323+
1324+
done = False
1325+
1326+
def notify_fini():
1327+
global done
1328+
done = True
1329+
os.write({w_interp}, {FINI!r})
1330+
t.join()
1331+
threading._register_atexit(notify_fini)
1332+
1333+
def task():
1334+
while not done:
1335+
time.sleep(0.1)
1336+
os.write({w_interp}, {DONE!r})
1337+
t = threading.Thread(target=task)
1338+
t.start()
1339+
1340+
os.write({w_interp}, {INTERP!r})
1341+
""")
1342+
interp.close()
1343+
1344+
self.assertEqual(os.read(r_interp, 1), INTERP)
1345+
self.assertEqual(os.read(r_interp, 1), FINI)
1346+
self.assertEqual(os.read(r_interp, 1), DONE)
1347+
12991348
@cpython_only
13001349
def test_daemon_threads_fatal_error(self):
13011350
subinterp_code = f"""if 1:

‎Lib/threading.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
_allocate_lock = _thread.allocate_lock
3838
_set_sentinel = _thread._set_sentinel
3939
get_ident = _thread.get_ident
40+
_is_main_interpreter = _thread._is_main_interpreter
4041
try:
4142
get_native_id = _thread.get_native_id
4243
_HAVE_THREAD_NATIVE_ID = True
@@ -1566,7 +1567,7 @@ def _shutdown():
15661567
# the main thread's tstate_lock - that won't happen until the interpreter
15671568
# is nearly dead. So we release it here. Note that just calling _stop()
15681569
# isn't enough: other threads may already be waiting on _tstate_lock.
1569-
if _main_thread._is_stopped:
1570+
if _main_thread._is_stopped and _is_main_interpreter():
15701571
# _shutdown() was already called
15711572
return
15721573

@@ -1619,6 +1620,7 @@ def main_thread():
16191620
In normal conditions, the main thread is the thread from which the
16201621
Python interpreter was started.
16211622
"""
1623+
# XXX Figure this out for subinterpreters. (See gh-75698.)
16221624
return _main_thread
16231625

16241626
# get thread-local implementation, either from the thread

‎Misc/NEWS.d/next/Core and Builtins/2023-09-26-14-00-25.gh-issue-105716.SUJkW1.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Subinterpreters now correctly handle the case where they have threads
2+
running in the background. Before, such threads would interfere with
3+
cleaning up and destroying them, as well as prevent running another script.

‎Modules/_threadmodule.c‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1604,6 +1604,18 @@ PyDoc_STRVAR(excepthook_doc,
16041604
\n\
16051605
Handle uncaught Thread.run() exception.");
16061606

1607+
static PyObject *
1608+
thread__is_main_interpreter(PyObject *module, PyObject *Py_UNUSED(ignored))
1609+
{
1610+
PyInterpreterState *interp = _PyInterpreterState_GET();
1611+
return PyBool_FromLong(_Py_IsMainInterpreter(interp));
1612+
}
1613+
1614+
PyDoc_STRVAR(thread__is_main_interpreter_doc,
1615+
"_is_main_interpreter()\n\
1616+
\n\
1617+
Return True if the current interpreter is the main Python interpreter.");
1618+
16071619
static PyMethodDef thread_methods[] = {
16081620
{"start_new_thread", (PyCFunction)thread_PyThread_start_new_thread,
16091621
METH_VARARGS, start_new_doc},
@@ -1633,8 +1645,10 @@ static PyMethodDef thread_methods[] = {
16331645
METH_VARARGS, stack_size_doc},
16341646
{"_set_sentinel", thread__set_sentinel,
16351647
METH_NOARGS, _set_sentinel_doc},
1636-
{"_excepthook", thread_excepthook,
1648+
{"_excepthook", thread_excepthook,
16371649
METH_O, excepthook_doc},
1650+
{"_is_main_interpreter", thread__is_main_interpreter,
1651+
METH_NOARGS, thread__is_main_interpreter_doc},
16381652
{NULL, NULL} /* sentinel */
16391653
};
16401654

0 commit comments

Comments
 (0)