Skip to content

Commit 2f87a7d

Browse files
miss-islingtonYury Selivanov
miss-islington
and
Yury Selivanov
committed
bpo-30773: Fix ag_running; prohibit running athrow/asend/aclose in parallel (GH-7468) (#16486)
(cherry picked from commit fc4a044) Co-authored-by: Yury Selivanov <yury@magic.io>
1 parent 1c19d65 commit 2f87a7d

4 files changed

Lines changed: 54 additions & 64 deletions

File tree

‎Include/genobject.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ typedef struct {
8080
/* Flag is set to 1 when aclose() is called for the first time, or
8181
when a StopAsyncIteration exception is raised. */
8282
int ag_closed;
83+
84+
int ag_running_async;
8385
} PyAsyncGenObject;
8486

8587
PyAPI_DATA(PyTypeObject) PyAsyncGen_Type;

‎Lib/test/test_asyncgen.py‎

Lines changed: 16 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -133,24 +133,6 @@ def async_iterate(g):
133133
break
134134
return res
135135

136-
def async_iterate(g):
137-
res = []
138-
while True:
139-
try:
140-
g.__anext__().__next__()
141-
except StopAsyncIteration:
142-
res.append('STOP')
143-
break
144-
except StopIteration as ex:
145-
if ex.args:
146-
res.append(ex.args[0])
147-
else:
148-
res.append('EMPTY StopIteration')
149-
break
150-
except Exception as ex:
151-
res.append(str(type(ex)))
152-
return res
153-
154136
sync_gen_result = sync_iterate(sync_gen)
155137
async_gen_result = async_iterate(async_gen)
156138
self.assertEqual(sync_gen_result, async_gen_result)
@@ -176,19 +158,22 @@ async def gen():
176158

177159
g = gen()
178160
ai = g.__aiter__()
179-
self.assertEqual(ai.__anext__().__next__(), ('result',))
161+
162+
an = ai.__anext__()
163+
self.assertEqual(an.__next__(), ('result',))
180164

181165
try:
182-
ai.__anext__().__next__()
166+
an.__next__()
183167
except StopIteration as ex:
184168
self.assertEqual(ex.args[0], 123)
185169
else:
186170
self.fail('StopIteration was not raised')
187171

188-
self.assertEqual(ai.__anext__().__next__(), ('result',))
172+
an = ai.__anext__()
173+
self.assertEqual(an.__next__(), ('result',))
189174

190175
try:
191-
ai.__anext__().__next__()
176+
an.__next__()
192177
except StopAsyncIteration as ex:
193178
self.assertFalse(ex.args)
194179
else:
@@ -212,10 +197,11 @@ async def gen():
212197

213198
g = gen()
214199
ai = g.__aiter__()
215-
self.assertEqual(ai.__anext__().__next__(), ('result',))
200+
an = ai.__anext__()
201+
self.assertEqual(an.__next__(), ('result',))
216202

217203
try:
218-
ai.__anext__().__next__()
204+
an.__next__()
219205
except StopIteration as ex:
220206
self.assertEqual(ex.args[0], 123)
221207
else:
@@ -646,17 +632,13 @@ async def run():
646632
gen = foo()
647633
it = gen.__aiter__()
648634
self.assertEqual(await it.__anext__(), 1)
649-
t = self.loop.create_task(it.__anext__())
650-
await asyncio.sleep(0.01)
651635
await gen.aclose()
652-
return t
653636

654-
t = self.loop.run_until_complete(run())
637+
self.loop.run_until_complete(run())
655638
self.assertEqual(DONE, 1)
656639

657640
# Silence ResourceWarnings
658641
fut.cancel()
659-
t.cancel()
660642
self.loop.run_until_complete(asyncio.sleep(0.01))
661643

662644
def test_async_gen_asyncio_gc_aclose_09(self):
@@ -1053,46 +1035,18 @@ async def wait():
10531035

10541036
self.loop.run_until_complete(asyncio.sleep(0.1))
10551037

1056-
self.loop.run_until_complete(self.loop.shutdown_asyncgens())
1057-
self.assertEqual(finalized, 2)
1058-
10591038
# Silence warnings
10601039
t1.cancel()
10611040
t2.cancel()
1062-
self.loop.run_until_complete(asyncio.sleep(0.1))
10631041

1064-
def test_async_gen_asyncio_shutdown_02(self):
1065-
logged = 0
1066-
1067-
def logger(loop, context):
1068-
nonlocal logged
1069-
self.assertIn('asyncgen', context)
1070-
expected = 'an error occurred during closing of asynchronous'
1071-
if expected in context['message']:
1072-
logged += 1
1073-
1074-
async def waiter(timeout):
1075-
try:
1076-
await asyncio.sleep(timeout)
1077-
yield 1
1078-
finally:
1079-
1 / 0
1080-
1081-
async def wait():
1082-
async for _ in waiter(1):
1083-
pass
1084-
1085-
t = self.loop.create_task(wait())
1086-
self.loop.run_until_complete(asyncio.sleep(0.1))
1042+
with self.assertRaises(asyncio.CancelledError):
1043+
self.loop.run_until_complete(t1)
1044+
with self.assertRaises(asyncio.CancelledError):
1045+
self.loop.run_until_complete(t2)
10871046

1088-
self.loop.set_exception_handler(logger)
10891047
self.loop.run_until_complete(self.loop.shutdown_asyncgens())
10901048

1091-
self.assertEqual(logged, 1)
1092-
1093-
# Silence warnings
1094-
t.cancel()
1095-
self.loop.run_until_complete(asyncio.sleep(0.1))
1049+
self.assertEqual(finalized, 2)
10961050

10971051
def test_async_gen_expression_01(self):
10981052
async def arange(n):

‎Misc/NEWS.d/next/Core and Builtins/2018-06-07-01-01-20.bpo-30773.C31rVE.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Prohibit parallel running of aclose() / asend() / athrow(). Fix ag_running
2+
to reflect the actual running status of the AG.

‎Objects/genobject.c‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1342,7 +1342,8 @@ static PyGetSetDef async_gen_getsetlist[] = {
13421342

13431343
static PyMemberDef async_gen_memberlist[] = {
13441344
{"ag_frame", T_OBJECT, offsetof(PyAsyncGenObject, ag_frame), READONLY},
1345-
{"ag_running", T_BOOL, offsetof(PyAsyncGenObject, ag_running), READONLY},
1345+
{"ag_running", T_BOOL, offsetof(PyAsyncGenObject, ag_running_async),
1346+
READONLY},
13461347
{"ag_code", T_OBJECT, offsetof(PyAsyncGenObject, ag_code), READONLY},
13471348
{NULL} /* Sentinel */
13481349
};
@@ -1436,6 +1437,7 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
14361437
o->ag_finalizer = NULL;
14371438
o->ag_closed = 0;
14381439
o->ag_hooks_inited = 0;
1440+
o->ag_running_async = 0;
14391441
return (PyObject*)o;
14401442
}
14411443

@@ -1483,13 +1485,15 @@ async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result)
14831485
gen->ag_closed = 1;
14841486
}
14851487

1488+
gen->ag_running_async = 0;
14861489
return NULL;
14871490
}
14881491

14891492
if (_PyAsyncGenWrappedValue_CheckExact(result)) {
14901493
/* async yield */
14911494
_PyGen_SetStopIterationValue(((_PyAsyncGenWrappedValue*)result)->agw_val);
14921495
Py_DECREF(result);
1496+
gen->ag_running_async = 0;
14931497
return NULL;
14941498
}
14951499

@@ -1534,12 +1538,20 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg)
15341538
}
15351539

15361540
if (o->ags_state == AWAITABLE_STATE_INIT) {
1541+
if (o->ags_gen->ag_running_async) {
1542+
PyErr_SetString(
1543+
PyExc_RuntimeError,
1544+
"anext(): asynchronous generator is already running");
1545+
return NULL;
1546+
}
1547+
15371548
if (arg == NULL || arg == Py_None) {
15381549
arg = o->ags_sendval;
15391550
}
15401551
o->ags_state = AWAITABLE_STATE_ITER;
15411552
}
15421553

1554+
o->ags_gen->ag_running_async = 1;
15431555
result = gen_send_ex((PyGenObject*)o->ags_gen, arg, 0, 0);
15441556
result = async_gen_unwrap_value(o->ags_gen, result);
15451557

@@ -1803,8 +1815,23 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
18031815
}
18041816

18051817
if (o->agt_state == AWAITABLE_STATE_INIT) {
1818+
if (o->agt_gen->ag_running_async) {
1819+
if (o->agt_args == NULL) {
1820+
PyErr_SetString(
1821+
PyExc_RuntimeError,
1822+
"aclose(): asynchronous generator is already running");
1823+
}
1824+
else {
1825+
PyErr_SetString(
1826+
PyExc_RuntimeError,
1827+
"athrow(): asynchronous generator is already running");
1828+
}
1829+
return NULL;
1830+
}
1831+
18061832
if (o->agt_gen->ag_closed) {
1807-
PyErr_SetNone(PyExc_StopIteration);
1833+
o->agt_state = AWAITABLE_STATE_CLOSED;
1834+
PyErr_SetNone(PyExc_StopAsyncIteration);
18081835
return NULL;
18091836
}
18101837

@@ -1814,6 +1841,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
18141841
}
18151842

18161843
o->agt_state = AWAITABLE_STATE_ITER;
1844+
o->agt_gen->ag_running_async = 1;
18171845

18181846
if (o->agt_args == NULL) {
18191847
/* aclose() mode */
@@ -1859,6 +1887,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
18591887
/* aclose() mode */
18601888
if (retval) {
18611889
if (_PyAsyncGenWrappedValue_CheckExact(retval)) {
1890+
o->agt_gen->ag_running_async = 0;
18621891
Py_DECREF(retval);
18631892
goto yield_close;
18641893
}
@@ -1872,11 +1901,13 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
18721901
}
18731902

18741903
yield_close:
1904+
o->agt_gen->ag_running_async = 0;
18751905
PyErr_SetString(
18761906
PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG);
18771907
return NULL;
18781908

18791909
check_error:
1910+
o->agt_gen->ag_running_async = 0;
18801911
if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) ||
18811912
PyErr_ExceptionMatches(PyExc_GeneratorExit))
18821913
{
@@ -1911,6 +1942,7 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *args)
19111942
} else {
19121943
/* aclose() mode */
19131944
if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) {
1945+
o->agt_gen->ag_running_async = 0;
19141946
Py_DECREF(retval);
19151947
PyErr_SetString(PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG);
19161948
return NULL;

0 commit comments

Comments
 (0)