Skip to content

Commit 5f841b5

Browse files
asvetlovasvetlov
authored
bpo-32193: Convert asyncio to async/await usage (#4753)
* Convert asyncio/tasks.py to async/await * Convert asyncio/queues.py to async/await * Convert asyncio/test_utils.py to async/await * Convert asyncio/base_subprocess.py to async/await * Convert asyncio/subprocess.py to async/await * Convert asyncio/streams.py to async/await * Fix comments * Convert asyncio/locks.py to async/await * Convert asyncio.sleep to async def * Add a comment * Add missing news * Convert stubs from AbstrctEventLoop to async functions * Convert subprocess_shell/subprocess_exec * Convert connect_read_pipe/connect_write_pip to async/await syntax * Convert create_datagram_endpoint * Convert create_unix_server/create_unix_connection * Get rid of old style coroutines in unix_events.py * Convert selector_events.py to async/await * Convert wait_closed and create_connection * Drop redundant line * Convert base_events.py * Code cleanup * Drop redundant comments * Fix indentation * Add explicit tests for compatibility between old and new coroutines * Convert windows event loop to use async/await * Fix double awaiting of async function * Convert asyncio/locks.py * Improve docstring * Convert tests to async/await * Convert more tests * Convert more tests * Convert more tests * Convert tests * Improve test
1 parent ede1573 commit 5f841b5

22 files changed

Lines changed: 646 additions & 770 deletions

‎Lib/asyncio/base_events.py‎

Lines changed: 59 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
from . import events
3434
from . import futures
3535
from . import tasks
36-
from .coroutines import coroutine
3736
from .log import logger
3837

3938

@@ -220,13 +219,12 @@ def _wakeup(self):
220219
if not waiter.done():
221220
waiter.set_result(waiter)
222221

223-
@coroutine
224-
def wait_closed(self):
222+
async def wait_closed(self):
225223
if self.sockets is None or self._waiters is None:
226224
return
227225
waiter = self._loop.create_future()
228226
self._waiters.append(waiter)
229-
yield from waiter
227+
await waiter
230228

231229

232230
class BaseEventLoop(events.AbstractEventLoop):
@@ -330,10 +328,9 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None,
330328
"""Create write pipe transport."""
331329
raise NotImplementedError
332330

333-
@coroutine
334-
def _make_subprocess_transport(self, protocol, args, shell,
335-
stdin, stdout, stderr, bufsize,
336-
extra=None, **kwargs):
331+
async def _make_subprocess_transport(self, protocol, args, shell,
332+
stdin, stdout, stderr, bufsize,
333+
extra=None, **kwargs):
337334
"""Create subprocess transport."""
338335
raise NotImplementedError
339336

@@ -371,8 +368,7 @@ def _asyncgen_firstiter_hook(self, agen):
371368

372369
self._asyncgens.add(agen)
373370

374-
@coroutine
375-
def shutdown_asyncgens(self):
371+
async def shutdown_asyncgens(self):
376372
"""Shutdown all active asynchronous generators."""
377373
self._asyncgens_shutdown_called = True
378374

@@ -384,12 +380,11 @@ def shutdown_asyncgens(self):
384380
closing_agens = list(self._asyncgens)
385381
self._asyncgens.clear()
386382

387-
shutdown_coro = tasks.gather(
383+
results = await tasks.gather(
388384
*[ag.aclose() for ag in closing_agens],
389385
return_exceptions=True,
390386
loop=self)
391387

392-
results = yield from shutdown_coro
393388
for result, agen in zip(results, closing_agens):
394389
if isinstance(result, Exception):
395390
self.call_exception_handler({
@@ -671,10 +666,10 @@ def getaddrinfo(self, host, port, *,
671666
def getnameinfo(self, sockaddr, flags=0):
672667
return self.run_in_executor(None, socket.getnameinfo, sockaddr, flags)
673668

674-
@coroutine
675-
def create_connection(self, protocol_factory, host=None, port=None, *,
676-
ssl=None, family=0, proto=0, flags=0, sock=None,
677-
local_addr=None, server_hostname=None):
669+
async def create_connection(self, protocol_factory, host=None, port=None,
670+
*, ssl=None, family=0,
671+
proto=0, flags=0, sock=None,
672+
local_addr=None, server_hostname=None):
678673
"""Connect to a TCP server.
679674
680675
Create a streaming transport connection to a given Internet host and
@@ -722,7 +717,7 @@ def create_connection(self, protocol_factory, host=None, port=None, *,
722717
else:
723718
f2 = None
724719

725-
yield from tasks.wait(fs, loop=self)
720+
await tasks.wait(fs, loop=self)
726721

727722
infos = f1.result()
728723
if not infos:
@@ -755,7 +750,7 @@ def create_connection(self, protocol_factory, host=None, port=None, *,
755750
continue
756751
if self._debug:
757752
logger.debug("connect %r to %r", sock, address)
758-
yield from self.sock_connect(sock, address)
753+
await self.sock_connect(sock, address)
759754
except OSError as exc:
760755
if sock is not None:
761756
sock.close()
@@ -793,7 +788,7 @@ def create_connection(self, protocol_factory, host=None, port=None, *,
793788
raise ValueError(
794789
'A Stream Socket was expected, got {!r}'.format(sock))
795790

796-
transport, protocol = yield from self._create_connection_transport(
791+
transport, protocol = await self._create_connection_transport(
797792
sock, protocol_factory, ssl, server_hostname)
798793
if self._debug:
799794
# Get the socket from the transport because SSL transport closes
@@ -803,9 +798,8 @@ def create_connection(self, protocol_factory, host=None, port=None, *,
803798
sock, host, port, transport, protocol)
804799
return transport, protocol
805800

806-
@coroutine
807-
def _create_connection_transport(self, sock, protocol_factory, ssl,
808-
server_hostname, server_side=False):
801+
async def _create_connection_transport(self, sock, protocol_factory, ssl,
802+
server_hostname, server_side=False):
809803

810804
sock.setblocking(False)
811805

@@ -820,19 +814,18 @@ def _create_connection_transport(self, sock, protocol_factory, ssl,
820814
transport = self._make_socket_transport(sock, protocol, waiter)
821815

822816
try:
823-
yield from waiter
817+
await waiter
824818
except:
825819
transport.close()
826820
raise
827821

828822
return transport, protocol
829823

830-
@coroutine
831-
def create_datagram_endpoint(self, protocol_factory,
832-
local_addr=None, remote_addr=None, *,
833-
family=0, proto=0, flags=0,
834-
reuse_address=None, reuse_port=None,
835-
allow_broadcast=None, sock=None):
824+
async def create_datagram_endpoint(self, protocol_factory,
825+
local_addr=None, remote_addr=None, *,
826+
family=0, proto=0, flags=0,
827+
reuse_address=None, reuse_port=None,
828+
allow_broadcast=None, sock=None):
836829
"""Create datagram connection."""
837830
if sock is not None:
838831
if not _is_dgram_socket(sock):
@@ -872,7 +865,7 @@ def create_datagram_endpoint(self, protocol_factory,
872865
assert isinstance(addr, tuple) and len(addr) == 2, (
873866
'2-tuple is expected')
874867

875-
infos = yield from _ensure_resolved(
868+
infos = await _ensure_resolved(
876869
addr, family=family, type=socket.SOCK_DGRAM,
877870
proto=proto, flags=flags, loop=self)
878871
if not infos:
@@ -918,7 +911,7 @@ def create_datagram_endpoint(self, protocol_factory,
918911
if local_addr:
919912
sock.bind(local_address)
920913
if remote_addr:
921-
yield from self.sock_connect(sock, remote_address)
914+
await self.sock_connect(sock, remote_address)
922915
r_addr = remote_address
923916
except OSError as exc:
924917
if sock is not None:
@@ -948,32 +941,30 @@ def create_datagram_endpoint(self, protocol_factory,
948941
remote_addr, transport, protocol)
949942

950943
try:
951-
yield from waiter
944+
await waiter
952945
except:
953946
transport.close()
954947
raise
955948

956949
return transport, protocol
957950

958-
@coroutine
959-
def _create_server_getaddrinfo(self, host, port, family, flags):
960-
infos = yield from _ensure_resolved((host, port), family=family,
961-
type=socket.SOCK_STREAM,
962-
flags=flags, loop=self)
951+
async def _create_server_getaddrinfo(self, host, port, family, flags):
952+
infos = await _ensure_resolved((host, port), family=family,
953+
type=socket.SOCK_STREAM,
954+
flags=flags, loop=self)
963955
if not infos:
964956
raise OSError('getaddrinfo({!r}) returned empty list'.format(host))
965957
return infos
966958

967-
@coroutine
968-
def create_server(self, protocol_factory, host=None, port=None,
969-
*,
970-
family=socket.AF_UNSPEC,
971-
flags=socket.AI_PASSIVE,
972-
sock=None,
973-
backlog=100,
974-
ssl=None,
975-
reuse_address=None,
976-
reuse_port=None):
959+
async def create_server(self, protocol_factory, host=None, port=None,
960+
*,
961+
family=socket.AF_UNSPEC,
962+
flags=socket.AI_PASSIVE,
963+
sock=None,
964+
backlog=100,
965+
ssl=None,
966+
reuse_address=None,
967+
reuse_port=None):
977968
"""Create a TCP server.
978969
979970
The host parameter can be a string, in that case the TCP server is bound
@@ -1011,7 +1002,7 @@ def create_server(self, protocol_factory, host=None, port=None,
10111002
fs = [self._create_server_getaddrinfo(host, port, family=family,
10121003
flags=flags)
10131004
for host in hosts]
1014-
infos = yield from tasks.gather(*fs, loop=self)
1005+
infos = await tasks.gather(*fs, loop=self)
10151006
infos = set(itertools.chain.from_iterable(infos))
10161007

10171008
completed = False
@@ -1068,8 +1059,8 @@ def create_server(self, protocol_factory, host=None, port=None,
10681059
logger.info("%r is serving", server)
10691060
return server
10701061

1071-
@coroutine
1072-
def connect_accepted_socket(self, protocol_factory, sock, *, ssl=None):
1062+
async def connect_accepted_socket(self, protocol_factory, sock,
1063+
*, ssl=None):
10731064
"""Handle an accepted connection.
10741065
10751066
This is used by servers that accept connections outside of
@@ -1082,7 +1073,7 @@ def connect_accepted_socket(self, protocol_factory, sock, *, ssl=None):
10821073
raise ValueError(
10831074
'A Stream Socket was expected, got {!r}'.format(sock))
10841075

1085-
transport, protocol = yield from self._create_connection_transport(
1076+
transport, protocol = await self._create_connection_transport(
10861077
sock, protocol_factory, ssl, '', server_side=True)
10871078
if self._debug:
10881079
# Get the socket from the transport because SSL transport closes
@@ -1091,14 +1082,13 @@ def connect_accepted_socket(self, protocol_factory, sock, *, ssl=None):
10911082
logger.debug("%r handled: (%r, %r)", sock, transport, protocol)
10921083
return transport, protocol
10931084

1094-
@coroutine
1095-
def connect_read_pipe(self, protocol_factory, pipe):
1085+
async def connect_read_pipe(self, protocol_factory, pipe):
10961086
protocol = protocol_factory()
10971087
waiter = self.create_future()
10981088
transport = self._make_read_pipe_transport(pipe, protocol, waiter)
10991089

11001090
try:
1101-
yield from waiter
1091+
await waiter
11021092
except:
11031093
transport.close()
11041094
raise
@@ -1108,14 +1098,13 @@ def connect_read_pipe(self, protocol_factory, pipe):
11081098
pipe.fileno(), transport, protocol)
11091099
return transport, protocol
11101100

1111-
@coroutine
1112-
def connect_write_pipe(self, protocol_factory, pipe):
1101+
async def connect_write_pipe(self, protocol_factory, pipe):
11131102
protocol = protocol_factory()
11141103
waiter = self.create_future()
11151104
transport = self._make_write_pipe_transport(pipe, protocol, waiter)
11161105

11171106
try:
1118-
yield from waiter
1107+
await waiter
11191108
except:
11201109
transport.close()
11211110
raise
@@ -1138,11 +1127,13 @@ def _log_subprocess(self, msg, stdin, stdout, stderr):
11381127
info.append('stderr=%s' % _format_pipe(stderr))
11391128
logger.debug(' '.join(info))
11401129

1141-
@coroutine
1142-
def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE,
1143-
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1144-
universal_newlines=False, shell=True, bufsize=0,
1145-
**kwargs):
1130+
async def subprocess_shell(self, protocol_factory, cmd, *,
1131+
stdin=subprocess.PIPE,
1132+
stdout=subprocess.PIPE,
1133+
stderr=subprocess.PIPE,
1134+
universal_newlines=False,
1135+
shell=True, bufsize=0,
1136+
**kwargs):
11461137
if not isinstance(cmd, (bytes, str)):
11471138
raise ValueError("cmd must be a string")
11481139
if universal_newlines:
@@ -1157,17 +1148,16 @@ def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE,
11571148
# (password) and may be too long
11581149
debug_log = 'run shell command %r' % cmd
11591150
self._log_subprocess(debug_log, stdin, stdout, stderr)
1160-
transport = yield from self._make_subprocess_transport(
1151+
transport = await self._make_subprocess_transport(
11611152
protocol, cmd, True, stdin, stdout, stderr, bufsize, **kwargs)
11621153
if self._debug:
11631154
logger.info('%s: %r', debug_log, transport)
11641155
return transport, protocol
11651156

1166-
@coroutine
1167-
def subprocess_exec(self, protocol_factory, program, *args,
1168-
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1169-
stderr=subprocess.PIPE, universal_newlines=False,
1170-
shell=False, bufsize=0, **kwargs):
1157+
async def subprocess_exec(self, protocol_factory, program, *args,
1158+
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1159+
stderr=subprocess.PIPE, universal_newlines=False,
1160+
shell=False, bufsize=0, **kwargs):
11711161
if universal_newlines:
11721162
raise ValueError("universal_newlines must be False")
11731163
if shell:
@@ -1186,7 +1176,7 @@ def subprocess_exec(self, protocol_factory, program, *args,
11861176
# (password) and may be too long
11871177
debug_log = 'execute program %r' % program
11881178
self._log_subprocess(debug_log, stdin, stdout, stderr)
1189-
transport = yield from self._make_subprocess_transport(
1179+
transport = await self._make_subprocess_transport(
11901180
protocol, popen_args, False, stdin, stdout, stderr,
11911181
bufsize, **kwargs)
11921182
if self._debug:

‎Lib/asyncio/base_subprocess.py‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
from . import protocols
66
from . import transports
7-
from .coroutines import coroutine
87
from .log import logger
98

109

@@ -154,26 +153,25 @@ def kill(self):
154153
self._check_proc()
155154
self._proc.kill()
156155

157-
@coroutine
158-
def _connect_pipes(self, waiter):
156+
async def _connect_pipes(self, waiter):
159157
try:
160158
proc = self._proc
161159
loop = self._loop
162160

163161
if proc.stdin is not None:
164-
_, pipe = yield from loop.connect_write_pipe(
162+
_, pipe = await loop.connect_write_pipe(
165163
lambda: WriteSubprocessPipeProto(self, 0),
166164
proc.stdin)
167165
self._pipes[0] = pipe
168166

169167
if proc.stdout is not None:
170-
_, pipe = yield from loop.connect_read_pipe(
168+
_, pipe = await loop.connect_read_pipe(
171169
lambda: ReadSubprocessPipeProto(self, 1),
172170
proc.stdout)
173171
self._pipes[1] = pipe
174172

175173
if proc.stderr is not None:
176-
_, pipe = yield from loop.connect_read_pipe(
174+
_, pipe = await loop.connect_read_pipe(
177175
lambda: ReadSubprocessPipeProto(self, 2),
178176
proc.stderr)
179177
self._pipes[2] = pipe
@@ -224,8 +222,7 @@ def _process_exited(self, returncode):
224222
waiter.set_result(returncode)
225223
self._exit_waiters = None
226224

227-
@coroutine
228-
def _wait(self):
225+
async def _wait(self):
229226
"""Wait until the process exit and return the process return code.
230227
231228
This method is a coroutine."""
@@ -234,7 +231,7 @@ def _wait(self):
234231

235232
waiter = self._loop.create_future()
236233
self._exit_waiters.append(waiter)
237-
return (yield from waiter)
234+
return await waiter
238235

239236
def _try_finish(self):
240237
assert not self._finished

0 commit comments

Comments
 (0)