Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
4cb8e76
Basic `_queue` module
ShaharNaveh May 29, 2026
ce6a961
`RingBuf_Get` like behavior
ShaharNaveh May 29, 2026
00118ed
Cleanup
ShaharNaveh May 29, 2026
372d8ca
clippy
ShaharNaveh May 29, 2026
ac3aa7a
Fix failing test
ShaharNaveh May 29, 2026
8a9974d
impl some more methods
ShaharNaveh May 30, 2026
00cca8e
Don't overcomplicate things
ShaharNaveh May 30, 2026
f8357c3
Fix logic error
ShaharNaveh May 30, 2026
5501877
Use a Mutext instead
ShaharNaveh May 30, 2026
c45f058
Allow `_queue.Empty` to be `weakref.ref`
ShaharNaveh May 30, 2026
fc092d2
condtional when threading is missing
ShaharNaveh May 30, 2026
e71e9ad
Update `multiprocessing/connection.py` to 3.14.5
ShaharNaveh May 30, 2026
0037a56
Merge remote-tracking branch 'upstream/main' into queue-module
ShaharNaveh May 30, 2026
8aebcd0
Impl `self.borrow_buf`
ShaharNaveh May 31, 2026
eac4c32
Update `multiprocessing/popen_fork.py` to 3.14.5
ShaharNaveh May 31, 2026
995c582
Update `multiprocessing/resource_tracker.py` to 3.14.5
ShaharNaveh May 31, 2026
93465cc
Code cleanup
ShaharNaveh May 31, 2026
f33635f
Refactor using ReentrantMutex
ShaharNaveh May 31, 2026
8257cfc
Fix non-thrradded build
ShaharNaveh May 31, 2026
5f6088f
Merge remote-tracking branch 'upstream/main' into queue-module
ShaharNaveh May 31, 2026
4257f68
Fix
ShaharNaveh May 31, 2026
c04b7a9
Clippy
ShaharNaveh May 31, 2026
aedf8f3
Clippy
ShaharNaveh May 31, 2026
c37a9c1
Lock before push
ShaharNaveh May 31, 2026
cca9bff
Fix compile error
ShaharNaveh May 31, 2026
12d0d5c
Merge remote-tracking branch 'upstream/main' into queue-module
ShaharNaveh May 31, 2026
fd30276
Update `multiprocessing` to 3.14.5
ShaharNaveh Jun 1, 2026
9174845
Use basic Semaphore
ShaharNaveh Jun 1, 2026
ce3e486
clippy
ShaharNaveh Jun 1, 2026
583f440
Drop lock before notify
ShaharNaveh Jun 1, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 79 additions & 37 deletions Lib/multiprocessing/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,12 @@

import errno
import io
import itertools
import os
import sys
import socket
import struct
import time
import tempfile
import itertools


from . import util
Expand All @@ -39,11 +38,14 @@
#
#

BUFSIZE = 8192
# 64 KiB is the default PIPE buffer size of most POSIX platforms.
BUFSIZE = 64 * 1024

# A very generous timeout when it comes to local connections...
CONNECTION_TIMEOUT = 20.

_mmap_counter = itertools.count()
_MAX_PIPE_ATTEMPTS = 100

default_family = 'AF_INET'
families = ['AF_INET']
Expand Down Expand Up @@ -74,10 +76,14 @@ def arbitrary_address(family):
if family == 'AF_INET':
return ('localhost', 0)
elif family == 'AF_UNIX':
return tempfile.mktemp(prefix='sock-', dir=util.get_temp_dir())
# NOTE: util.get_temp_dir() is a 0o700 per-process directory. A
# mktemp-style ToC vs ToU concern is not important; bind() surfaces
# the extremely unlikely collision as EADDRINUSE.
return os.path.join(util.get_temp_dir(),
f'sock-{os.urandom(6).hex()}')
elif family == 'AF_PIPE':
return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
(os.getpid(), next(_mmap_counter)), dir="")
return (r'\\.\pipe\pyc-%d-%d-%s' %
(os.getpid(), next(_mmap_counter), os.urandom(8).hex()))
else:
raise ValueError('unrecognized family')

Expand Down Expand Up @@ -179,6 +185,10 @@ def close(self):
finally:
self._handle = None

def _detach(self):
"""Stop managing the underlying file descriptor or handle."""
self._handle = None

def send_bytes(self, buf, offset=0, size=None):
"""Send the bytes data from a bytes-like object"""
self._check_closed()
Expand Down Expand Up @@ -316,22 +326,32 @@ def _recv_bytes(self, maxsize=None):
try:
ov, err = _winapi.ReadFile(self._handle, bsize,
overlapped=True)

sentinel = object()
return_value = sentinel
try:
if err == _winapi.ERROR_IO_PENDING:
waitres = _winapi.WaitForMultipleObjects(
[ov.event], False, INFINITE)
assert waitres == WAIT_OBJECT_0
try:
if err == _winapi.ERROR_IO_PENDING:
waitres = _winapi.WaitForMultipleObjects(
[ov.event], False, INFINITE)
assert waitres == WAIT_OBJECT_0
except:
ov.cancel()
raise
finally:
nread, err = ov.GetOverlappedResult(True)
if err == 0:
f = io.BytesIO()
f.write(ov.getbuffer())
return_value = f
elif err == _winapi.ERROR_MORE_DATA:
return_value = self._get_more_data(ov, maxsize)
except:
ov.cancel()
raise
finally:
nread, err = ov.GetOverlappedResult(True)
if err == 0:
f = io.BytesIO()
f.write(ov.getbuffer())
return f
elif err == _winapi.ERROR_MORE_DATA:
return self._get_more_data(ov, maxsize)
if return_value is sentinel:
raise

if return_value is not sentinel:
return return_value
except OSError as e:
if e.winerror == _winapi.ERROR_BROKEN_PIPE:
raise EOFError
Expand Down Expand Up @@ -392,7 +412,8 @@ def _recv(self, size, read=_read):
handle = self._handle
remaining = size
while remaining > 0:
chunk = read(handle, remaining)
to_read = min(BUFSIZE, remaining)
chunk = read(handle, to_read)
n = len(chunk)
if n == 0:
if remaining == size:
Expand Down Expand Up @@ -455,17 +476,29 @@ class Listener(object):
def __init__(self, address=None, family=None, backlog=1, authkey=None):
family = family or (address and address_type(address)) \
or default_family
address = address or arbitrary_address(family)

_validate_family(family)
if authkey is not None and not isinstance(authkey, bytes):
raise TypeError('authkey should be a byte string')

if family == 'AF_PIPE':
self._listener = PipeListener(address, backlog)
if address:
self._listener = PipeListener(address, backlog)
else:
for attempts in itertools.count():
address = arbitrary_address(family)
try:
self._listener = PipeListener(address, backlog)
break
except OSError as e:
if attempts >= _MAX_PIPE_ATTEMPTS:
raise
if e.winerror not in (_winapi.ERROR_PIPE_BUSY,
_winapi.ERROR_ACCESS_DENIED):
raise
else:
address = address or arbitrary_address(family)
self._listener = SocketListener(address, family, backlog)

if authkey is not None and not isinstance(authkey, bytes):
raise TypeError('authkey should be a byte string')

self._authkey = authkey

def accept(self):
Expand Down Expand Up @@ -553,7 +586,6 @@ def Pipe(duplex=True):
'''
Returns pair of connection objects at either end of a pipe
'''
address = arbitrary_address('AF_PIPE')
if duplex:
openmode = _winapi.PIPE_ACCESS_DUPLEX
access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
Expand All @@ -563,15 +595,25 @@ def Pipe(duplex=True):
access = _winapi.GENERIC_WRITE
obsize, ibsize = 0, BUFSIZE

h1 = _winapi.CreateNamedPipe(
address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
_winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
_winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
_winapi.PIPE_WAIT,
1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER,
# default security descriptor: the handle cannot be inherited
_winapi.NULL
)
for attempts in itertools.count():
address = arbitrary_address('AF_PIPE')
try:
h1 = _winapi.CreateNamedPipe(
address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
_winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
_winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
_winapi.PIPE_WAIT,
1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER,
# default security descriptor: the handle cannot be inherited
_winapi.NULL
)
break
except OSError as e:
if attempts >= _MAX_PIPE_ATTEMPTS:
raise
if e.winerror not in (_winapi.ERROR_PIPE_BUSY,
_winapi.ERROR_ACCESS_DENIED):
raise
h2 = _winapi.CreateFile(
address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
_winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
Expand Down
36 changes: 21 additions & 15 deletions Lib/multiprocessing/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,13 @@ def freeze_support(self):
'''Check whether this is a fake forked process in a frozen executable.
If so then run code specified by commandline and exit.
'''
if self.get_start_method() == 'spawn' and getattr(sys, 'frozen', False):
# gh-140814: allow_none=True avoids locking in the default start
# method, which would cause a later set_start_method() to fail.
# None is safe to pass through: spawn.freeze_support()
# independently detects whether this process is a spawned
# child, so the start method check here is only an optimization.
if (getattr(sys, 'frozen', False)
and self.get_start_method(allow_none=True) in ('spawn', None)):
from .spawn import freeze_support
freeze_support()

Expand All @@ -167,7 +173,7 @@ def allow_connection_pickling(self):
'''
# This is undocumented. In previous versions of multiprocessing
# its only effect was to make socket objects inheritable on Windows.
from . import connection
from . import connection # noqa: F401

def set_executable(self, executable):
'''Sets the path to a python.exe or pythonw.exe binary used to run
Expand Down Expand Up @@ -259,13 +265,12 @@ def get_start_method(self, allow_none=False):

def get_all_start_methods(self):
"""Returns a list of the supported start methods, default first."""
if sys.platform == 'win32':
return ['spawn']
else:
methods = ['spawn', 'fork'] if sys.platform == 'darwin' else ['fork', 'spawn']
if reduction.HAVE_SEND_HANDLE:
methods.append('forkserver')
return methods
default = self._default_context.get_start_method()
start_method_names = [default]
start_method_names.extend(
name for name in _concrete_contexts if name != default
)
return start_method_names


#
Expand Down Expand Up @@ -320,14 +325,15 @@ def _check_available(self):
'spawn': SpawnContext(),
'forkserver': ForkServerContext(),
}
if sys.platform == 'darwin':
# bpo-33725: running arbitrary code after fork() is no longer reliable
# on macOS since macOS 10.14 (Mojave). Use spawn by default instead.
_default_context = DefaultContext(_concrete_contexts['spawn'])
# bpo-33725: running arbitrary code after fork() is no longer reliable
# on macOS since macOS 10.14 (Mojave). Use spawn by default instead.
# gh-84559: We changed everyones default to a thread safeish one in 3.14.
if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin':
_default_context = DefaultContext(_concrete_contexts['forkserver'])
else:
_default_context = DefaultContext(_concrete_contexts['fork'])
_default_context = DefaultContext(_concrete_contexts['spawn'])

else:
else: # Windows

class SpawnProcess(process.BaseProcess):
_start_method = 'spawn'
Expand Down
2 changes: 1 addition & 1 deletion Lib/multiprocessing/dummy/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@

class DummyProcess(threading.Thread):

def __init__(self, group=None, target=None, name=None, args=(), kwargs={}):
def __init__(self, group=None, target=None, name=None, args=(), kwargs=None):
threading.Thread.__init__(self, group, target, name, args, kwargs)
self._pid = None
self._children = weakref.WeakKeyDictionary()
Expand Down
Loading
Loading