-
-
Notifications
You must be signed in to change notification settings - Fork 106
Expand file tree
/
Copy pathtest_parallel_process.py
More file actions
229 lines (192 loc) · 6.8 KB
/
Copy pathtest_parallel_process.py
File metadata and controls
229 lines (192 loc) · 6.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
"""Process/interpreter pool support, chunking and executor passthrough."""
from __future__ import annotations
import concurrent.futures
import multiprocessing
import sys
import typing
import pytest
from progressbar._parallel import (
_common,
_sync,
)
# Process tests pay real spawn cost; keep batches small.
_INIT_VALUE: int = 0
def _square(value: int) -> int:
return value * value
def _boom_on_two(value: int) -> int:
if value == 2:
raise ValueError('boom')
return value
def _init_worker(value: int) -> None:
global _INIT_VALUE # noqa: PLW0603 - the per-worker setup contract
_INIT_VALUE = value
def _read_init(_: int) -> int:
return _INIT_VALUE
class TestProcessPool:
def test_ordered_results(self) -> None:
assert _sync.map(_square, range(12), pool='process', bar=False) == [
value * value for value in range(12)
]
def test_explicit_chunksize(self) -> None:
assert _sync.map(
_square, range(10), pool='process', chunksize=3, bar=False
) == [value * value for value in range(10)]
def test_auto_chunksize_engaged(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
calls: list[tuple[typing.Any, int]] = []
original: typing.Callable[[typing.Any, int], int] = (
_common.auto_chunksize
)
def _spy(total: typing.Any, workers: int) -> int:
calls.append((total, workers))
return original(total, workers)
monkeypatch.setattr(_sync._common, 'auto_chunksize', _spy)
_sync.map(_square, range(4), pool='process', workers=2, bar=False)
assert calls == [(4, 2)]
def test_auto_chunksize_not_used_for_threads(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
def _fail(total: typing.Any, workers: int) -> int:
raise AssertionError('auto_chunksize must not run for threads')
monkeypatch.setattr(_sync._common, 'auto_chunksize', _fail)
_sync.map(_square, range(4), pool='thread', bar=False)
def test_initializer_reaches_workers(self) -> None:
results = _sync.map(
_read_init,
range(4),
pool='process',
workers=2,
initializer=_init_worker,
initargs=(42,),
bar=False,
)
assert results == [42, 42, 42, 42]
def test_mp_context(self) -> None:
context = multiprocessing.get_context('spawn')
assert _sync.map(
_square,
range(4),
pool='process',
workers=2,
mp_context=context,
bar=False,
) == [0, 1, 4, 9]
@pytest.mark.skipif(
sys.version_info >= (3, 11),
reason='the ValueError applies before Python 3.11',
)
def test_max_tasks_per_child_rejected_on_310(
self,
) -> None: # pragma: no cover
with pytest.raises(ValueError, match=r'3\.11'):
_sync.map(
_square,
range(2),
pool='process',
max_tasks_per_child=2,
bar=False,
)
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason='max_tasks_per_child needs Python 3.11+',
)
def test_max_tasks_per_child(self) -> None:
assert _sync.map(
_square,
range(4),
pool='process',
workers=2,
max_tasks_per_child=2,
bar=False,
) == [0, 1, 4, 9]
def test_chunked_on_error_return_keeps_partial_chunk(self) -> None:
results = _sync.map(
_boom_on_two,
range(6),
pool='process',
chunksize=3,
on_error='return',
bar=False,
)
# Item 2 fails inside the first chunk; 0, 1 and the whole
# second chunk survive (per-item catch, no data loss).
assert results[0] == 0
assert results[1] == 1
assert isinstance(results[2], ValueError)
assert results[3:] == [3, 4, 5]
def test_chunked_on_error_raise(self) -> None:
with pytest.raises(ValueError, match='boom'):
_sync.map(
_boom_on_two,
range(6),
pool='process',
chunksize=3,
bar=False,
)
class TestInterpreterPool:
@pytest.mark.skipif(
sys.version_info < (3, 14),
reason='InterpreterPoolExecutor needs Python 3.14+',
)
def test_interpreter_pool_runs(self) -> None: # pragma: no cover
# Subinterpreter workers cannot import test modules, so the
# worker callable must come from an importable module -- the
# builtin `abs` qualifies everywhere.
assert _sync.map(
abs, [-1, -2, -3], pool='interpreter', workers=2, bar=False
) == [1, 2, 3]
@pytest.mark.skipif(
sys.version_info >= (3, 14),
reason='the ValueError applies before Python 3.14',
)
def test_interpreter_pool_rejected_before_314(self) -> None:
with pytest.raises(ValueError, match=r'3\.14'):
_sync.map(_square, range(4), pool='interpreter', bar=False)
class TestExecutorInstance:
def test_used_but_not_shut_down(self) -> None:
with concurrent.futures.ThreadPoolExecutor(2) as executor:
assert _sync.map(_square, range(5), pool=executor, bar=False) == [
0,
1,
4,
9,
16,
]
# Still usable afterwards: the engine must not shut it down.
assert executor.submit(_square, 3).result() == 9
def test_construction_kwargs_rejected(self) -> None:
with (
concurrent.futures.ThreadPoolExecutor(2) as executor,
pytest.raises(ValueError, match='initializer'),
):
_sync.map(
_square,
range(3),
pool=executor,
initializer=_init_worker,
initargs=(1,),
bar=False,
)
class TestPoolValidation:
def test_unknown_pool_string(self) -> None:
with pytest.raises(ValueError, match='bogus'):
_sync.map(_square, range(3), pool='bogus', bar=False)
def test_thread_pool_rejects_process_options(self) -> None:
with pytest.raises(ValueError, match='process pools'):
_sync.map(
_square,
range(3),
pool='thread',
mp_context=multiprocessing.get_context('spawn'),
bar=False,
)
def test_process_pool_rejects_thread_options(self) -> None:
with pytest.raises(ValueError, match='thread pools'):
_sync.map(
_square,
range(3),
pool='process',
thread_name_prefix='x',
bar=False,
)