Skip to content

Commit e30f005

Browse files
youknowdotcodex
youknowdot
and
codex
authored
Update Lib stream concurrency tests and pyrepl decorators to Python 3.15 (#9434)
* Update Lib stream concurrency tests and pyrepl decorators to Python 3.15 Sync canonical bz2, lzma, and pickle free-threading tests and retire obsolete class-decorator symbol-table workarounds while preserving the Android marker. Assisted-by: OpenAI Codex: model version unavailable * Fix LZMA bounded-output stream progress Keep processing header notifications while output space remains, drain buffered output with empty input, and retain capped-output state for needs_input. Assisted-by: OpenAI Codex: model version unavailable * Retire passing bounded LZMA decoding markers Regenerate the canonical LZMA tests with update_lib, retaining all other existing markers. Both complete affected methods pass with the native buffered-output fix. Assisted-by: OpenAI Codex: model version unavailable --------- Co-authored-by: Codex <codex@openai.com>
1 parent bbbdcf0 commit e30f005

6 files changed

Lines changed: 261 additions & 94 deletions

File tree

‎Lib/test/test_free_threading/test_bz2.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import unittest
2+
3+
from test.support import import_helper, threading_helper
4+
from test.support.threading_helper import run_concurrently
5+
6+
bz2 = import_helper.import_module("bz2")
7+
from bz2 import BZ2Compressor, BZ2Decompressor
8+
9+
from test.test_bz2 import ext_decompress, BaseTest
10+
11+
12+
NTHREADS = 10
13+
TEXT = BaseTest.TEXT
14+
15+
16+
@threading_helper.requires_working_threading()
17+
class TestBZ2(unittest.TestCase):
18+
def test_compressor(self):
19+
bz2c = BZ2Compressor()
20+
21+
def worker():
22+
# it should return empty bytes as it buffers data internally
23+
data = bz2c.compress(TEXT)
24+
self.assertEqual(data, b"")
25+
26+
run_concurrently(worker_func=worker, nthreads=NTHREADS)
27+
data = bz2c.flush()
28+
# The decompressed data should be TEXT repeated NTHREADS times
29+
decompressed = ext_decompress(data)
30+
self.assertEqual(decompressed, TEXT * NTHREADS)
31+
32+
def test_decompressor(self):
33+
chunk_size = 128
34+
chunks = [bytes([ord("a") + i]) * chunk_size for i in range(NTHREADS)]
35+
input_data = b"".join(chunks)
36+
compressed = bz2.compress(input_data)
37+
38+
bz2d = BZ2Decompressor()
39+
output = []
40+
41+
def worker():
42+
data = bz2d.decompress(compressed, chunk_size)
43+
self.assertEqual(len(data), chunk_size)
44+
output.append(data)
45+
# Read attributes concurrently with other threads decompressing
46+
self.assertIsInstance(bz2d.eof, bool)
47+
self.assertIsInstance(bz2d.needs_input, bool)
48+
self.assertIsInstance(bz2d.unused_data, bytes)
49+
50+
run_concurrently(worker_func=worker, nthreads=NTHREADS)
51+
self.assertEqual(len(output), NTHREADS)
52+
# Verify the expected chunks (order doesn't matter due to append race)
53+
self.assertEqual(set(output), set(chunks))
54+
self.assertTrue(bz2d.eof)
55+
self.assertFalse(bz2d.needs_input)
56+
# Each thread added full compressed data to the buffer, but only 1 copy
57+
# is consumed to produce the output. The rest remains as unused_data.
58+
self.assertEqual(
59+
len(bz2d.unused_data), len(compressed) * (NTHREADS - 1)
60+
)
61+
62+
63+
if __name__ == "__main__":
64+
unittest.main()

‎Lib/test/test_free_threading/test_lzma.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import unittest
2+
3+
from test.support import import_helper, threading_helper
4+
from test.support.threading_helper import run_concurrently
5+
6+
lzma = import_helper.import_module("lzma")
7+
from lzma import LZMACompressor, LZMADecompressor
8+
9+
from test.test_lzma import INPUT
10+
11+
12+
NTHREADS = 10
13+
14+
15+
@threading_helper.requires_working_threading()
16+
class TestLZMA(unittest.TestCase):
17+
def test_compressor(self):
18+
lzc = LZMACompressor()
19+
20+
# First compress() outputs LZMA header
21+
header = lzc.compress(INPUT)
22+
self.assertGreater(len(header), 0)
23+
24+
def worker():
25+
# it should return empty bytes as it buffers data internally
26+
data = lzc.compress(INPUT)
27+
self.assertEqual(data, b"")
28+
29+
run_concurrently(worker_func=worker, nthreads=NTHREADS - 1)
30+
full_compressed = header + lzc.flush()
31+
decompressed = lzma.decompress(full_compressed)
32+
# The decompressed data should be INPUT repeated NTHREADS times
33+
self.assertEqual(decompressed, INPUT * NTHREADS)
34+
35+
def test_decompressor(self):
36+
chunk_size = 128
37+
chunks = [bytes([ord("a") + i]) * chunk_size for i in range(NTHREADS)]
38+
input_data = b"".join(chunks)
39+
compressed = lzma.compress(input_data)
40+
41+
lzd = LZMADecompressor()
42+
output = []
43+
44+
def worker():
45+
data = lzd.decompress(compressed, chunk_size)
46+
self.assertEqual(len(data), chunk_size)
47+
output.append(data)
48+
# Read attributes concurrently with other threads decompressing
49+
self.assertEqual(lzd.check, lzma.CHECK_CRC64)
50+
self.assertIsInstance(lzd.eof, bool)
51+
self.assertIsInstance(lzd.needs_input, bool)
52+
self.assertIsInstance(lzd.unused_data, bytes)
53+
54+
run_concurrently(worker_func=worker, nthreads=NTHREADS)
55+
self.assertEqual(len(output), NTHREADS)
56+
# Verify the expected chunks (order doesn't matter due to append race)
57+
self.assertSetEqual(set(output), set(chunks))
58+
self.assertEqual(lzd.check, lzma.CHECK_CRC64)
59+
self.assertTrue(lzd.eof)
60+
self.assertFalse(lzd.needs_input)
61+
# Each thread added full compressed data to the buffer, but only 1 copy
62+
# is consumed to produce the output. The rest remains as unused_data.
63+
self.assertEqual(
64+
len(lzd.unused_data), len(compressed) * (NTHREADS - 1)
65+
)
66+
67+
68+
if __name__ == "__main__":
69+
unittest.main()

‎Lib/test/test_free_threading/test_pickle.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import pickle
2+
import threading
3+
import unittest
4+
5+
from test.support import threading_helper
6+
7+
8+
@threading_helper.requires_working_threading()
9+
class TestPickleFreeThreading(unittest.TestCase):
10+
11+
def test_pickle_dumps_with_concurrent_dict_mutation(self):
12+
# gh-146452: Pickling a dict while another thread mutates it
13+
# used to segfault. batch_dict_exact() iterated dict items via
14+
# PyDict_Next() which returns borrowed references, and a
15+
# concurrent pop/replace could free the value before Py_INCREF
16+
# got to it.
17+
shared = {str(i): list(range(20)) for i in range(50)}
18+
19+
def dumper():
20+
for _ in range(1000):
21+
try:
22+
pickle.dumps(shared)
23+
except RuntimeError:
24+
# "dictionary changed size during iteration" is expected
25+
pass
26+
27+
def mutator():
28+
for j in range(1000):
29+
key = str(j % 50)
30+
shared[key] = list(range(j % 20))
31+
if j % 10 == 0:
32+
shared.pop(key, None)
33+
shared[key] = [j]
34+
35+
threads = []
36+
for _ in range(10):
37+
threads.append(threading.Thread(target=dumper))
38+
threads.append(threading.Thread(target=mutator))
39+
40+
with threading_helper.start_threads(threads):
41+
pass
42+
43+
def test_pickle_dumps_with_concurrent_list_mutations(self):
44+
# gh-149816: Pickling a list while another thread mutates it
45+
# used to be a UAF in free-threaded mode. batch_list_exact()
46+
# used PyList_GET_ITEM (borrowed) followed by Py_INCREF, and a
47+
# concurrent replace/pop could free the item between those two
48+
# operations.
49+
shared = [list(range(20)) for _ in range(50)]
50+
51+
def dumper():
52+
for _ in range(1000):
53+
try:
54+
pickle.dumps(shared)
55+
except (RuntimeError, IndexError):
56+
pass
57+
58+
def mutator():
59+
for i in range(1000):
60+
idx = i % 50
61+
shared[idx] = list(range(i % 20))
62+
if i % 10 == 0:
63+
try:
64+
shared.pop()
65+
except IndexError:
66+
pass
67+
shared.append([i])
68+
69+
threads = []
70+
for _ in range(10):
71+
threads.append(threading.Thread(target=dumper))
72+
threads.append(threading.Thread(target=mutator))
73+
74+
with threading_helper.start_threads(threads):
75+
pass
76+
77+
if __name__ == "__main__":
78+
unittest.main()

‎Lib/test/test_lzma.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,6 @@ def test_decompressor_chunks_empty(self):
157157
self.assertTrue(lzd.eof)
158158
self.assertEqual(lzd.unused_data, b"")
159159

160-
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check'
161160
def test_decompressor_chunks_maxsize(self):
162161
lzd = LZMADecompressor()
163162
max_length = 100
@@ -1349,7 +1348,6 @@ def test_tell_bad_args(self):
13491348
f.close()
13501349
self.assertRaises(ValueError, f.tell)
13511350

1352-
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: True is not false
13531351
def test_issue21872(self):
13541352
# sometimes decompress data incompletely
13551353

‎Lib/test/test_pyrepl/test_unix_console.py‎

Lines changed: 10 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,14 @@ def unix_console(events, **kwargs):
6161
)
6262

6363

64-
def __rustpython_patch_terminfo_tparm(s, *args):
65-
return s + b":" + b",".join(str(i).encode() for i in args)
66-
67-
68-
def __rustpython_patch_termios_tcgetattr(_):
69-
return [
64+
@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows")
65+
@patch(
66+
"_pyrepl.terminfo.tparm",
67+
lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args),
68+
)
69+
@patch(
70+
"termios.tcgetattr",
71+
lambda _: [
7072
27394,
7173
3,
7274
19200,
@@ -95,60 +97,9 @@ def __rustpython_patch_termios_tcgetattr(_):
9597
b"\x14",
9698
b"\x00",
9799
],
98-
]
99-
100-
101-
def __rustpython_patch_termios_tcsetattr(a, b, c):
102-
return None
103-
104-
@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows")
105-
@patch(
106-
"_pyrepl.terminfo.tparm",
107-
# TODO: RUSTPYTHON; SyntaxError: the symbol 's' must be present in the symbol table
108-
# lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args),
109-
__rustpython_patch_terminfo_tparm,
110-
)
111-
# TODO: RUSTPYTHON; SyntaxError: no symbol table available in lambda (type: Lambda)
112-
# @patch(
113-
# "termios.tcgetattr",
114-
# lambda _: [
115-
# 27394,
116-
# 3,
117-
# 19200,
118-
# 536872399,
119-
# 38400,
120-
# 38400,
121-
# [
122-
# b"\x04",
123-
# b"\xff",
124-
# b"\xff",
125-
# b"\x7f",
126-
# b"\x17",
127-
# b"\x15",
128-
# b"\x12",
129-
# b"\x00",
130-
# b"\x03",
131-
# b"\x1c",
132-
# b"\x1a",
133-
# b"\x19",
134-
# b"\x11",
135-
# b"\x13",
136-
# b"\x16",
137-
# b"\x0f",
138-
# b"\x01",
139-
# b"\x00",
140-
# b"\x14",
141-
# b"\x00",
142-
# ],
143-
# ],
144-
# )
145-
@patch("termios.tcgetattr", __rustpython_patch_termios_tcgetattr)
146-
@patch(
147-
"termios.tcsetattr",
148-
# TODO: RUSTPYTHON; SyntaxError: no symbol table available in lambda (type: Lambda)
149-
# lambda a, b, c: None
150-
__rustpython_patch_termios_tcsetattr,
100+
],
151101
)
102+
@patch("termios.tcsetattr", lambda a, b, c: None)
152103
@patch("os.write")
153104
@force_not_colorized_test_class
154105
class TestConsole(TestCase):

0 commit comments

Comments
 (0)