Skip to content

Commit e833153

Browse files
youknowdotcodex
youknowdot
and
codex
authored
Fix UTF-8 startup option precedence and refresh canonical tests (#9422)
* Fix UTF-8 startup option precedence and refresh canonical tests Resolve the first explicit UTF-8 option before environment validation and align startup diagnostics with CPython 3.15. Refresh canonical encoding tests through update_lib and narrow the existing environment test failure to Windows. Support only monotonic expected-failure narrowing in updater metadata with focused regression tests. Assisted-by: OpenAI Codex: model version unavailable * Normalize configured standard-stream codec names Resolve explicit stdio encodings through CodecInfo.name before creating standard streams, matching CPython initialization. Preserve ordinary TextIOWrapper spelling and existing standard-stream error policies; leave canonical tests unchanged. Assisted-by: OpenAI Codex: model version unavailable * Retire passing standard-stream UTF-8 expected failure The configured-codec normalization fix makes canonical test_stdio pass in both Linux CI attempts. Remove only its stale expectedFailure through update_lib, preserving canonical assertions and all other platform guards. Assisted-by: OpenAI Codex: model version unavailable --------- Co-authored-by: Codex <codex@openai.com>
1 parent 12b8045 commit e833153

6 files changed

Lines changed: 192 additions & 52 deletions

File tree

‎Lib/test/test_c_locale_coercion.py‎

Lines changed: 75 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -12,31 +12,33 @@
1212
from test.support.script_helper import run_python_until_end
1313

1414

15-
# Set the list of ways we expect to be able to ask for the "C" locale
16-
EXPECTED_C_LOCALE_EQUIVALENTS = ["C", "invalid.ascii"]
15+
# Set the list of ways we expect to be able to ask for the "C" locale.
16+
# 'invalid.ascii' is an invalid LOCALE name and so should get turned in to the
17+
# default locale, which is traditionally C.
18+
EXPECTED_C_LOCALE_EQUIVALENTS = ["C", "POSIX", "invalid.ascii"]
1719

1820
# Set our expectation for the default encoding used in the C locale
1921
# for the filesystem encoding and the standard streams
2022
EXPECTED_C_LOCALE_STREAM_ENCODING = "ascii"
2123
EXPECTED_C_LOCALE_FS_ENCODING = "ascii"
2224

2325
# Set our expectation for the default locale used when none is specified
26+
DEFAULT_LOCALE_IS_C = True
2427
EXPECT_COERCION_IN_DEFAULT_LOCALE = True
2528

2629
TARGET_LOCALES = ["C.UTF-8", "C.utf8", "UTF-8"]
2730

2831
# Apply some platform dependent overrides
29-
if sys.platform.startswith("linux"):
30-
if support.is_android:
31-
# Android defaults to using UTF-8 for all system interfaces
32-
EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8"
33-
EXPECTED_C_LOCALE_FS_ENCODING = "utf-8"
34-
else:
35-
# Linux distros typically alias the POSIX locale directly to the C
36-
# locale.
37-
# TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be
38-
# able to check this case unconditionally
39-
EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX")
32+
if sys.platform == "android":
33+
# Android defaults to using UTF-8 for all system interfaces
34+
EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8"
35+
EXPECTED_C_LOCALE_FS_ENCODING = "utf-8"
36+
elif support.linked_to_musl():
37+
# MUSL defaults to utf-8 unless the C locale is set explicitly.
38+
EXPECTED_C_LOCALE_EQUIVALENTS = ["C"]
39+
DEFAULT_LOCALE_IS_C = False
40+
DEFAULT_ENCODING = 'utf-8'
41+
EXPECT_COERCION_IN_DEFAULT_LOCALE = False
4042
elif sys.platform.startswith("aix"):
4143
# AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII
4244
EXPECTED_C_LOCALE_STREAM_ENCODING = "iso8859-1"
@@ -112,12 +114,16 @@ class EncodingDetails(_EncodingDetails):
112114
])
113115

114116
@classmethod
115-
def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, env_vars):
117+
def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, stream_errors, env_vars):
116118
"""Returns expected child process details for a given encoding"""
117119
_stream = stream_encoding + ":{}"
118-
# stdin and stdout should use surrogateescape either because the
119-
# coercion triggered, or because the C locale was detected
120-
stream_info = 2*[_stream.format("surrogateescape")]
120+
if stream_errors is None:
121+
# stdin and stdout should use surrogateescape either because the
122+
# coercion triggered, or because the C locale was detected
123+
stream_errors = "surrogateescape"
124+
125+
stream_info = [_stream.format(stream_errors)] * 2
126+
121127
# stderr should always use backslashreplace
122128
stream_info.append(_stream.format("backslashreplace"))
123129
expected_lang = env_vars.get("LANG", "not set")
@@ -210,6 +216,7 @@ def _check_child_encoding_details(self,
210216
env_vars,
211217
expected_fs_encoding,
212218
expected_stream_encoding,
219+
expected_stream_errors,
213220
expected_warnings,
214221
coercion_expected):
215222
"""Check the C locale handling for the given process environment
@@ -225,6 +232,7 @@ def _check_child_encoding_details(self,
225232
coercion_expected,
226233
expected_fs_encoding,
227234
expected_stream_encoding,
235+
expected_stream_errors,
228236
env_vars
229237
)
230238
self.assertEqual(encoding_details, expected_details)
@@ -257,6 +265,7 @@ def test_external_target_locale_configuration(self):
257265
"LC_CTYPE": "",
258266
"LC_ALL": "",
259267
"PYTHONCOERCECLOCALE": "",
268+
"PYTHONIOENCODING": "",
260269
}
261270
for env_var in ("LANG", "LC_CTYPE"):
262271
for locale_to_set in AVAILABLE_TARGETS:
@@ -273,10 +282,43 @@ def test_external_target_locale_configuration(self):
273282
self._check_child_encoding_details(var_dict,
274283
expected_fs_encoding,
275284
expected_stream_encoding,
285+
expected_stream_errors=None,
276286
expected_warnings=None,
277287
coercion_expected=False)
278288

289+
def test_with_ioencoding(self):
290+
# Explicitly setting a target locale should give the same behaviour as
291+
# is seen when implicitly coercing to that target locale
292+
self.maxDiff = None
293+
294+
expected_fs_encoding = "utf-8"
295+
expected_stream_encoding = "utf-8"
296+
297+
base_var_dict = {
298+
"LANG": "",
299+
"LC_CTYPE": "",
300+
"LC_ALL": "",
301+
"PYTHONCOERCECLOCALE": "",
302+
"PYTHONIOENCODING": "UTF-8",
303+
}
304+
for env_var in ("LANG", "LC_CTYPE"):
305+
for locale_to_set in AVAILABLE_TARGETS:
306+
# XXX (ncoghlan): LANG=UTF-8 doesn't appear to work as
307+
# expected, so skip that combination for now
308+
# See https://bugs.python.org/issue30672 for discussion
309+
if env_var == "LANG" and locale_to_set == "UTF-8":
310+
continue
279311

312+
with self.subTest(env_var=env_var,
313+
configured_locale=locale_to_set):
314+
var_dict = base_var_dict.copy()
315+
var_dict[env_var] = locale_to_set
316+
self._check_child_encoding_details(var_dict,
317+
expected_fs_encoding,
318+
expected_stream_encoding,
319+
expected_stream_errors="strict",
320+
expected_warnings=None,
321+
coercion_expected=False)
280322

281323
@support.cpython_only
282324
@unittest.skipUnless(sysconfig.get_config_var("PY_COERCE_C_LOCALE"),
@@ -316,15 +358,21 @@ def _check_c_locale_coercion(self,
316358
"LC_CTYPE": "",
317359
"LC_ALL": "",
318360
"PYTHONCOERCECLOCALE": "",
361+
"PYTHONIOENCODING": "",
319362
}
320363
base_var_dict.update(extra_vars)
321364
if coerce_c_locale is not None:
322365
base_var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale
323366

324367
# Check behaviour for the default locale
368+
_fs_encoding = fs_encoding
369+
_stream_encoding = stream_encoding
370+
if not DEFAULT_LOCALE_IS_C and 'LC_ALL' not in extra_vars:
371+
_fs_encoding = _stream_encoding = DEFAULT_ENCODING
325372
with self.subTest(default_locale=True,
326373
PYTHONCOERCECLOCALE=coerce_c_locale):
327-
if EXPECT_COERCION_IN_DEFAULT_LOCALE:
374+
if (EXPECT_COERCION_IN_DEFAULT_LOCALE
375+
or (not DEFAULT_LOCALE_IS_C and 'LC_ALL' in extra_vars)):
328376
_expected_warnings = expected_warnings
329377
_coercion_expected = coercion_expected
330378
else:
@@ -338,8 +386,9 @@ def _check_c_locale_coercion(self,
338386
_expected_warnings == [CLI_COERCION_WARNING]):
339387
_expected_warnings = None
340388
self._check_child_encoding_details(base_var_dict,
341-
fs_encoding,
342-
stream_encoding,
389+
_fs_encoding,
390+
_stream_encoding,
391+
None,
343392
_expected_warnings,
344393
_coercion_expected)
345394

@@ -348,13 +397,15 @@ def _check_c_locale_coercion(self,
348397
for env_var in ("LANG", "LC_CTYPE"):
349398
with self.subTest(env_var=env_var,
350399
nominal_locale=locale_to_set,
351-
PYTHONCOERCECLOCALE=coerce_c_locale):
400+
PYTHONCOERCECLOCALE=coerce_c_locale,
401+
PYTHONIOENCODING=""):
352402
var_dict = base_var_dict.copy()
353403
var_dict[env_var] = locale_to_set
354404
# Check behaviour on successful coercion
355405
self._check_child_encoding_details(var_dict,
356406
fs_encoding,
357407
stream_encoding,
408+
None,
358409
expected_warnings,
359410
coercion_expected)
360411

@@ -411,8 +462,9 @@ def test_PYTHONCOERCECLOCALE_set_to_one(self):
411462
loc = locale.setlocale(locale.LC_CTYPE, "")
412463
except locale.Error as e:
413464
self.skipTest(str(e))
414-
if loc == "C":
415-
self.skipTest("test requires LC_CTYPE locale different than C")
465+
if loc in ("C", "POSIX"):
466+
self.skipTest("test requires LC_CTYPE locale different "
467+
"than C and POSIX")
416468
if loc in TARGET_LOCALES :
417469
self.skipTest("coerced LC_CTYPE locale: %s" % loc)
418470

‎Lib/test/test_utf8_mode.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ def test_xoption(self):
6666
PYTHONLEGACYWINDOWSFSENCODING='1')
6767
self.assertEqual(out, '0')
6868

69-
@unittest.expectedFailure # TODO: RUSTPYTHON
69+
@unittest.expectedFailureIf(MS_WINDOWS, "TODO: RUSTPYTHON")
7070
def test_env_var(self):
7171
code = 'import sys; print(sys.flags.utf8_mode)'
7272

@@ -91,8 +91,8 @@ def test_env_var(self):
9191
# the UTF-8 mode
9292
if not self.posix_locale():
9393
# PYTHONUTF8 should be ignored if -E is used
94-
out = self.get_output('-E', '-c', code, PYTHONUTF8='1')
95-
self.assertEqual(out, '0')
94+
out = self.get_output('-E', '-c', code, PYTHONUTF8='0')
95+
self.assertEqual(out, '1')
9696

9797
# invalid mode
9898
out = self.get_output('-c', code, PYTHONUTF8='xxx', failure=True)
@@ -119,11 +119,10 @@ def test_filesystemencoding(self):
119119
# PYTHONLEGACYWINDOWSFSENCODING disables the UTF-8 mode
120120
# and has the priority over -X utf8 and PYTHONUTF8
121121
out = self.get_output('-X', 'utf8', '-c', code,
122-
PYTHONUTF8='strict',
122+
PYTHONUTF8='xxx',
123123
PYTHONLEGACYWINDOWSFSENCODING='1')
124124
self.assertEqual(out, 'mbcs/replace')
125125

126-
@unittest.expectedFailure # TODO: RUSTPYTHON
127126
def test_stdio(self):
128127
code = textwrap.dedent('''
129128
import sys

‎crates/vm/src/vm/mod.rs‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1568,6 +1568,25 @@ impl VirtualMachine {
15681568
{
15691569
let io = import::import_builtin(self, "_io")?;
15701570

1571+
// CPython normalizes configured stdio encodings to CodecInfo.name.
1572+
#[cfg(all(feature = "host_env", feature = "stdio"))]
1573+
let stdio_encoding: Option<PyRef<PyUtf8Str>> = self
1574+
.state
1575+
.config
1576+
.settings
1577+
.stdio_encoding
1578+
.as_deref()
1579+
.map(|encoding| {
1580+
self.state
1581+
.codec_registry
1582+
.lookup(encoding, self)?
1583+
.as_tuple()
1584+
.as_object()
1585+
.get_attr("name", self)?
1586+
.try_into_value(self)
1587+
})
1588+
.transpose()?;
1589+
15711590
// Full stdio: FileIO → BufferedWriter → TextIOWrapper
15721591
#[cfg(all(feature = "host_env", feature = "stdio"))]
15731592
let make_stdio = |name: &str, fd: i32, write: bool| -> PyResult<PyObjectRef> {
@@ -1594,7 +1613,7 @@ impl VirtualMachine {
15941613
let line_buffering = buffered_stdio && (isatty || fd == 2);
15951614

15961615
let newline = if cfg!(windows) { None } else { Some("\n") };
1597-
let encoding = self.state.config.settings.stdio_encoding.as_deref();
1616+
let encoding = stdio_encoding.clone();
15981617
// stderr always uses backslashreplace (ignores stdio_errors)
15991618
let errors = if fd == 2 {
16001619
Some("backslashreplace")

‎scripts/update_lib/patch_spec.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -635,11 +635,21 @@ def select_remaining_patches(
635635
pending = list(existing)
636636
omitted = []
637637
for marker in remaining.get(cls, {}).get(method, []):
638-
if marker not in pending:
638+
original = marker
639+
if (
640+
original not in pending
641+
and marker.ut_method == UtMethod.ExpectedFailureIf
642+
and marker.cond
643+
):
644+
# A condition only retires part of an unconditional failure.
645+
original = marker._replace(
646+
ut_method=UtMethod.ExpectedFailure, cond=None
647+
)
648+
if original not in pending:
639649
raise ValueError(
640650
"Marker override cannot add, change, or reorder markers"
641651
)
642-
index = pending.index(marker)
652+
index = pending.index(original)
643653
omitted.extend(pending[:index])
644654
pending = pending[index + 1 :]
645655
omitted.extend(pending)

‎scripts/update_lib/tests/test_migrate.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -601,6 +601,66 @@ def test_added_changed_or_reordered_markers_fail(self):
601601
with self.subTest(markers=markers), self.assertRaises(ValueError):
602602
select_remaining_patches(current, document, Path(document["target"]))
603603

604+
def test_cli_narrows_unconditional_failure(self):
605+
def narrow(document):
606+
document["patches"]["Example"]["test_known"] = [
607+
{
608+
"ut_method": "expectedFailureIf",
609+
"cond": "MS_WINDOWS",
610+
"reason": "old failure",
611+
}
612+
]
613+
614+
status, result = self.invoke(narrow)
615+
self.assertEqual(status, 0)
616+
self.assertIn(
617+
'unittest.expectedFailureIf(MS_WINDOWS, "TODO: RUSTPYTHON; old failure")',
618+
result,
619+
)
620+
self.assertIn('unittest.skip("TODO: RUSTPYTHON; retained guard")', result)
621+
self.assertIn("self.assertEqual(1, 1)", result)
622+
623+
def test_narrowing_preserves_reason_order_and_existing_targets(self):
624+
failure = PatchSpec(UtMethod.ExpectedFailure, reason="known failure")
625+
narrowed = failure._replace(
626+
ut_method=UtMethod.ExpectedFailureIf, cond="MS_WINDOWS"
627+
)
628+
success = PatchSpec(UtMethod.ExpectedSuccess, reason="known success")
629+
current = {"Example": {"test_known": [failure, success]}}
630+
for markers in (
631+
[narrowed, success, narrowed],
632+
[narrowed._replace(reason="changed"), success],
633+
[success, narrowed],
634+
[narrowed._replace(ut_method=UtMethod.SkipIf), success],
635+
[narrowed._replace(ut_method=UtMethod.ExpectedSuccess), success],
636+
[narrowed._replace(cond=None), success],
637+
):
638+
document = {
639+
"target": "Lib/test/test_example.py",
640+
"patches": patches_to_json({"Example": {"test_known": markers}}),
641+
}
642+
with self.subTest(markers=markers), self.assertRaises(ValueError):
643+
select_remaining_patches(current, document, Path(document["target"]))
644+
645+
document["patches"] = patches_to_json({"Example": {"test_new": [narrowed]}})
646+
with self.assertRaises(ValueError):
647+
select_remaining_patches(current, document, Path(document["target"]))
648+
649+
def test_conditional_failure_cannot_be_broadened_or_changed(self):
650+
conditional = PatchSpec(UtMethod.ExpectedFailureIf, cond="MS_WINDOWS")
651+
current = {"Example": {"test_known": [conditional]}}
652+
for marker in (
653+
conditional._replace(ut_method=UtMethod.ExpectedFailure, cond=None),
654+
conditional._replace(cond="True"),
655+
conditional._replace(cond="not MS_WINDOWS"),
656+
):
657+
document = {
658+
"target": "Lib/test/test_example.py",
659+
"patches": patches_to_json({"Example": {"test_known": [marker]}}),
660+
}
661+
with self.subTest(marker=marker), self.assertRaises(ValueError):
662+
select_remaining_patches(current, document, Path(document["target"]))
663+
604664

605665
class ConditionalMarkerOverrideTests(unittest.TestCase):
606666
decorator = (

0 commit comments

Comments
 (0)