Skip to content

Fix UTF-8 startup option precedence and refresh canonical tests #9422

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 3 commits into from
Oct 28, 2026
Merged

Fix UTF-8 startup option precedence and refresh canonical tests #9422

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d5b160a
Fix UTF-8 startup option precedence and refresh canonical tests
codex Oct 28, 2026
d5341f6
Normalize configured standard-stream codec names
codex Oct 28, 2026
04b7a26
Retire passing standard-stream UTF-8 expected failure
codex Oct 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 75 additions & 23 deletions Lib/test/test_c_locale_coercion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,31 +12,33 @@
from test.support.script_helper import run_python_until_end


# Set the list of ways we expect to be able to ask for the "C" locale
EXPECTED_C_LOCALE_EQUIVALENTS = ["C", "invalid.ascii"]
# Set the list of ways we expect to be able to ask for the "C" locale.
# 'invalid.ascii' is an invalid LOCALE name and so should get turned in to the
# default locale, which is traditionally C.
EXPECTED_C_LOCALE_EQUIVALENTS = ["C", "POSIX", "invalid.ascii"]

# Set our expectation for the default encoding used in the C locale
# for the filesystem encoding and the standard streams
EXPECTED_C_LOCALE_STREAM_ENCODING = "ascii"
EXPECTED_C_LOCALE_FS_ENCODING = "ascii"

# Set our expectation for the default locale used when none is specified
DEFAULT_LOCALE_IS_C = True
EXPECT_COERCION_IN_DEFAULT_LOCALE = True

TARGET_LOCALES = ["C.UTF-8", "C.utf8", "UTF-8"]

# Apply some platform dependent overrides
if sys.platform.startswith("linux"):
if support.is_android:
# Android defaults to using UTF-8 for all system interfaces
EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8"
EXPECTED_C_LOCALE_FS_ENCODING = "utf-8"
else:
# Linux distros typically alias the POSIX locale directly to the C
# locale.
# TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be
# able to check this case unconditionally
EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX")
if sys.platform == "android":
# Android defaults to using UTF-8 for all system interfaces
EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8"
EXPECTED_C_LOCALE_FS_ENCODING = "utf-8"
elif support.linked_to_musl():
# MUSL defaults to utf-8 unless the C locale is set explicitly.
EXPECTED_C_LOCALE_EQUIVALENTS = ["C"]
DEFAULT_LOCALE_IS_C = False
DEFAULT_ENCODING = 'utf-8'
EXPECT_COERCION_IN_DEFAULT_LOCALE = False
elif sys.platform.startswith("aix"):
# AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII
EXPECTED_C_LOCALE_STREAM_ENCODING = "iso8859-1"
Expand Down Expand Up @@ -112,12 +114,16 @@ class EncodingDetails(_EncodingDetails):
])

@classmethod
def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, env_vars):
def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, stream_errors, env_vars):
"""Returns expected child process details for a given encoding"""
_stream = stream_encoding + ":{}"
# stdin and stdout should use surrogateescape either because the
# coercion triggered, or because the C locale was detected
stream_info = 2*[_stream.format("surrogateescape")]
if stream_errors is None:
# stdin and stdout should use surrogateescape either because the
# coercion triggered, or because the C locale was detected
stream_errors = "surrogateescape"

stream_info = [_stream.format(stream_errors)] * 2

# stderr should always use backslashreplace
stream_info.append(_stream.format("backslashreplace"))
expected_lang = env_vars.get("LANG", "not set")
Expand Down Expand Up @@ -210,6 +216,7 @@ def _check_child_encoding_details(self,
env_vars,
expected_fs_encoding,
expected_stream_encoding,
expected_stream_errors,
expected_warnings,
coercion_expected):
"""Check the C locale handling for the given process environment
Expand All @@ -225,6 +232,7 @@ def _check_child_encoding_details(self,
coercion_expected,
expected_fs_encoding,
expected_stream_encoding,
expected_stream_errors,
env_vars
)
self.assertEqual(encoding_details, expected_details)
Expand Down Expand Up @@ -257,6 +265,7 @@ def test_external_target_locale_configuration(self):
"LC_CTYPE": "",
"LC_ALL": "",
"PYTHONCOERCECLOCALE": "",
"PYTHONIOENCODING": "",
}
for env_var in ("LANG", "LC_CTYPE"):
for locale_to_set in AVAILABLE_TARGETS:
Expand All @@ -273,10 +282,43 @@ def test_external_target_locale_configuration(self):
self._check_child_encoding_details(var_dict,
expected_fs_encoding,
expected_stream_encoding,
expected_stream_errors=None,
expected_warnings=None,
coercion_expected=False)

def test_with_ioencoding(self):
# Explicitly setting a target locale should give the same behaviour as
# is seen when implicitly coercing to that target locale
self.maxDiff = None

expected_fs_encoding = "utf-8"
expected_stream_encoding = "utf-8"

base_var_dict = {
"LANG": "",
"LC_CTYPE": "",
"LC_ALL": "",
"PYTHONCOERCECLOCALE": "",
"PYTHONIOENCODING": "UTF-8",
}
for env_var in ("LANG", "LC_CTYPE"):
for locale_to_set in AVAILABLE_TARGETS:
# XXX (ncoghlan): LANG=UTF-8 doesn't appear to work as
# expected, so skip that combination for now
# See https://bugs.python.org/issue30672 for discussion
if env_var == "LANG" and locale_to_set == "UTF-8":
continue

with self.subTest(env_var=env_var,
configured_locale=locale_to_set):
var_dict = base_var_dict.copy()
var_dict[env_var] = locale_to_set
self._check_child_encoding_details(var_dict,
expected_fs_encoding,
expected_stream_encoding,
expected_stream_errors="strict",
expected_warnings=None,
coercion_expected=False)

@support.cpython_only
@unittest.skipUnless(sysconfig.get_config_var("PY_COERCE_C_LOCALE"),
Expand Down Expand Up @@ -316,15 +358,21 @@ def _check_c_locale_coercion(self,
"LC_CTYPE": "",
"LC_ALL": "",
"PYTHONCOERCECLOCALE": "",
"PYTHONIOENCODING": "",
}
base_var_dict.update(extra_vars)
if coerce_c_locale is not None:
base_var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale

# Check behaviour for the default locale
_fs_encoding = fs_encoding
_stream_encoding = stream_encoding
if not DEFAULT_LOCALE_IS_C and 'LC_ALL' not in extra_vars:
_fs_encoding = _stream_encoding = DEFAULT_ENCODING
with self.subTest(default_locale=True,
PYTHONCOERCECLOCALE=coerce_c_locale):
if EXPECT_COERCION_IN_DEFAULT_LOCALE:
if (EXPECT_COERCION_IN_DEFAULT_LOCALE
or (not DEFAULT_LOCALE_IS_C and 'LC_ALL' in extra_vars)):
_expected_warnings = expected_warnings
_coercion_expected = coercion_expected
else:
Expand All @@ -338,8 +386,9 @@ def _check_c_locale_coercion(self,
_expected_warnings == [CLI_COERCION_WARNING]):
_expected_warnings = None
self._check_child_encoding_details(base_var_dict,
fs_encoding,
stream_encoding,
_fs_encoding,
_stream_encoding,
None,
_expected_warnings,
_coercion_expected)

Expand All @@ -348,13 +397,15 @@ def _check_c_locale_coercion(self,
for env_var in ("LANG", "LC_CTYPE"):
with self.subTest(env_var=env_var,
nominal_locale=locale_to_set,
PYTHONCOERCECLOCALE=coerce_c_locale):
PYTHONCOERCECLOCALE=coerce_c_locale,
PYTHONIOENCODING=""):
var_dict = base_var_dict.copy()
var_dict[env_var] = locale_to_set
# Check behaviour on successful coercion
self._check_child_encoding_details(var_dict,
fs_encoding,
stream_encoding,
None,
expected_warnings,
coercion_expected)

Expand Down Expand Up @@ -411,8 +462,9 @@ def test_PYTHONCOERCECLOCALE_set_to_one(self):
loc = locale.setlocale(locale.LC_CTYPE, "")
except locale.Error as e:
self.skipTest(str(e))
if loc == "C":
self.skipTest("test requires LC_CTYPE locale different than C")
if loc in ("C", "POSIX"):
self.skipTest("test requires LC_CTYPE locale different "
"than C and POSIX")
if loc in TARGET_LOCALES :
self.skipTest("coerced LC_CTYPE locale: %s" % loc)

Expand Down
9 changes: 4 additions & 5 deletions Lib/test/test_utf8_mode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def test_xoption(self):
PYTHONLEGACYWINDOWSFSENCODING='1')
self.assertEqual(out, '0')

@unittest.expectedFailure # TODO: RUSTPYTHON
@unittest.expectedFailureIf(MS_WINDOWS, "TODO: RUSTPYTHON")
def test_env_var(self):
code = 'import sys; print(sys.flags.utf8_mode)'

Expand All @@ -91,8 +91,8 @@ def test_env_var(self):
# the UTF-8 mode
if not self.posix_locale():
# PYTHONUTF8 should be ignored if -E is used
out = self.get_output('-E', '-c', code, PYTHONUTF8='1')
self.assertEqual(out, '0')
out = self.get_output('-E', '-c', code, PYTHONUTF8='0')
self.assertEqual(out, '1')

# invalid mode
out = self.get_output('-c', code, PYTHONUTF8='xxx', failure=True)
Expand All @@ -119,11 +119,10 @@ def test_filesystemencoding(self):
# PYTHONLEGACYWINDOWSFSENCODING disables the UTF-8 mode
# and has the priority over -X utf8 and PYTHONUTF8
out = self.get_output('-X', 'utf8', '-c', code,
PYTHONUTF8='strict',
PYTHONUTF8='xxx',
PYTHONLEGACYWINDOWSFSENCODING='1')
self.assertEqual(out, 'mbcs/replace')

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_stdio(self):
code = textwrap.dedent('''
import sys
Expand Down
21 changes: 20 additions & 1 deletion crates/vm/src/vm/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1568,6 +1568,25 @@ impl VirtualMachine {
{
let io = import::import_builtin(self, "_io")?;

// CPython normalizes configured stdio encodings to CodecInfo.name.
#[cfg(all(feature = "host_env", feature = "stdio"))]
let stdio_encoding: Option<PyRef<PyUtf8Str>> = self
.state
.config
.settings
.stdio_encoding
.as_deref()
.map(|encoding| {
self.state
.codec_registry
.lookup(encoding, self)?
.as_tuple()
.as_object()
.get_attr("name", self)?
.try_into_value(self)
})
.transpose()?;

// Full stdio: FileIO → BufferedWriter → TextIOWrapper
#[cfg(all(feature = "host_env", feature = "stdio"))]
let make_stdio = |name: &str, fd: i32, write: bool| -> PyResult<PyObjectRef> {
Expand All @@ -1594,7 +1613,7 @@ impl VirtualMachine {
let line_buffering = buffered_stdio && (isatty || fd == 2);

let newline = if cfg!(windows) { None } else { Some("\n") };
let encoding = self.state.config.settings.stdio_encoding.as_deref();
let encoding = stdio_encoding.clone();
// stderr always uses backslashreplace (ignores stdio_errors)
let errors = if fd == 2 {
Some("backslashreplace")
Expand Down
14 changes: 12 additions & 2 deletions scripts/update_lib/patch_spec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -635,11 +635,21 @@ def select_remaining_patches(
pending = list(existing)
omitted = []
for marker in remaining.get(cls, {}).get(method, []):
if marker not in pending:
original = marker
if (
original not in pending
and marker.ut_method == UtMethod.ExpectedFailureIf
and marker.cond
):
# A condition only retires part of an unconditional failure.
original = marker._replace(
ut_method=UtMethod.ExpectedFailure, cond=None
)
if original not in pending:
raise ValueError(
"Marker override cannot add, change, or reorder markers"
)
index = pending.index(marker)
index = pending.index(original)
omitted.extend(pending[:index])
pending = pending[index + 1 :]
omitted.extend(pending)
Expand Down
60 changes: 60 additions & 0 deletions scripts/update_lib/tests/test_migrate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -513,6 +513,66 @@ def test_added_changed_or_reordered_markers_fail(self):
with self.subTest(markers=markers), self.assertRaises(ValueError):
select_remaining_patches(current, document, Path(document["target"]))

def test_cli_narrows_unconditional_failure(self):
def narrow(document):
document["patches"]["Example"]["test_known"] = [
{
"ut_method": "expectedFailureIf",
"cond": "MS_WINDOWS",
"reason": "old failure",
}
]

status, result = self.invoke(narrow)
self.assertEqual(status, 0)
self.assertIn(
'unittest.expectedFailureIf(MS_WINDOWS, "TODO: RUSTPYTHON; old failure")',
result,
)
self.assertIn('unittest.skip("TODO: RUSTPYTHON; retained guard")', result)
self.assertIn("self.assertEqual(1, 1)", result)

def test_narrowing_preserves_reason_order_and_existing_targets(self):
failure = PatchSpec(UtMethod.ExpectedFailure, reason="known failure")
narrowed = failure._replace(
ut_method=UtMethod.ExpectedFailureIf, cond="MS_WINDOWS"
)
success = PatchSpec(UtMethod.ExpectedSuccess, reason="known success")
current = {"Example": {"test_known": [failure, success]}}
for markers in (
[narrowed, success, narrowed],
[narrowed._replace(reason="changed"), success],
[success, narrowed],
[narrowed._replace(ut_method=UtMethod.SkipIf), success],
[narrowed._replace(ut_method=UtMethod.ExpectedSuccess), success],
[narrowed._replace(cond=None), success],
):
document = {
"target": "Lib/test/test_example.py",
"patches": patches_to_json({"Example": {"test_known": markers}}),
}
with self.subTest(markers=markers), self.assertRaises(ValueError):
select_remaining_patches(current, document, Path(document["target"]))

document["patches"] = patches_to_json({"Example": {"test_new": [narrowed]}})
with self.assertRaises(ValueError):
select_remaining_patches(current, document, Path(document["target"]))

def test_conditional_failure_cannot_be_broadened_or_changed(self):
conditional = PatchSpec(UtMethod.ExpectedFailureIf, cond="MS_WINDOWS")
current = {"Example": {"test_known": [conditional]}}
for marker in (
conditional._replace(ut_method=UtMethod.ExpectedFailure, cond=None),
conditional._replace(cond="True"),
conditional._replace(cond="not MS_WINDOWS"),
):
document = {
"target": "Lib/test/test_example.py",
"patches": patches_to_json({"Example": {"test_known": [marker]}}),
}
with self.subTest(marker=marker), self.assertRaises(ValueError):
select_remaining_patches(current, document, Path(document["target"]))


class ConditionalMarkerOverrideTests(unittest.TestCase):
decorator = (
Expand Down
Loading
Loading