Skip to content

bpo-34527: POSIX locale enables the UTF-8 Mode #8972

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 2 commits into from
Aug 28, 2018
Merged

bpo-34527: POSIX locale enables the UTF-8 Mode #8972

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a08f921
bpo-34527: POSIX locale enables the UTF-8 Mode
vstinner Aug 28, 2018
3464d64
Fix test_utf8_mode.test_env_var()
vstinner Aug 28, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 17 additions & 6 deletions Lib/test/test_utf8_mode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@


MS_WINDOWS = (sys.platform == 'win32')
POSIX_LOCALES = ('C', 'POSIX')


class UTF8ModeTests(unittest.TestCase):
Expand All @@ -23,7 +24,7 @@ class UTF8ModeTests(unittest.TestCase):

def posix_locale(self):
loc = locale.setlocale(locale.LC_CTYPE, None)
return (loc == 'C')
return (loc in POSIX_LOCALES)

def get_output(self, *args, failure=False, **kw):
kw = dict(self.DEFAULT_ENV, **kw)
Expand All @@ -39,8 +40,10 @@ def get_output(self, *args, failure=False, **kw):
def test_posix_locale(self):
code = 'import sys; print(sys.flags.utf8_mode)'

out = self.get_output('-c', code, LC_ALL='C')
self.assertEqual(out, '1')
for loc in POSIX_LOCALES:
with self.subTest(LC_ALL=loc):
out = self.get_output('-c', code, LC_ALL=loc)
self.assertEqual(out, '1')

def test_xoption(self):
code = 'import sys; print(sys.flags.utf8_mode)'
Expand Down Expand Up @@ -201,8 +204,10 @@ def test_locale_getpreferredencoding(self):
out = self.get_output('-X', 'utf8', '-c', code)
self.assertEqual(out, 'UTF-8 UTF-8')

out = self.get_output('-X', 'utf8', '-c', code, LC_ALL='C')
self.assertEqual(out, 'UTF-8 UTF-8')
for loc in POSIX_LOCALES:
with self.subTest(LC_ALL=loc):
out = self.get_output('-X', 'utf8', '-c', code, LC_ALL=loc)
self.assertEqual(out, 'UTF-8 UTF-8')

@unittest.skipIf(MS_WINDOWS, 'test specific to Unix')
def test_cmd_line(self):
Expand All @@ -217,13 +222,19 @@ def check(utf8_opt, expected, **kw):
self.assertEqual(args, ascii(expected), out)

check('utf8', [arg_utf8])
for loc in POSIX_LOCALES:
with self.subTest(LC_ALL=loc):
check('utf8', [arg_utf8], LC_ALL=loc)

if sys.platform == 'darwin' or support.is_android:
c_arg = arg_utf8
elif sys.platform.startswith("aix"):
c_arg = arg.decode('iso-8859-1')
else:
c_arg = arg_ascii
check('utf8=0', [c_arg], LC_ALL='C')
for loc in POSIX_LOCALES:
with self.subTest(LC_ALL=loc):
check('utf8=0', [c_arg], LC_ALL=loc)

def test_optim_level(self):
# CPython: check that Py_Main() doesn't increment Py_OptimizeFlag
Expand Down
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
The UTF-8 Mode is now also enabled by the "POSIX" locale, not only by the "C"
locale.
3 changes: 3 additions & 0 deletions Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
On FreeBSD, Py_DecodeLocale() and Py_EncodeLocale() now also forces the
ASCII encoding if the LC_CTYPE locale is "POSIX", not only if the LC_CTYPE
locale is "C".
16 changes: 12 additions & 4 deletions Python/coreconfig.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "Python.h"
#include "internal/pystate.h"
#include <locale.h>


#define DECODE_LOCALE_ERR(NAME, LEN) \
Expand Down Expand Up @@ -828,14 +829,21 @@ static void
config_init_locale(_PyCoreConfig *config)
{
if (_Py_LegacyLocaleDetected()) {
/* POSIX locale: enable C locale coercion and UTF-8 Mode */
if (config->utf8_mode < 0) {
config->utf8_mode = 1;
}
/* The C locale enables the C locale coercion (PEP 538) */
if (config->coerce_c_locale < 0) {
config->coerce_c_locale = 1;
}
}
#ifndef MS_WINDOWS
const char *ctype_loc = setlocale(LC_CTYPE, NULL);
if (ctype_loc != NULL
&& (strcmp(ctype_loc, "C") == 0 || strcmp(ctype_loc, "POSIX") == 0)) {
/* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */
if (config->utf8_mode < 0) {
config->utf8_mode = 1;
}
}
#endif
}


Expand Down
2 changes: 1 addition & 1 deletion Python/fileutils.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ check_force_ascii(void)
loc = setlocale(LC_CTYPE, NULL);
if (loc == NULL)
goto error;
if (strcmp(loc, "C") != 0) {
if (strcmp(loc, "C") != 0 && strcmp(loc, "POSIX") != 0) {
/* the LC_CTYPE locale is different than C */
return 0;
}
Expand Down