Skip to content

Commit ae6265f

Browse files
author
Victor Stinner
committed
Issue python#8715: Create PyUnicode_EncodeFSDefault() function: Encode a Unicode
object to Py_FileSystemDefaultEncoding with the "surrogateescape" error handler, return a bytes object. If Py_FileSystemDefaultEncoding is not set, fall back to UTF-8.
1 parent 59e62db commit ae6265f

10 files changed

Lines changed: 46 additions & 26 deletions

File tree

‎Doc/c-api/unicode.rst‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -396,6 +396,7 @@ used, passsing :func:PyUnicode_FSConverter as the conversion function:
396396

397397
Use :func:`PyUnicode_DecodeFSDefaultAndSize` if you know the string length.
398398

399+
399400
.. cfunction:: PyObject* PyUnicode_DecodeFSDefault(const char *s)
400401

401402
Decode a string using :cdata:`Py_FileSystemDefaultEncoding` and
@@ -404,6 +405,16 @@ used, passsing :func:PyUnicode_FSConverter as the conversion function:
404405
If :cdata:`Py_FileSystemDefaultEncoding` is not set, fall back to UTF-8.
405406

406407

408+
.. cfunction:: PyObject* PyUnicode_EncodeFSDefault(PyObject *unicode)
409+
410+
Encode a Unicode object to :cdata:`Py_FileSystemDefaultEncoding` with the
411+
``'surrogateescape'`` error handler, return a :func:`bytes` object.
412+
413+
If :cdata:`Py_FileSystemDefaultEncoding` is not set, fall back to UTF-8.
414+
415+
.. versionadded:: 3.2
416+
417+
407418
wchar_t Support
408419
"""""""""""""""
409420

‎Include/unicodeobject.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1268,6 +1268,16 @@ PyAPI_FUNC(PyObject*) PyUnicode_DecodeFSDefaultAndSize(
12681268
Py_ssize_t size /* size */
12691269
);
12701270

1271+
/* Encode a Unicode object to Py_FileSystemDefaultEncoding with the
1272+
"surrogateescape" error handler, return a bytes object.
1273+
1274+
If Py_FileSystemDefaultEncoding is not set, fall back to UTF-8.
1275+
*/
1276+
1277+
PyAPI_FUNC(PyObject*) PyUnicode_EncodeFSDefault(
1278+
PyObject *unicode
1279+
);
1280+
12711281
/* --- Methods & Slots ----------------------------------------------------
12721282
12731283
These are capable of handling Unicode objects and strings on input

‎Misc/NEWS‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ What's New in Python 3.2 Alpha 1?
1212
Core and Builtins
1313
-----------------
1414

15+
- Issue #8715: Create PyUnicode_EncodeFSDefault() function: Encode a Unicode
16+
object to Py_FileSystemDefaultEncoding with the "surrogateescape" error
17+
handler, return a bytes object. If Py_FileSystemDefaultEncoding is not set,
18+
fall back to UTF-8.
19+
1520
- Enable shortcuts for common encodings in PyUnicode_AsEncodedString() for any
1621
error handler, not only the default error handler (strict)
1722

‎Modules/_io/fileio.c‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -247,8 +247,7 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
247247
if (u == NULL)
248248
return -1;
249249

250-
stringobj = PyUnicode_AsEncodedString(
251-
u, Py_FileSystemDefaultEncoding, "surrogateescape");
250+
stringobj = PyUnicode_EncodeFSDefault(u);
252251
Py_DECREF(u);
253252
if (stringobj == NULL)
254253
return -1;

‎Modules/_tkinter.c‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3147,9 +3147,7 @@ PyInit__tkinter(void)
31473147
it also helps Tcl find its encodings. */
31483148
uexe = PyUnicode_FromWideChar(Py_GetProgramName(), -1);
31493149
if (uexe) {
3150-
cexe = PyUnicode_AsEncodedString(uexe,
3151-
Py_FileSystemDefaultEncoding,
3152-
NULL);
3150+
cexe = PyUnicode_EncodeFSDefault(uexe);
31533151
if (cexe)
31543152
Tcl_FindExecutable(PyBytes_AsString(cexe));
31553153
Py_XDECREF(cexe);

‎Modules/grpmodule.c‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,7 @@ grp_getgrnam(PyObject *self, PyObject *args)
111111

112112
if (!PyArg_ParseTuple(args, "U:getgrnam", &arg))
113113
return NULL;
114-
if ((bytes = PyUnicode_AsEncodedString(arg, Py_FileSystemDefaultEncoding,
115-
"surrogateescape")) == NULL)
114+
if ((bytes = PyUnicode_EncodeFSDefault(arg)) == NULL)
116115
return NULL;
117116
if (PyBytes_AsStringAndSize(bytes, &name, NULL) == -1)
118117
goto out;

‎Modules/pwdmodule.c‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,9 +132,7 @@ pwd_getpwnam(PyObject *self, PyObject *args)
132132

133133
if (!PyArg_ParseTuple(args, "U:getpwnam", &arg))
134134
return NULL;
135-
if ((bytes = PyUnicode_AsEncodedString(arg,
136-
Py_FileSystemDefaultEncoding,
137-
"surrogateescape")) == NULL)
135+
if ((bytes = PyUnicode_EncodeFSDefault(arg)) == NULL)
138136
return NULL;
139137
if (PyBytes_AsStringAndSize(bytes, &name, NULL) == -1)
140138
goto out;

‎Modules/spwdmodule.c‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,9 +118,7 @@ static PyObject* spwd_getspnam(PyObject *self, PyObject *args)
118118

119119
if (!PyArg_ParseTuple(args, "U:getspnam", &arg))
120120
return NULL;
121-
if ((bytes = PyUnicode_AsEncodedString(arg,
122-
Py_FileSystemDefaultEncoding,
123-
"surrogateescape")) == NULL)
121+
if ((bytes = PyUnicode_EncodeFSDefault(arg)) == NULL)
124122
return NULL;
125123
if (PyBytes_AsStringAndSize(bytes, &name, NULL) == -1)
126124
goto out;

‎Objects/unicodeobject.c‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1461,6 +1461,18 @@ PyObject *PyUnicode_AsEncodedObject(PyObject *unicode,
14611461
return NULL;
14621462
}
14631463

1464+
PyObject *PyUnicode_EncodeFSDefault(PyObject *unicode)
1465+
{
1466+
if (Py_FileSystemDefaultEncoding)
1467+
return PyUnicode_AsEncodedString(unicode,
1468+
Py_FileSystemDefaultEncoding,
1469+
"surrogateescape");
1470+
else
1471+
return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),
1472+
PyUnicode_GET_SIZE(unicode),
1473+
"surrogateescape");
1474+
}
1475+
14641476
PyObject *PyUnicode_AsEncodedString(PyObject *unicode,
14651477
const char *encoding,
14661478
const char *errors)
@@ -1646,9 +1658,7 @@ PyUnicode_FSConverter(PyObject* arg, void* addr)
16461658
arg = PyUnicode_FromObject(arg);
16471659
if (!arg)
16481660
return 0;
1649-
output = PyUnicode_AsEncodedObject(arg,
1650-
Py_FileSystemDefaultEncoding,
1651-
"surrogateescape");
1661+
output = PyUnicode_EncodeFSDefault(arg);
16521662
Py_DECREF(arg);
16531663
if (!output)
16541664
return 0;

‎Python/import.c‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1633,8 +1633,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
16331633
if (!v)
16341634
return NULL;
16351635
if (PyUnicode_Check(v)) {
1636-
v = PyUnicode_AsEncodedString(v,
1637-
Py_FileSystemDefaultEncoding, NULL);
1636+
v = PyUnicode_EncodeFSDefault(v);
16381637
if (v == NULL)
16391638
return NULL;
16401639
}
@@ -2752,14 +2751,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen,
27522751
char *subname;
27532752
PyObject *submod;
27542753
char *p;
2755-
if (!Py_FileSystemDefaultEncoding) {
2756-
item8 = PyUnicode_EncodeASCII(PyUnicode_AsUnicode(item),
2757-
PyUnicode_GetSize(item),
2758-
NULL);
2759-
} else {
2760-
item8 = PyUnicode_AsEncodedString(item,
2761-
Py_FileSystemDefaultEncoding, NULL);
2762-
}
2754+
item8 = PyUnicode_EncodeFSDefault(item);
27632755
if (!item8) {
27642756
PyErr_SetString(PyExc_ValueError, "Cannot encode path item");
27652757
return 0;

0 commit comments

Comments
 (0)