Skip to content

Commit 6c154d3

Browse files
serhiy-storchakaserhiy-storchaka
committed
gh-110815: Support non-ASCII keyword names in PyArg_ParseTupleAndKeywords()
It already mostly worked, except in the case when invalid keyword argument with non-ASCII name was passed to function with non-ASCII parameter names. Then it crashed in the debug mode.
1 parent 2a68f77 commit 6c154d3

File tree

6 files changed

+117
-21
lines changed

6 files changed

+117
-21
lines changed

‎Doc/c-api/arg.rst‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -416,8 +416,10 @@ API Functions
416416
.. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], ...)
417417
418418
Parse the parameters of a function that takes both positional and keyword
419-
parameters into local variables. The *keywords* argument is a
420-
``NULL``-terminated array of keyword parameter names. Empty names denote
419+
parameters into local variables.
420+
The *keywords* argument is a ``NULL``-terminated array of keyword parameter
421+
names specified as null-terminated ASCII or UTF-8 encoded C strings.
422+
Empty names denote
421423
:ref:`positional-only parameters <positional-only_parameter>`.
422424
Returns true on success; on failure, it returns false and raises the
423425
appropriate exception.
@@ -426,6 +428,9 @@ API Functions
426428
Added support for :ref:`positional-only parameters
427429
<positional-only_parameter>`.
428430
431+
.. versionchanged:: 3.13
432+
Added support for non-ASCII keyword parameter names.
433+
429434
430435
.. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], va_list vargs)
431436

‎Doc/whatsnew/3.13.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1045,6 +1045,10 @@ New Features
10451045
but pass event arguments as a Python :class:`tuple` object.
10461046
(Contributed by Victor Stinner in :gh:`85283`.)
10471047

1048+
* :c:func:`PyArg_ParseTupleAndKeywords` now supports non-ASCII keyword
1049+
parameter names.
1050+
(Contributed by Serhiy Storchaka in :gh:`110815`.)
1051+
10481052
Porting to Python 3.13
10491053
----------------------
10501054

‎Lib/test/test_capi/test_getargs.py‎

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@
5555
LLONG_MIN = -2**63
5656
ULLONG_MAX = 2**64-1
5757

58+
NULL = None
59+
5860
class Index:
5961
def __index__(self):
6062
return 99
@@ -1187,20 +1189,23 @@ def test_bad_use(self):
11871189
def test_positional_only(self):
11881190
parse = _testcapi.parse_tuple_and_keywords
11891191

1190-
parse((1, 2, 3), {}, 'OOO', ['', '', 'a'])
1191-
parse((1, 2), {'a': 3}, 'OOO', ['', '', 'a'])
1192+
self.assertEqual(parse((1, 2, 3), {}, 'OOO', ['', '', 'a']), (1, 2, 3))
1193+
self.assertEqual(parse((1, 2), {'a': 3}, 'OOO', ['', '', 'a']), (1, 2, 3))
11921194
with self.assertRaisesRegex(TypeError,
11931195
r'function takes at least 2 positional arguments \(1 given\)'):
11941196
parse((1,), {'a': 3}, 'OOO', ['', '', 'a'])
1195-
parse((1,), {}, 'O|OO', ['', '', 'a'])
1197+
self.assertEqual(parse((1,), {}, 'O|OO', ['', '', 'a']),
1198+
(1, NULL, NULL))
11961199
with self.assertRaisesRegex(TypeError,
11971200
r'function takes at least 1 positional argument \(0 given\)'):
11981201
parse((), {}, 'O|OO', ['', '', 'a'])
1199-
parse((1, 2), {'a': 3}, 'OO$O', ['', '', 'a'])
1202+
self.assertEqual(parse((1, 2), {'a': 3}, 'OO$O', ['', '', 'a']),
1203+
(1, 2, 3))
12001204
with self.assertRaisesRegex(TypeError,
12011205
r'function takes exactly 2 positional arguments \(1 given\)'):
12021206
parse((1,), {'a': 3}, 'OO$O', ['', '', 'a'])
1203-
parse((1,), {}, 'O|O$O', ['', '', 'a'])
1207+
self.assertEqual(parse((1,), {}, 'O|O$O', ['', '', 'a']),
1208+
(1, NULL, NULL))
12041209
with self.assertRaisesRegex(TypeError,
12051210
r'function takes at least 1 positional argument \(0 given\)'):
12061211
parse((), {}, 'O|O$O', ['', '', 'a'])
@@ -1209,6 +1214,61 @@ def test_positional_only(self):
12091214
with self.assertRaisesRegex(SystemError, 'Empty keyword'):
12101215
parse((1,), {}, 'O|OO', ['', 'a', ''])
12111216

1217+
def test_nonascii_keywords(self):
1218+
parse = _testcapi.parse_tuple_and_keywords
1219+
1220+
for name in ('a', 'ä', 'ŷ', '㷷', '𐀀'):
1221+
with self.subTest(name=name):
1222+
self.assertEqual(parse((), {name: 1}, 'O', [name]), (1,))
1223+
self.assertEqual(parse((), {}, '|O', [name]), (NULL,))
1224+
with self.assertRaisesRegex(TypeError,
1225+
f"function missing required argument '{name}'"):
1226+
parse((), {}, 'O', [name])
1227+
with self.assertRaisesRegex(TypeError,
1228+
fr"argument for function given by name \('{name}'\) "
1229+
fr"and position \(1\)"):
1230+
parse((1,), {name: 2}, 'O|O', [name, 'b'])
1231+
with self.assertRaisesRegex(TypeError,
1232+
f"'{name}' is an invalid keyword argument"):
1233+
parse((), {name: 1}, '|O', ['b'])
1234+
with self.assertRaisesRegex(TypeError,
1235+
"'b' is an invalid keyword argument"):
1236+
parse((), {'b': 1}, '|O', [name])
1237+
1238+
invalid = name.encode() + (name.encode()[:-1] or b'\x80')
1239+
self.assertEqual(parse((), {}, '|O', [invalid]), (NULL,))
1240+
self.assertEqual(parse((1,), {'b': 2}, 'O|O', [invalid, 'b']),
1241+
(1, 2))
1242+
with self.assertRaisesRegex(TypeError,
1243+
f"function missing required argument '{name}\ufffd'"):
1244+
parse((), {}, 'O', [invalid])
1245+
with self.assertRaisesRegex(UnicodeDecodeError,
1246+
f"'utf-8' codec can't decode bytes? "):
1247+
parse((), {'b': 1}, '|OO', [invalid, 'b'])
1248+
with self.assertRaisesRegex(UnicodeDecodeError,
1249+
f"'utf-8' codec can't decode bytes? "):
1250+
parse((), {'b': 1}, '|O', [invalid])
1251+
1252+
for name2 in ('b', 'ë', 'ĉ', 'Ɐ', '𐀁'):
1253+
with self.subTest(name2=name2):
1254+
with self.assertRaisesRegex(TypeError,
1255+
f"'{name2}' is an invalid keyword argument"):
1256+
parse((), {name2: 1}, '|O', [name])
1257+
1258+
name2 = name.encode().decode('latin1')
1259+
if name2 != name:
1260+
with self.assertRaisesRegex(TypeError,
1261+
f"'{name2}' is an invalid keyword argument"):
1262+
#with self.assertRaisesRegex(TypeError,
1263+
#f"^invalid keyword argument"):
1264+
parse((), {name2: 1}, '|O', [name])
1265+
name3 = name + '3'
1266+
with self.assertRaisesRegex(TypeError,
1267+
f"'{name2}' is an invalid keyword argument"):
1268+
#with self.assertRaisesRegex(TypeError,
1269+
#f"^invalid keyword argument"):
1270+
parse((), {name2: 1, name3: 2}, '|OO', [name, name3])
1271+
12121272

12131273
class Test_testcapi(unittest.TestCase):
12141274
locals().update((name, getattr(_testcapi, name))

‎Misc/NEWS.d/next/C API/2023-10-13-14-18-06.gh-issue-110815.tEFLVl.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Support non-ASCII keyword names in :c:func:`PyArg_ParseTupleAndKeywords`.

‎Modules/_testcapi/getargs.c‎

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args)
1313
const char *sub_format;
1414
PyObject *sub_keywords;
1515

16-
double buffers[8][4]; /* double ensures alignment where necessary */
17-
PyObject *converted[8];
18-
char *keywords[8 + 1]; /* space for NULL at end */
16+
#define MAX_PARAMS 8
17+
double buffers[MAX_PARAMS][4]; /* double ensures alignment where necessary */
18+
char *keywords[MAX_PARAMS + 1]; /* space for NULL at end */
1919

2020
PyObject *return_value = NULL;
2121

@@ -35,41 +35,67 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args)
3535
}
3636

3737
memset(buffers, 0, sizeof(buffers));
38-
memset(converted, 0, sizeof(converted));
3938
memset(keywords, 0, sizeof(keywords));
4039

4140
Py_ssize_t size = PySequence_Fast_GET_SIZE(sub_keywords);
42-
if (size > 8) {
41+
if (size > MAX_PARAMS) {
4342
PyErr_SetString(PyExc_ValueError,
4443
"parse_tuple_and_keywords: too many keywords in sub_keywords");
4544
goto exit;
4645
}
4746

4847
for (Py_ssize_t i = 0; i < size; i++) {
4948
PyObject *o = PySequence_Fast_GET_ITEM(sub_keywords, i);
50-
if (!PyUnicode_FSConverter(o, (void *)(converted + i))) {
49+
if (PyUnicode_Check(o)) {
50+
keywords[i] = (char *)PyUnicode_AsUTF8(o);
51+
if (keywords[i] == NULL) {
52+
goto exit;
53+
}
54+
}
55+
else if (PyBytes_Check(o)) {
56+
keywords[i] = PyBytes_AS_STRING(o);
57+
}
58+
else {
5159
PyErr_Format(PyExc_ValueError,
5260
"parse_tuple_and_keywords: "
53-
"could not convert keywords[%zd] to narrow string", i);
61+
"keywords must be str or bytes", i);
5462
goto exit;
5563
}
56-
keywords[i] = PyBytes_AS_STRING(converted[i]);
5764
}
5865

66+
assert(MAX_PARAMS == 8);
5967
int result = PyArg_ParseTupleAndKeywords(sub_args, sub_kwargs,
6068
sub_format, keywords,
6169
buffers + 0, buffers + 1, buffers + 2, buffers + 3,
6270
buffers + 4, buffers + 5, buffers + 6, buffers + 7);
6371

6472
if (result) {
65-
return_value = Py_NewRef(Py_None);
73+
int objects_only = 1;
74+
for (const char *f = sub_format; *f; f++) {
75+
if (Py_ISALNUM(*f) && strchr("OSUY", *f) == NULL) {
76+
objects_only = 0;
77+
break;
78+
}
79+
}
80+
if (objects_only) {
81+
return_value = PyTuple_New(size);
82+
if (return_value == NULL) {
83+
goto exit;
84+
}
85+
for (Py_ssize_t i = 0; i < size; i++) {
86+
PyObject *arg = *(PyObject **)(buffers + i);
87+
if (arg == NULL) {
88+
arg = Py_None;
89+
}
90+
PyTuple_SET_ITEM(return_value, i, Py_NewRef(arg));
91+
}
92+
}
93+
else {
94+
return_value = Py_NewRef(Py_None);
95+
}
6696
}
6797

6898
exit:
69-
size = sizeof(converted) / sizeof(converted[0]);
70-
for (Py_ssize_t i = 0; i < size; i++) {
71-
Py_XDECREF(converted[i]);
72-
}
7399
return return_value;
74100
}
75101

‎Python/getargs.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1729,7 +1729,7 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format,
17291729
return cleanreturn(0, &freelist);
17301730
}
17311731
for (i = pos; i < len; i++) {
1732-
if (_PyUnicode_EqualToASCIIString(key, kwlist[i])) {
1732+
if (PyUnicode_EqualToUTF8(key, kwlist[i])) {
17331733
match = 1;
17341734
break;
17351735
}

0 commit comments

Comments
 (0)