Skip to content

Commit e822b03

Browse files
Issue python#15866: The xmlcharrefreplace error handler no more produces two XML
entities for a non-BMP character on narrow build.
1 parent 5ad3514 commit e822b03

6 files changed

Lines changed: 135 additions & 55 deletions

File tree

‎Lib/test/test_codeccallbacks.py‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,15 +66,34 @@ def test_xmlcharrefreplace(self):
6666
# replace unencodable characters which numeric character entities.
6767
# For ascii, latin-1 and charmaps this is completely implemented
6868
# in C and should be reasonably fast.
69-
s = u"\u30b9\u30d1\u30e2 \xe4nd eggs"
69+
s = u"\u30b9\u30d1\u30e2 \xe4nd egg\u0161"
7070
self.assertEqual(
7171
s.encode("ascii", "xmlcharrefreplace"),
72-
"スパモ änd eggs"
72+
"スパモ änd eggš"
7373
)
7474
self.assertEqual(
7575
s.encode("latin-1", "xmlcharrefreplace"),
76-
"スパモ \xe4nd eggs"
76+
"スパモ \xe4nd eggš"
7777
)
78+
self.assertEqual(
79+
s.encode("iso-8859-15", "xmlcharrefreplace"),
80+
"スパモ \xe4nd egg\xa8"
81+
)
82+
83+
def test_xmlcharrefreplace_with_surrogates(self):
84+
tests = [(u'\U0001f49d', '💝'),
85+
(u'\ud83d', '�'),
86+
(u'\udc9d', '�'),
87+
(u'\ud83d\udc9d', '💝' if len(u'\U0001f49d') > 1 else
88+
'��'),
89+
]
90+
for encoding in ['ascii', 'latin1', 'iso-8859-15']:
91+
for s, exp in tests:
92+
self.assertEqual(s.encode(encoding, 'xmlcharrefreplace'),
93+
exp, msg='%r.encode(%r)' % (s, encoding))
94+
self.assertEqual((s+'X').encode(encoding, 'xmlcharrefreplace'),
95+
exp+'X',
96+
msg='%r.encode(%r)' % (s + 'X', encoding))
7897

7998
def test_xmlcharnamereplace(self):
8099
# This time use a named character entity for unencodable

‎Lib/test/test_unicode.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1658,6 +1658,18 @@ def test_encode_decimal(self):
16581658
self.assertEqual(unicode_encodedecimal(u"123\u20ac\u0660", "replace"),
16591659
b'123?0')
16601660

1661+
def test_encode_decimal_with_surrogates(self):
1662+
from _testcapi import unicode_encodedecimal
1663+
tests = [(u'\U0001f49d', '💝'),
1664+
(u'\ud83d', '�'),
1665+
(u'\udc9d', '�'),
1666+
(u'\ud83d\udc9d', '💝' if len(u'\U0001f49d') > 1 else
1667+
'��'),
1668+
]
1669+
for s, exp in tests:
1670+
self.assertEqual(
1671+
unicode_encodedecimal(u"123" + s, "xmlcharrefreplace"),
1672+
'123' + exp)
16611673

16621674
def test_main():
16631675
test_support.run_unittest(__name__)

‎Misc/NEWS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ What's New in Python 2.7.6?
99
Core and Builtins
1010
-----------------
1111

12+
- Issue #15866: The xmlcharrefreplace error handler no more produces two XML
13+
entities for a non-BMP character on narrow build.
14+
1215
- Issue #18184: PyUnicode_FromFormat() and PyUnicode_FromFormatV() now raise
1316
OverflowError when an argument of %c format is out of range.
1417

‎Modules/_testcapimodule.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1118,7 +1118,7 @@ unicode_encodedecimal(PyObject *self, PyObject *args)
11181118
if (!PyArg_ParseTuple(args, "u#|s", &unicode, &length, &errors))
11191119
return NULL;
11201120

1121-
decimal_length = length * 7; /* len('€') */
1121+
decimal_length = length * 10; /* len('') */
11221122
decimal = PyBytes_FromStringAndSize(NULL, decimal_length);
11231123
if (decimal == NULL)
11241124
return NULL;

‎Objects/unicodeobject.c‎

Lines changed: 61 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -547,6 +547,37 @@ PyObject *PyUnicode_FromString(const char *u)
547547
return PyUnicode_FromStringAndSize(u, size);
548548
}
549549

550+
/* _Py_UNICODE_NEXT is a private macro used to retrieve the character pointed
551+
* by 'ptr', possibly combining surrogate pairs on narrow builds.
552+
* 'ptr' and 'end' must be Py_UNICODE*, with 'ptr' pointing at the character
553+
* that should be returned and 'end' pointing to the end of the buffer.
554+
* ('end' is used on narrow builds to detect a lone surrogate at the
555+
* end of the buffer that should be returned unchanged.)
556+
* The ptr and end arguments should be side-effect free and ptr must an lvalue.
557+
* The type of the returned char is always Py_UCS4.
558+
*
559+
* Note: the macro advances ptr to next char, so it might have side-effects
560+
* (especially if used with other macros).
561+
*/
562+
563+
/* helper macros used by _Py_UNICODE_NEXT */
564+
#define _Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDBFF)
565+
#define _Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= ch && ch <= 0xDFFF)
566+
/* Join two surrogate characters and return a single Py_UCS4 value. */
567+
#define _Py_UNICODE_JOIN_SURROGATES(high, low) \
568+
(((((Py_UCS4)(high) & 0x03FF) << 10) | \
569+
((Py_UCS4)(low) & 0x03FF)) + 0x10000)
570+
571+
#ifdef Py_UNICODE_WIDE
572+
#define _Py_UNICODE_NEXT(ptr, end) *(ptr)++
573+
#else
574+
#define _Py_UNICODE_NEXT(ptr, end) \
575+
(((_Py_UNICODE_IS_HIGH_SURROGATE(*(ptr)) && (ptr) < (end)) && \
576+
_Py_UNICODE_IS_LOW_SURROGATE((ptr)[1])) ? \
577+
((ptr) += 2,_Py_UNICODE_JOIN_SURROGATES((ptr)[-2], (ptr)[-1])) : \
578+
(Py_UCS4)*(ptr)++)
579+
#endif
580+
550581
#ifdef HAVE_WCHAR_H
551582

552583
#if (Py_UNICODE_SIZE == 2) && defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4)
@@ -3642,26 +3673,22 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
36423673
case 4: /* xmlcharrefreplace */
36433674
respos = str-PyString_AS_STRING(res);
36443675
/* determine replacement size (temporarily (mis)uses p) */
3645-
for (p = collstart, repsize = 0; p < collend; ++p) {
3646-
if (*p<10)
3676+
for (p = collstart, repsize = 0; p < collend;) {
3677+
Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend);
3678+
if (ch < 10)
36473679
repsize += 2+1+1;
3648-
else if (*p<100)
3680+
else if (ch < 100)
36493681
repsize += 2+2+1;
3650-
else if (*p<1000)
3682+
else if (ch < 1000)
36513683
repsize += 2+3+1;
3652-
else if (*p<10000)
3684+
else if (ch < 10000)
36533685
repsize += 2+4+1;
3654-
#ifndef Py_UNICODE_WIDE
3655-
else
3686+
else if (ch < 100000)
36563687
repsize += 2+5+1;
3657-
#else
3658-
else if (*p<100000)
3659-
repsize += 2+5+1;
3660-
else if (*p<1000000)
3688+
else if (ch < 1000000)
36613689
repsize += 2+6+1;
36623690
else
36633691
repsize += 2+7+1;
3664-
#endif
36653692
}
36663693
requiredsize = respos+repsize+(endp-collend);
36673694
if (requiredsize > ressize) {
@@ -3673,8 +3700,9 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
36733700
ressize = requiredsize;
36743701
}
36753702
/* generate replacement (temporarily (mis)uses p) */
3676-
for (p = collstart; p < collend; ++p) {
3677-
str += sprintf(str, "&#%d;", (int)*p);
3703+
for (p = collstart; p < collend;) {
3704+
Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend);
3705+
str += sprintf(str, "&#%d;", (int)ch);
36783706
}
36793707
p = collend;
36803708
break;
@@ -4649,11 +4677,20 @@ int charmap_encoding_error(
46494677
*inpos = collendpos;
46504678
break;
46514679
case 4: /* xmlcharrefreplace */
4652-
/* generate replacement (temporarily (mis)uses p) */
4653-
for (collpos = collstartpos; collpos < collendpos; ++collpos) {
4680+
/* generate replacement */
4681+
for (collpos = collstartpos; collpos < collendpos;) {
46544682
char buffer[2+29+1+1];
46554683
char *cp;
4656-
sprintf(buffer, "&#%d;", (int)p[collpos]);
4684+
Py_UCS4 ch = p[collpos++];
4685+
#ifndef Py_UNICODE_WIDE
4686+
if ((0xD800 <= ch && ch <= 0xDBFF) &&
4687+
(collpos < collendpos) &&
4688+
(0xDC00 <= p[collpos] && p[collpos] <= 0xDFFF)) {
4689+
ch = ((((ch & 0x03FF) << 10) |
4690+
((Py_UCS4)p[collpos++] & 0x03FF)) + 0x10000);
4691+
}
4692+
#endif
4693+
sprintf(buffer, "&#%d;", (int)ch);
46574694
for (cp = buffer; *cp; ++cp) {
46584695
x = charmapencode_output(*cp, mapping, res, respos);
46594696
if (x==enc_EXCEPTION)
@@ -5068,10 +5105,11 @@ PyObject *PyUnicode_TranslateCharmap(const Py_UNICODE *p,
50685105
break;
50695106
case 4: /* xmlcharrefreplace */
50705107
/* generate replacement (temporarily (mis)uses p) */
5071-
for (p = collstart; p < collend; ++p) {
5108+
for (p = collstart; p < collend;) {
50725109
char buffer[2+29+1+1];
50735110
char *cp;
5074-
sprintf(buffer, "&#%d;", (int)*p);
5111+
Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend);
5112+
sprintf(buffer, "&#%d;", (int)ch);
50755113
if (charmaptranslate_makespace(&res, &str,
50765114
(str-PyUnicode_AS_UNICODE(res))+strlen(buffer)+(endp-collend)))
50775115
goto onError;
@@ -5222,8 +5260,10 @@ int PyUnicode_EncodeDecimal(Py_UNICODE *s,
52225260
break;
52235261
case 4: /* xmlcharrefreplace */
52245262
/* generate replacement (temporarily (mis)uses p) */
5225-
for (p = collstart; p < collend; ++p)
5226-
output += sprintf(output, "&#%d;", (int)*p);
5263+
for (p = collstart; p < collend;) {
5264+
Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend);
5265+
output += sprintf(output, "&#%d;", ch);
5266+
}
52275267
p = collend;
52285268
break;
52295269
default:

‎Python/codecs.c‎

Lines changed: 36 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -556,6 +556,7 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc)
556556
PyObject *res;
557557
Py_UNICODE *p;
558558
Py_UNICODE *startp;
559+
Py_UNICODE *e;
559560
Py_UNICODE *outp;
560561
int ressize;
561562
if (PyUnicodeEncodeError_GetStart(exc, &start))
@@ -565,26 +566,31 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc)
565566
if (!(object = PyUnicodeEncodeError_GetObject(exc)))
566567
return NULL;
567568
startp = PyUnicode_AS_UNICODE(object);
568-
for (p = startp+start, ressize = 0; p < startp+end; ++p) {
569-
if (*p<10)
569+
e = startp + end;
570+
for (p = startp+start, ressize = 0; p < e;) {
571+
Py_UCS4 ch = *p++;
572+
#ifndef Py_UNICODE_WIDE
573+
if ((0xD800 <= ch && ch <= 0xDBFF) &&
574+
(p < e) &&
575+
(0xDC00 <= *p && *p <= 0xDFFF)) {
576+
ch = ((((ch & 0x03FF) << 10) |
577+
((Py_UCS4)*p++ & 0x03FF)) + 0x10000);
578+
}
579+
#endif
580+
if (ch < 10)
570581
ressize += 2+1+1;
571-
else if (*p<100)
582+
else if (ch < 100)
572583
ressize += 2+2+1;
573-
else if (*p<1000)
584+
else if (ch < 1000)
574585
ressize += 2+3+1;
575-
else if (*p<10000)
586+
else if (ch < 10000)
576587
ressize += 2+4+1;
577-
#ifndef Py_UNICODE_WIDE
578-
else
579-
ressize += 2+5+1;
580-
#else
581-
else if (*p<100000)
588+
else if (ch < 100000)
582589
ressize += 2+5+1;
583-
else if (*p<1000000)
590+
else if (ch < 1000000)
584591
ressize += 2+6+1;
585592
else
586593
ressize += 2+7+1;
587-
#endif
588594
}
589595
/* allocate replacement */
590596
res = PyUnicode_FromUnicode(NULL, ressize);
@@ -593,51 +599,51 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc)
593599
return NULL;
594600
}
595601
/* generate replacement */
596-
for (p = startp+start, outp = PyUnicode_AS_UNICODE(res);
597-
p < startp+end; ++p) {
598-
Py_UNICODE c = *p;
602+
for (p = startp+start, outp = PyUnicode_AS_UNICODE(res); p < e;) {
599603
int digits;
600604
int base;
605+
Py_UCS4 ch = *p++;
606+
#ifndef Py_UNICODE_WIDE
607+
if ((0xD800 <= ch && ch <= 0xDBFF) &&
608+
(p < startp+end) &&
609+
(0xDC00 <= *p && *p <= 0xDFFF)) {
610+
ch = ((((ch & 0x03FF) << 10) |
611+
((Py_UCS4)*p++ & 0x03FF)) + 0x10000);
612+
}
613+
#endif
601614
*outp++ = '&';
602615
*outp++ = '#';
603-
if (*p<10) {
616+
if (ch < 10) {
604617
digits = 1;
605618
base = 1;
606619
}
607-
else if (*p<100) {
620+
else if (ch < 100) {
608621
digits = 2;
609622
base = 10;
610623
}
611-
else if (*p<1000) {
624+
else if (ch < 1000) {
612625
digits = 3;
613626
base = 100;
614627
}
615-
else if (*p<10000) {
628+
else if (ch < 10000) {
616629
digits = 4;
617630
base = 1000;
618631
}
619-
#ifndef Py_UNICODE_WIDE
620-
else {
621-
digits = 5;
622-
base = 10000;
623-
}
624-
#else
625-
else if (*p<100000) {
632+
else if (ch < 100000) {
626633
digits = 5;
627634
base = 10000;
628635
}
629-
else if (*p<1000000) {
636+
else if (ch < 1000000) {
630637
digits = 6;
631638
base = 100000;
632639
}
633640
else {
634641
digits = 7;
635642
base = 1000000;
636643
}
637-
#endif
638644
while (digits-->0) {
639-
*outp++ = '0' + c/base;
640-
c %= base;
645+
*outp++ = '0' + ch/base;
646+
ch %= base;
641647
base /= 10;
642648
}
643649
*outp++ = ';';

0 commit comments

Comments
 (0)