@@ -547,6 +547,37 @@ PyObject *PyUnicode_FromString(const char *u)
547547 return PyUnicode_FromStringAndSize (u , size );
548548}
549549
550+ /* _Py_UNICODE_NEXT is a private macro used to retrieve the character pointed
551+ * by 'ptr', possibly combining surrogate pairs on narrow builds.
552+ * 'ptr' and 'end' must be Py_UNICODE*, with 'ptr' pointing at the character
553+ * that should be returned and 'end' pointing to the end of the buffer.
554+ * ('end' is used on narrow builds to detect a lone surrogate at the
555+ * end of the buffer that should be returned unchanged.)
556+ * The ptr and end arguments should be side-effect free and ptr must an lvalue.
557+ * The type of the returned char is always Py_UCS4.
558+ *
559+ * Note: the macro advances ptr to next char, so it might have side-effects
560+ * (especially if used with other macros).
561+ */
562+
563+ /* helper macros used by _Py_UNICODE_NEXT */
564+ #define _Py_UNICODE_IS_HIGH_SURROGATE (ch ) (0xD800 <= ch && ch <= 0xDBFF)
565+ #define _Py_UNICODE_IS_LOW_SURROGATE (ch ) (0xDC00 <= ch && ch <= 0xDFFF)
566+ /* Join two surrogate characters and return a single Py_UCS4 value. */
567+ #define _Py_UNICODE_JOIN_SURROGATES (high , low ) \
568+ (((((Py_UCS4)(high) & 0x03FF) << 10) | \
569+ ((Py_UCS4)(low) & 0x03FF)) + 0x10000)
570+
571+ #ifdef Py_UNICODE_WIDE
572+ #define _Py_UNICODE_NEXT (ptr , end ) *(ptr)++
573+ #else
574+ #define _Py_UNICODE_NEXT (ptr , end ) \
575+ (((_Py_UNICODE_IS_HIGH_SURROGATE(*(ptr)) && (ptr) < (end)) && \
576+ _Py_UNICODE_IS_LOW_SURROGATE((ptr)[1])) ? \
577+ ((ptr) += 2,_Py_UNICODE_JOIN_SURROGATES((ptr)[-2], (ptr)[-1])) : \
578+ (Py_UCS4)*(ptr)++)
579+ #endif
580+
550581#ifdef HAVE_WCHAR_H
551582
552583#if (Py_UNICODE_SIZE == 2 ) && defined(SIZEOF_WCHAR_T ) && (SIZEOF_WCHAR_T == 4 )
@@ -3642,26 +3673,22 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
36423673 case 4 : /* xmlcharrefreplace */
36433674 respos = str - PyString_AS_STRING (res );
36443675 /* determine replacement size (temporarily (mis)uses p) */
3645- for (p = collstart , repsize = 0 ; p < collend ; ++ p ) {
3646- if (* p < 10 )
3676+ for (p = collstart , repsize = 0 ; p < collend ;) {
3677+ Py_UCS4 ch = _Py_UNICODE_NEXT (p , collend );
3678+ if (ch < 10 )
36473679 repsize += 2 + 1 + 1 ;
3648- else if (* p < 100 )
3680+ else if (ch < 100 )
36493681 repsize += 2 + 2 + 1 ;
3650- else if (* p < 1000 )
3682+ else if (ch < 1000 )
36513683 repsize += 2 + 3 + 1 ;
3652- else if (* p < 10000 )
3684+ else if (ch < 10000 )
36533685 repsize += 2 + 4 + 1 ;
3654- #ifndef Py_UNICODE_WIDE
3655- else
3686+ else if (ch < 100000 )
36563687 repsize += 2 + 5 + 1 ;
3657- #else
3658- else if (* p < 100000 )
3659- repsize += 2 + 5 + 1 ;
3660- else if (* p < 1000000 )
3688+ else if (ch < 1000000 )
36613689 repsize += 2 + 6 + 1 ;
36623690 else
36633691 repsize += 2 + 7 + 1 ;
3664- #endif
36653692 }
36663693 requiredsize = respos + repsize + (endp - collend );
36673694 if (requiredsize > ressize ) {
@@ -3673,8 +3700,9 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
36733700 ressize = requiredsize ;
36743701 }
36753702 /* generate replacement (temporarily (mis)uses p) */
3676- for (p = collstart ; p < collend ; ++ p ) {
3677- str += sprintf (str , "&#%d;" , (int )* p );
3703+ for (p = collstart ; p < collend ;) {
3704+ Py_UCS4 ch = _Py_UNICODE_NEXT (p , collend );
3705+ str += sprintf (str , "&#%d;" , (int )ch );
36783706 }
36793707 p = collend ;
36803708 break ;
@@ -4649,11 +4677,20 @@ int charmap_encoding_error(
46494677 * inpos = collendpos ;
46504678 break ;
46514679 case 4 : /* xmlcharrefreplace */
4652- /* generate replacement (temporarily (mis)uses p) */
4653- for (collpos = collstartpos ; collpos < collendpos ; ++ collpos ) {
4680+ /* generate replacement */
4681+ for (collpos = collstartpos ; collpos < collendpos ;) {
46544682 char buffer [2 + 29 + 1 + 1 ];
46554683 char * cp ;
4656- sprintf (buffer , "&#%d;" , (int )p [collpos ]);
4684+ Py_UCS4 ch = p [collpos ++ ];
4685+ #ifndef Py_UNICODE_WIDE
4686+ if ((0xD800 <= ch && ch <= 0xDBFF ) &&
4687+ (collpos < collendpos ) &&
4688+ (0xDC00 <= p [collpos ] && p [collpos ] <= 0xDFFF )) {
4689+ ch = ((((ch & 0x03FF ) << 10 ) |
4690+ ((Py_UCS4 )p [collpos ++ ] & 0x03FF )) + 0x10000 );
4691+ }
4692+ #endif
4693+ sprintf (buffer , "&#%d;" , (int )ch );
46574694 for (cp = buffer ; * cp ; ++ cp ) {
46584695 x = charmapencode_output (* cp , mapping , res , respos );
46594696 if (x == enc_EXCEPTION )
@@ -5068,10 +5105,11 @@ PyObject *PyUnicode_TranslateCharmap(const Py_UNICODE *p,
50685105 break ;
50695106 case 4 : /* xmlcharrefreplace */
50705107 /* generate replacement (temporarily (mis)uses p) */
5071- for (p = collstart ; p < collend ; ++ p ) {
5108+ for (p = collstart ; p < collend ;) {
50725109 char buffer [2 + 29 + 1 + 1 ];
50735110 char * cp ;
5074- sprintf (buffer , "&#%d;" , (int )* p );
5111+ Py_UCS4 ch = _Py_UNICODE_NEXT (p , collend );
5112+ sprintf (buffer , "&#%d;" , (int )ch );
50755113 if (charmaptranslate_makespace (& res , & str ,
50765114 (str - PyUnicode_AS_UNICODE (res ))+ strlen (buffer )+ (endp - collend )))
50775115 goto onError ;
@@ -5222,8 +5260,10 @@ int PyUnicode_EncodeDecimal(Py_UNICODE *s,
52225260 break ;
52235261 case 4 : /* xmlcharrefreplace */
52245262 /* generate replacement (temporarily (mis)uses p) */
5225- for (p = collstart ; p < collend ; ++ p )
5226- output += sprintf (output , "&#%d;" , (int )* p );
5263+ for (p = collstart ; p < collend ;) {
5264+ Py_UCS4 ch = _Py_UNICODE_NEXT (p , collend );
5265+ output += sprintf (output , "&#%d;" , ch );
5266+ }
52275267 p = collend ;
52285268 break ;
52295269 default :
0 commit comments