#include "Python.h"

#include "unicodeobject.h"

#include "ucnhash.h"

#ifdef MS_WIN32

#include <windows.h>

#define MAX_UNICODE_FREELIST_SIZE 1024

#define KEEPALIVE_SIZE_LIMIT 9

#ifdef WORDS_BIGENDIAN

# define BYTEORDER_IS_BIG_ENDIAN

# define BYTEORDER_IS_LITTLE_ENDIAN

static PyUnicodeObject *unicode_freelist;

static int unicode_freelist_size;

static PyUnicodeObject *unicode_empty;

static PyUnicodeObject *unicode_latin1[256];

static char unicode_default_encoding[100];

PyUnicode_GetMax(void)

{

#ifdef Py_UNICODE_WIDE

return 0x10FFFF;

/* This is actually an illegal character, so it should

return 0xFFFF;

}

int unicode_resize(register PyUnicodeObject *unicode,

int length)

{

void *oldstr;

/* Shortcut if there's nothing much to do. */

if (unicode->length == length)

goto reset;

/* Resizing shared object (unicode_empty or single character

if (unicode == unicode_empty ||

(unicode->length == 1 &&

/* MvL said unicode->str[] may be signed. Python generally assumes

(unsigned int)unicode->str[0] < 256U &&

unicode_latin1[unicode->str[0]] == unicode)) {

PyErr_SetString(PyExc_SystemError,

"can't resize shared unicode objects");

return -1;

}

/* We allocate one more byte to make sure the string is

oldstr = unicode->str;

PyMem_RESIZE(unicode->str, Py_UNICODE, length + 1);

if (!unicode->str) {

unicode->str = oldstr;

PyErr_NoMemory();

return -1;

}

unicode->str[length] = 0;

unicode->length = length;

reset:

/* Reset the object caches */

if (unicode->defenc) {

Py_DECREF(unicode->defenc);

unicode->defenc = NULL;

}

unicode->hash = -1;

return 0;

}

PyUnicodeObject *_PyUnicode_New(int length)

{

register PyUnicodeObject *unicode;

/* Optimization for empty strings */

if (length == 0 && unicode_empty != NULL) {

Py_INCREF(unicode_empty);

return unicode_empty;

}

/* Unicode freelist & memory allocation */

if (unicode_freelist) {

unicode = unicode_freelist;

unicode_freelist = *(PyUnicodeObject **)unicode;

unicode_freelist_size--;

if (unicode->str) {

/* Keep-Alive optimization: we only upsize the buffer,

if ((unicode->length < length) &&

unicode_resize(unicode, length)) {

PyMem_DEL(unicode->str);

goto onError;

}

}

else {

unicode->str = PyMem_NEW(Py_UNICODE, length + 1);

}

PyObject_INIT(unicode, &PyUnicode_Type);

}

else {

unicode = PyObject_NEW(PyUnicodeObject, &PyUnicode_Type);

if (unicode == NULL)

return NULL;

unicode->str = PyMem_NEW(Py_UNICODE, length + 1);

}

if (!unicode->str) {

PyErr_NoMemory();

goto onError;

}

/* Initialize the first element to guard against cases where

unicode->str[0] = 0;

unicode->str[length] = 0;

unicode->length = length;

unicode->hash = -1;

unicode->defenc = NULL;

return unicode;

onError:

_Py_ForgetReference((PyObject *)unicode);

PyObject_DEL(unicode);

return NULL;

}

void unicode_dealloc(register PyUnicodeObject *unicode)

{

if (PyUnicode_CheckExact(unicode) &&

unicode_freelist_size < MAX_UNICODE_FREELIST_SIZE) {

/* Keep-Alive optimization */

if (unicode->length >= KEEPALIVE_SIZE_LIMIT) {

PyMem_DEL(unicode->str);

unicode->str = NULL;

unicode->length = 0;

}

if (unicode->defenc) {

Py_DECREF(unicode->defenc);

unicode->defenc = NULL;

}

/* Add to free list */

*(PyUnicodeObject **)unicode = unicode_freelist;

unicode_freelist = unicode;

unicode_freelist_size++;

}

else {

PyMem_DEL(unicode->str);

Py_XDECREF(unicode->defenc);

unicode->ob_type->tp_free((PyObject *)unicode);

}

}

int PyUnicode_Resize(PyObject **unicode,

int length)

{

register PyUnicodeObject *v;

/* Argument checks */

if (unicode == NULL) {

PyErr_BadInternalCall();

return -1;

}

v = (PyUnicodeObject *)*unicode;

if (v == NULL || !PyUnicode_Check(v) || v->ob_refcnt != 1 || length < 0) {

PyErr_BadInternalCall();

return -1;

}

/* Resizing unicode_empty and single character objects is not

if (v->length != length &&

(v == unicode_empty || v->length == 1)) {

PyUnicodeObject *w = _PyUnicode_New(length);

if (w == NULL)

return -1;

Py_UNICODE_COPY(w->str, v->str,

length < v->length ? length : v->length);

*unicode = (PyObject *)w;

return 0;

}

/* Note that we don't have to modify *unicode for unshared Unicode

return unicode_resize(v, length);

}

#define _PyUnicode_Resize(unicodevar, length) \

PyUnicode_Resize(((PyObject **)(unicodevar)), length)

PyObject *PyUnicode_FromUnicode(const Py_UNICODE *u,

int size)

{

PyUnicodeObject *unicode;

/* If the Unicode data is known at construction time, we can apply

if (u != NULL) {

/* Optimization for empty strings */

if (size == 0 && unicode_empty != NULL) {

Py_INCREF(unicode_empty);

return (PyObject *)unicode_empty;

}

/* Single character Unicode objects in the Latin-1 range are

if (size == 1 && *u < 256) {

unicode = unicode_latin1[*u];

if (!unicode) {

unicode = _PyUnicode_New(1);

if (!unicode)

return NULL;

unicode->str[0] = *u;

unicode_latin1[*u] = unicode;

}

Py_INCREF(unicode);

return (PyObject *)unicode;

}

}

unicode = _PyUnicode_New(size);

if (!unicode)

return NULL;

/* Copy the Unicode data into the new object */

if (u != NULL)

Py_UNICODE_COPY(unicode->str, u, size);

return (PyObject *)unicode;

}

#ifdef HAVE_WCHAR_H

PyObject *PyUnicode_FromWideChar(register const wchar_t *w,

int size)

{

PyUnicodeObject *unicode;

if (w == NULL) {

PyErr_BadInternalCall();

return NULL;

}

unicode = _PyUnicode_New(size);

if (!unicode)

return NULL;

/* Copy the wchar_t data into the new object */

#ifdef HAVE_USABLE_WCHAR_T

memcpy(unicode->str, w, size * sizeof(wchar_t));

#else

{

register Py_UNICODE *u;

register int i;

u = PyUnicode_AS_UNICODE(unicode);

for (i = size; i >= 0; i--)

*u++ = *w++;

}

return (PyObject *)unicode;

}

int PyUnicode_AsWideChar(PyUnicodeObject *unicode,

register wchar_t *w,

int size)

{

if (unicode == NULL) {

PyErr_BadInternalCall();

return -1;

}

if (size > PyUnicode_GET_SIZE(unicode))

size = PyUnicode_GET_SIZE(unicode);

#ifdef HAVE_USABLE_WCHAR_T

memcpy(w, unicode->str, size * sizeof(wchar_t));

{

register Py_UNICODE *u;

register int i;

u = PyUnicode_AS_UNICODE(unicode);

for (i = size; i >= 0; i--)

*w++ = *u++;

}

return size;

}

PyObject *PyUnicode_FromOrdinal(int ordinal)

{

Py_UNICODE s[2];

#ifdef Py_UNICODE_WIDE

if (ordinal < 0 || ordinal > 0x10ffff) {

PyErr_SetString(PyExc_ValueError,

"unichr() arg not in range(0x110000) "

"(wide Python build)");

return NULL;

}

if (ordinal < 0 || ordinal > 0xffff) {

PyErr_SetString(PyExc_ValueError,

"unichr() arg not in range(0x10000) "

"(narrow Python build)");

return NULL;

}

if (ordinal <= 0xffff) {

/* UCS-2 character */

s[0] = (Py_UNICODE) ordinal;

return PyUnicode_FromUnicode(s, 1);

}

else {

#ifndef Py_UNICODE_WIDE

/* UCS-4 character. store as two surrogate characters */

ordinal -= 0x10000L;

s[0] = 0xD800 + (Py_UNICODE) (ordinal >> 10);

s[1] = 0xDC00 + (Py_UNICODE) (ordinal & 0x03FF);

return PyUnicode_FromUnicode(s, 2);

s[0] = (Py_UNICODE)ordinal;

return PyUnicode_FromUnicode(s, 1);

}

}

PyObject *PyUnicode_FromObject(register PyObject *obj)

{

/* XXX Perhaps we should make this API an alias of

if (PyUnicode_CheckExact(obj)) {

Py_INCREF(obj);

return obj;

}

if (PyUnicode_Check(obj)) {

/* For a Unicode subtype that's not a Unicode object,

return PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(obj),

PyUnicode_GET_SIZE(obj));

}

return PyUnicode_FromEncodedObject(obj, NULL, "strict");

}

PyObject *PyUnicode_FromEncodedObject(register PyObject *obj,

const char *encoding,

const char *errors)

{

const char *s = NULL;

int len;

int owned = 0;

PyObject *v;

if (obj == NULL) {

PyErr_BadInternalCall();

return NULL;

}

#if 0

/* For b/w compatibility we also accept Unicode objects provided

if (PyUnicode_Check(obj)) {

if (encoding) {

PyErr_SetString(PyExc_TypeError,

"decoding Unicode is not supported");

return NULL;

}

return PyObject_Unicode(obj);

}

if (PyUnicode_Check(obj)) {

PyErr_SetString(PyExc_TypeError,

"decoding Unicode is not supported");

return NULL;

}

/* Coerce object */

if (PyString_Check(obj)) {

s = PyString_AS_STRING(obj);

len = PyString_GET_SIZE(obj);

}

else if (PyObject_AsCharBuffer(obj, &s, &len)) {

/* Overwrite the error message with something more useful in

if (PyErr_ExceptionMatches(PyExc_TypeError))

PyErr_Format(PyExc_TypeError,

"coercing to Unicode: need string or buffer, "

"%.80s found",

obj->ob_type->tp_name);

goto onError;

}

/* Convert to Unicode */

if (len == 0) {

Py_INCREF(unicode_empty);

v = (PyObject *)unicode_empty;

}

else

v = PyUnicode_Decode(s, len, encoding, errors);

if (owned) {

Py_DECREF(obj);

}

return v;

onError:

if (owned) {

Py_DECREF(obj);

}

return NULL;

}

PyObject *PyUnicode_Decode(const char *s,

int size,

const char *encoding,

const char *errors)

{

PyObject *buffer = NULL, *unicode;

if (encoding == NULL)

encoding = PyUnicode_GetDefaultEncoding();

/* Shortcuts for common default encodings */

if (strcmp(encoding, "utf-8") == 0)

return PyUnicode_DecodeUTF8(s, size, errors);

else if (strcmp(encoding, "latin-1") == 0)

return PyUnicode_DecodeLatin1(s, size, errors);

else if (strcmp(encoding, "ascii") == 0)

return PyUnicode_DecodeASCII(s, size, errors);

/* Decode via the codec registry */

buffer = PyBuffer_FromMemory((void *)s, size);

if (buffer == NULL)

goto onError;

unicode = PyCodec_Decode(buffer, encoding, errors);

if (unicode == NULL)

goto onError;

if (!PyUnicode_Check(unicode)) {

PyErr_Format(PyExc_TypeError,

"decoder did not return an unicode object (type=%.400s)",

unicode->ob_type->tp_name);

Py_DECREF(unicode);

goto onError;

}

Py_DECREF(buffer);

return unicode;

onError:

Py_XDECREF(buffer);

return NULL;

}

PyObject *PyUnicode_Encode(const Py_UNICODE *s,

int size,

const char *encoding,

const char *errors)

{

PyObject *v, *unicode;

unicode = PyUnicode_FromUnicode(s, size);

if (unicode == NULL)

return NULL;

v = PyUnicode_AsEncodedString(unicode, encoding, errors);

Py_DECREF(unicode);

return v;

}

PyObject *PyUnicode_AsEncodedString(PyObject *unicode,

const char *encoding,

const char *errors)

{

PyObject *v;

if (!PyUnicode_Check(unicode)) {

PyErr_BadArgument();

goto onError;

}

if (encoding == NULL)

encoding = PyUnicode_GetDefaultEncoding();

/* Shortcuts for common default encodings */

if (errors == NULL) {

if (strcmp(encoding, "utf-8") == 0)

return PyUnicode_AsUTF8String(unicode);

else if (strcmp(encoding, "latin-1") == 0)

return PyUnicode_AsLatin1String(unicode);

else if (strcmp(encoding, "ascii") == 0)

return PyUnicode_AsASCIIString(unicode);

}

/* Encode via the codec registry */

v = PyCodec_Encode(unicode, encoding, errors);

if (v == NULL)

goto onError;

/* XXX Should we really enforce this ? */

if (!PyString_Check(v)) {

PyErr_Format(PyExc_TypeError,

"encoder did not return a string object (type=%.400s)",

v->ob_type->tp_name);

Py_DECREF(v);

goto onError;

}

return v;

onError:

return NULL;

}

PyObject *_PyUnicode_AsDefaultEncodedString(PyObject *unicode,

const char *errors)

{

PyObject *v = ((PyUnicodeObject *)unicode)->defenc;

if (v)

return v;

v = PyUnicode_AsEncodedString(unicode, NULL, errors);

if (v && errors == NULL)

((PyUnicodeObject *)unicode)->defenc = v;

return v;

}

Py_UNICODE *PyUnicode_AsUnicode(PyObject *unicode)

{

if (!PyUnicode_Check(unicode)) {

PyErr_BadArgument();

goto onError;

}

return PyUnicode_AS_UNICODE(unicode);

onError:

return NULL;

}

int PyUnicode_GetSize(PyObject *unicode)

{

if (!PyUnicode_Check(unicode)) {

PyErr_BadArgument();

goto onError;

}

return PyUnicode_GET_SIZE(unicode);

onError:

return -1;

}

const char *PyUnicode_GetDefaultEncoding(void)

{

return unicode_default_encoding;

}

int PyUnicode_SetDefaultEncoding(const char *encoding)

{

PyObject *v;

/* Make sure the encoding is valid. As side effect, this also

v = _PyCodec_Lookup(encoding);

if (v == NULL)

goto onError;

Py_DECREF(v);

strncpy(unicode_default_encoding,

encoding,

sizeof(unicode_default_encoding));

return 0;

onError:

return -1;

}

static

char utf7_special[128] = {

/* indicate whether a UTF-7 character is special i.e. cannot be directly

1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0,

3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3,

3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1,

};

#define SPECIAL(c, encodeO, encodeWS) \

(((c)>127 || utf7_special[(c)] == 1) || \

(encodeWS && (utf7_special[(c)] == 2)) || \

(encodeO && (utf7_special[(c)] == 3)))

#define B64(n) ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(n) & 0x3f])

#define B64CHAR(c) (isalnum(c) || (c) == '+' || (c) == '/')

#define UB64(c) ((c) == '+' ? 62 : (c) == '/' ? 63 : (c) >= 'a' ? \

(c) - 71 : (c) >= 'A' ? (c) - 65 : (c) + 4)

#define ENCODE(out, ch, bits) \

while (bits >= 6) { \

*out++ = B64(ch >> (bits-6)); \

bits -= 6; \

}

#define DECODE(out, ch, bits, surrogate) \

while (bits >= 16) { \

Py_UNICODE outCh = (Py_UNICODE) ((ch >> (bits-16)) & 0xffff); \

bits -= 16; \

if (surrogate) { \

/* We have already generated an error for the high surrogate

so let's not bother seeing if the low surrogate is correct or not */\

surrogate = 0; \

} else if (0xDC00 <= outCh && outCh <= 0xDFFF) { \

/* This is a surrogate pair. Unfortunately we can't represent \

it in a 16-bit character */ \

surrogate = 1; \

errmsg = "code pairs are not supported"; \

goto utf7Error; \

} else { \

*out++ = outCh; \

} \

} \

int utf7_decoding_error(Py_UNICODE **dest,

const char *errors,

const char *details)

{

if ((errors == NULL) ||

(strcmp(errors,"strict") == 0)) {

PyErr_Format(PyExc_UnicodeError,

"UTF-7 decoding error: %.400s",

details);

return -1;

}

else if (strcmp(errors,"ignore") == 0) {

return 0;

}

else if (strcmp(errors,"replace") == 0) {

if (dest != NULL) {

**dest = Py_UNICODE_REPLACEMENT_CHARACTER;

(*dest)++;

}

return 0;

}

else {

PyErr_Format(PyExc_ValueError,

"UTF-7 decoding error; unknown error handling code: %.400s",

errors);

return -1;

}

}

PyObject *PyUnicode_DecodeUTF7(const char *s,

int size,

const char *errors)

{

const char *e;

PyUnicodeObject *unicode;

Py_UNICODE *p;

const char *errmsg = "";

int inShift = 0;

unsigned int bitsleft = 0;

unsigned long charsleft = 0;

int surrogate = 0;

unicode = _PyUnicode_New(size);

if (!unicode)

return NULL;

if (size == 0)

return (PyObject *)unicode;

p = unicode->str;

e = s + size;

while (s < e) {

Py_UNICODE ch = *s;

if (inShift) {

if ((ch == '-') || !B64CHAR(ch)) {

inShift = 0;

s++;

/* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);

if (bitsleft >= 6) {

/* The shift sequence has a partial character in it. If

errmsg = "partial character in shift sequence";

goto utf7Error;

}

/* According to RFC2152 the remaining bits should be zero. We

/* On x86, a << b == a << (b%32) so make sure that bitsleft != 0 */

if (bitsleft && charsleft << (sizeof(charsleft) * 8 - bitsleft)) {

errmsg = "non-zero padding bits in shift sequence";

goto utf7Error;

}

if (ch == '-') {

if ((s < e) && (*(s) == '-')) {

*p++ = '-';

inShift = 1;

}

} else if (SPECIAL(ch,0,0)) {

errmsg = "unexpected special character";

goto utf7Error;

} else {

*p++ = ch;

}

} else {

charsleft = (charsleft << 6) | UB64(ch);

bitsleft += 6;

s++;

/* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);

}

}

else if ( ch == '+' ) {

s++;

if (s < e && *s == '-') {

s++;

*p++ = '+';

} else

{

inShift = 1;

bitsleft = 0;

}

}

else if (SPECIAL(ch,0,0)) {

errmsg = "unexpected special character";

s++;

goto utf7Error;

}

else {

*p++ = ch;

s++;

}

continue;

utf7Error:

if (utf7_decoding_error(&p, errors, errmsg))

goto onError;

}

if (inShift) {

if (utf7_decoding_error(&p, errors, "unterminated shift sequence"))

goto onError;

}

if (_PyUnicode_Resize(&unicode, p - unicode->str))

goto onError;

return (PyObject *)unicode;

onError:

Py_DECREF(unicode);

return NULL;

}

PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s,

const char *errors)

{

PyObject *v;

/* It might be possible to tighten this worst case */

unsigned int cbAllocated = 5 * size;

int inShift = 0;

int i = 0;

unsigned int bitsleft = 0;

unsigned long charsleft = 0;

char * out;

char * start;

if (size == 0)

return PyString_FromStringAndSize(NULL, 0);

v = PyString_FromStringAndSize(NULL, cbAllocated);

if (v == NULL)

return NULL;

start = out = PyString_AS_STRING(v);

for (;i < size; ++i) {

Py_UNICODE ch = s[i];

if (!inShift) {

if (ch == '+') {

*out++ = '+';

*out++ = '-';

} else if (SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {

charsleft = ch;

bitsleft = 16;

*out++ = '+';

/* out, charsleft, bitsleft = */ ENCODE(out, charsleft, bitsleft);

inShift = bitsleft > 0;

} else {

*out++ = (char) ch;

}

} else {

if (!SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {

*out++ = B64(charsleft << (6-bitsleft));

charsleft = 0;

bitsleft = 0;

/* Characters not in the BASE64 set implicitly unshift the sequence

if (B64CHAR(ch) || ch == '-') {

*out++ = '-';

}

inShift = 0;

*out++ = (char) ch;

} else {

bitsleft += 16;

charsleft = (charsleft << 16) | ch;

/* out, charsleft, bitsleft = */ ENCODE(out, charsleft, bitsleft);

/* If the next character is special then we dont' need to terminate

if (bitsleft == 0) {

if (i + 1 < size) {

Py_UNICODE ch2 = s[i+1];

if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)) {

} else if (B64CHAR(ch2) || ch2 == '-') {

*out++ = '-';

inShift = 0;

} else {

inShift = 0;

}

}

else {

*out++ = '-';

inShift = 0;

}

}

}

}

}

if (bitsleft) {

*out++= B64(charsleft << (6-bitsleft) );

*out++ = '-';

}

_PyString_Resize(&v, out - start);

return v;

}

#undef SPECIAL

#undef B64

#undef B64CHAR

#undef UB64

#undef ENCODE

#undef DECODE

static

char utf8_code_length[256] = {

/* Map UTF-8 encoded prefix byte to sequence length. zero means

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,