#include "Python.h"

#include "unicodeobject.h"

#include "ucnhash.h"

#ifdef MS_WIN32

#include <windows.h>

#define MAX_UNICODE_FREELIST_SIZE 1024

#define KEEPALIVE_SIZE_LIMIT 9

#ifdef WORDS_BIGENDIAN

# define BYTEORDER_IS_BIG_ENDIAN

# define BYTEORDER_IS_LITTLE_ENDIAN

static PyUnicodeObject *unicode_empty;

static PyUnicodeObject *unicode_freelist;

static int unicode_freelist_size;

static char unicode_default_encoding[100];

int _PyUnicode_Resize(register PyUnicodeObject *unicode,

int length)

{

void *oldstr;

/* Shortcut if there's nothing much to do. */

if (unicode->length == length)

goto reset;

/* Resizing unicode_empty is not allowed. */

if (unicode == unicode_empty) {

PyErr_SetString(PyExc_SystemError,

"can't resize empty unicode object");

return -1;

}

/* We allocate one more byte to make sure the string is

oldstr = unicode->str;

PyMem_RESIZE(unicode->str, Py_UNICODE, length + 1);

if (!unicode->str) {

unicode->str = oldstr;

PyErr_NoMemory();

return -1;

}

unicode->str[length] = 0;

unicode->length = length;

reset:

/* Reset the object caches */

if (unicode->defenc) {

Py_DECREF(unicode->defenc);

unicode->defenc = NULL;

}

unicode->hash = -1;

return 0;

}

int PyUnicode_Resize(PyObject **unicode,

int length)

{

PyUnicodeObject *v;

if (unicode == NULL) {

PyErr_BadInternalCall();

return -1;

}

v = (PyUnicodeObject *)*unicode;

if (v == NULL || !PyUnicode_Check(v) || v->ob_refcnt != 1) {

PyErr_BadInternalCall();

return -1;

}

return _PyUnicode_Resize(v, length);

}

PyUnicodeObject *_PyUnicode_New(int length)

{

register PyUnicodeObject *unicode;

/* Optimization for empty strings */

if (length == 0 && unicode_empty != NULL) {

Py_INCREF(unicode_empty);

return unicode_empty;

}

/* Unicode freelist & memory allocation */

if (unicode_freelist) {

unicode = unicode_freelist;

unicode_freelist = *(PyUnicodeObject **)unicode;

unicode_freelist_size--;

if (unicode->str) {

/* Keep-Alive optimization: we only upsize the buffer,

if ((unicode->length < length) &&

_PyUnicode_Resize(unicode, length)) {

PyMem_DEL(unicode->str);

goto onError;

}

}

else {

unicode->str = PyMem_NEW(Py_UNICODE, length + 1);

}

PyObject_INIT(unicode, &PyUnicode_Type);

}

else {

unicode = PyObject_NEW(PyUnicodeObject, &PyUnicode_Type);

if (unicode == NULL)

return NULL;

unicode->str = PyMem_NEW(Py_UNICODE, length + 1);

}

if (!unicode->str) {

PyErr_NoMemory();

goto onError;

}

unicode->str[length] = 0;

unicode->length = length;

unicode->hash = -1;

unicode->defenc = NULL;

return unicode;

onError:

_Py_ForgetReference((PyObject *)unicode);

PyObject_DEL(unicode);

return NULL;

}

void _PyUnicode_Free(register PyUnicodeObject *unicode)

{

if (unicode_freelist_size < MAX_UNICODE_FREELIST_SIZE) {

/* Keep-Alive optimization */

if (unicode->length >= KEEPALIVE_SIZE_LIMIT) {

PyMem_DEL(unicode->str);

unicode->str = NULL;

unicode->length = 0;

}

if (unicode->defenc) {

Py_DECREF(unicode->defenc);

unicode->defenc = NULL;

}

/* Add to free list */

*(PyUnicodeObject **)unicode = unicode_freelist;

unicode_freelist = unicode;

unicode_freelist_size++;

}

else {

PyMem_DEL(unicode->str);

Py_XDECREF(unicode->defenc);

PyObject_DEL(unicode);

}

}

PyObject *PyUnicode_FromUnicode(const Py_UNICODE *u,

int size)

{

PyUnicodeObject *unicode;

unicode = _PyUnicode_New(size);

if (!unicode)

return NULL;

/* Copy the Unicode data into the new object */

if (u != NULL)

memcpy(unicode->str, u, size * sizeof(Py_UNICODE));

return (PyObject *)unicode;

}

#ifdef HAVE_WCHAR_H

PyObject *PyUnicode_FromWideChar(register const wchar_t *w,

int size)

{

PyUnicodeObject *unicode;

if (w == NULL) {

PyErr_BadInternalCall();

return NULL;

}

unicode = _PyUnicode_New(size);

if (!unicode)

return NULL;

/* Copy the wchar_t data into the new object */

#ifdef HAVE_USABLE_WCHAR_T

memcpy(unicode->str, w, size * sizeof(wchar_t));

#else

{

register Py_UNICODE *u;

register int i;

u = PyUnicode_AS_UNICODE(unicode);

for (i = size; i >= 0; i--)

*u++ = *w++;

}

return (PyObject *)unicode;

}

int PyUnicode_AsWideChar(PyUnicodeObject *unicode,

register wchar_t *w,

int size)

{

if (unicode == NULL) {

PyErr_BadInternalCall();

return -1;

}

if (size > PyUnicode_GET_SIZE(unicode))

size = PyUnicode_GET_SIZE(unicode);

#ifdef HAVE_USABLE_WCHAR_T

memcpy(w, unicode->str, size * sizeof(wchar_t));

{

register Py_UNICODE *u;

register int i;

u = PyUnicode_AS_UNICODE(unicode);

for (i = size; i >= 0; i--)

*w++ = *u++;

}

return size;

}

PyObject *PyUnicode_FromObject(register PyObject *obj)

{

return PyUnicode_FromEncodedObject(obj, NULL, "strict");

}

PyObject *PyUnicode_FromEncodedObject(register PyObject *obj,

const char *encoding,

const char *errors)

{

const char *s;

int len;

int owned = 0;

PyObject *v;

if (obj == NULL) {

PyErr_BadInternalCall();

return NULL;

}

/* Coerce object */

if (PyInstance_Check(obj)) {

PyObject *func;

func = PyObject_GetAttrString(obj, "__str__");

if (func == NULL) {

PyErr_SetString(PyExc_TypeError,

"coercing to Unicode: instance doesn't define __str__");

return NULL;

}

obj = PyEval_CallObject(func, NULL);

Py_DECREF(func);

if (obj == NULL)

return NULL;

owned = 1;

}

if (PyUnicode_Check(obj)) {

Py_INCREF(obj);

v = obj;

if (encoding) {

PyErr_SetString(PyExc_TypeError,

"decoding Unicode is not supported");

return NULL;

}

goto done;

}

else if (PyString_Check(obj)) {

s = PyString_AS_STRING(obj);

len = PyString_GET_SIZE(obj);

}

else if (PyObject_AsCharBuffer(obj, &s, &len)) {

/* Overwrite the error message with something more useful in

if (PyErr_ExceptionMatches(PyExc_TypeError))

PyErr_Format(PyExc_TypeError,

"coercing to Unicode: need string or buffer, "

"%.80s found",

obj->ob_type->tp_name);

goto onError;

}

/* Convert to Unicode */

if (len == 0) {

Py_INCREF(unicode_empty);

v = (PyObject *)unicode_empty;

}

else

v = PyUnicode_Decode(s, len, encoding, errors);

done:

if (owned) {

Py_DECREF(obj);

}

return v;

onError:

if (owned) {

Py_DECREF(obj);

}

return NULL;

}

PyObject *PyUnicode_Decode(const char *s,

int size,

const char *encoding,

const char *errors)

{

PyObject *buffer = NULL, *unicode;

if (encoding == NULL)

encoding = PyUnicode_GetDefaultEncoding();

/* Shortcuts for common default encodings */

if (strcmp(encoding, "utf-8") == 0)

return PyUnicode_DecodeUTF8(s, size, errors);

else if (strcmp(encoding, "latin-1") == 0)

return PyUnicode_DecodeLatin1(s, size, errors);

else if (strcmp(encoding, "ascii") == 0)

return PyUnicode_DecodeASCII(s, size, errors);

/* Decode via the codec registry */

buffer = PyBuffer_FromMemory((void *)s, size);

if (buffer == NULL)

goto onError;

unicode = PyCodec_Decode(buffer, encoding, errors);

if (unicode == NULL)

goto onError;

if (!PyUnicode_Check(unicode)) {

PyErr_Format(PyExc_TypeError,

"decoder did not return an unicode object (type=%.400s)",

unicode->ob_type->tp_name);

Py_DECREF(unicode);

goto onError;

}

Py_DECREF(buffer);

return unicode;

onError:

Py_XDECREF(buffer);

return NULL;

}

PyObject *PyUnicode_Encode(const Py_UNICODE *s,

int size,

const char *encoding,

const char *errors)

{

PyObject *v, *unicode;

unicode = PyUnicode_FromUnicode(s, size);

if (unicode == NULL)

return NULL;

v = PyUnicode_AsEncodedString(unicode, encoding, errors);

Py_DECREF(unicode);

return v;

}

PyObject *PyUnicode_AsEncodedString(PyObject *unicode,

const char *encoding,

const char *errors)

{

PyObject *v;

if (!PyUnicode_Check(unicode)) {

PyErr_BadArgument();

goto onError;

}

if (encoding == NULL)

encoding = PyUnicode_GetDefaultEncoding();

/* Shortcuts for common default encodings */

if (errors == NULL) {

if (strcmp(encoding, "utf-8") == 0)

return PyUnicode_AsUTF8String(unicode);

else if (strcmp(encoding, "latin-1") == 0)

return PyUnicode_AsLatin1String(unicode);

else if (strcmp(encoding, "ascii") == 0)

return PyUnicode_AsASCIIString(unicode);

}

/* Encode via the codec registry */

v = PyCodec_Encode(unicode, encoding, errors);

if (v == NULL)

goto onError;

/* XXX Should we really enforce this ? */

if (!PyString_Check(v)) {

PyErr_Format(PyExc_TypeError,

"encoder did not return a string object (type=%.400s)",

v->ob_type->tp_name);

Py_DECREF(v);

goto onError;

}

return v;

onError:

return NULL;

}

PyObject *_PyUnicode_AsDefaultEncodedString(PyObject *unicode,

const char *errors)

{

PyObject *v = ((PyUnicodeObject *)unicode)->defenc;

if (v)

return v;

v = PyUnicode_AsEncodedString(unicode, NULL, errors);

if (v && errors == NULL)

((PyUnicodeObject *)unicode)->defenc = v;

return v;

}

Py_UNICODE *PyUnicode_AsUnicode(PyObject *unicode)

{

if (!PyUnicode_Check(unicode)) {

PyErr_BadArgument();

goto onError;

}

return PyUnicode_AS_UNICODE(unicode);

onError:

return NULL;

}

int PyUnicode_GetSize(PyObject *unicode)

{

if (!PyUnicode_Check(unicode)) {

PyErr_BadArgument();

goto onError;

}

return PyUnicode_GET_SIZE(unicode);

onError:

return -1;

}

const char *PyUnicode_GetDefaultEncoding(void)

{

return unicode_default_encoding;

}

int PyUnicode_SetDefaultEncoding(const char *encoding)

{

PyObject *v;

/* Make sure the encoding is valid. As side effect, this also

v = _PyCodec_Lookup(encoding);

if (v == NULL)

goto onError;

Py_DECREF(v);

strncpy(unicode_default_encoding,

encoding,

sizeof(unicode_default_encoding));

return 0;

onError:

return -1;

}

static

char utf8_code_length[256] = {

/* Map UTF-8 encoded prefix byte to sequence length. zero means

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,

2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,

3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,

4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0

};

int utf8_decoding_error(const char **source,

Py_UNICODE **dest,

const char *errors,

const char *details)

{

if ((errors == NULL) ||

(strcmp(errors,"strict") == 0)) {

PyErr_Format(PyExc_UnicodeError,

"UTF-8 decoding error: %.400s",

details);

return -1;

}

else if (strcmp(errors,"ignore") == 0) {

(*source)++;

return 0;

}

else if (strcmp(errors,"replace") == 0) {

(*source)++;

**dest = Py_UNICODE_REPLACEMENT_CHARACTER;

(*dest)++;

return 0;

}

else {

PyErr_Format(PyExc_ValueError,

"UTF-8 decoding error; unknown error handling code: %.400s",

errors);

return -1;

}

}

PyObject *PyUnicode_DecodeUTF8(const char *s,

int size,

const char *errors)

{

int n;

const char *e;

PyUnicodeObject *unicode;

Py_UNICODE *p;

const char *errmsg = "";

/* Note: size will always be longer than the resulting Unicode

unicode = _PyUnicode_New(size);

if (!unicode)

return NULL;

if (size == 0)

return (PyObject *)unicode;

/* Unpack UTF-8 encoded data */

p = unicode->str;

e = s + size;

while (s < e) {

Py_UCS4 ch = (unsigned char)*s;

if (ch < 0x80) {

*p++ = (Py_UNICODE)ch;

s++;

continue;

}

n = utf8_code_length[ch];

if (s + n > e) {

errmsg = "unexpected end of data";

goto utf8Error;

}

switch (n) {

case 0:

errmsg = "unexpected code byte";

goto utf8Error;

break;

case 1:

errmsg = "internal error";

goto utf8Error;

break;

case 2:

if ((s[1] & 0xc0) != 0x80) {

errmsg = "invalid data";

goto utf8Error;

}

ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f);

if (ch < 0x80) {

errmsg = "illegal encoding";

goto utf8Error;

}

else

*p++ = (Py_UNICODE)ch;

break;

case 3:

if ((s[1] & 0xc0) != 0x80 ||

(s[2] & 0xc0) != 0x80) {

errmsg = "invalid data";

goto utf8Error;

}

ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);

if (ch < 0x800 || (ch >= 0xd800 && ch < 0xe000)) {

errmsg = "illegal encoding";

goto utf8Error;

}

else

*p++ = (Py_UNICODE)ch;

break;

case 4:

if ((s[1] & 0xc0) != 0x80 ||

(s[2] & 0xc0) != 0x80 ||

(s[3] & 0xc0) != 0x80) {

errmsg = "invalid data";

goto utf8Error;

}

ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) +

((s[2] & 0x3f) << 6) + (s[3] & 0x3f);

/* validate and convert to UTF-16 */

if ((ch < 0x10000) || /* minimum value allowed for 4

(ch > 0x10ffff)) { /* maximum value allowed for

errmsg = "illegal encoding";

goto utf8Error;

}

/* compute and append the two surrogates: */

/* translate from 10000..10FFFF to 0..FFFF */

ch -= 0x10000;

/* high surrogate = top 10 bits added to D800 */

*p++ = (Py_UNICODE)(0xD800 + (ch >> 10));

/* low surrogate = bottom 10 bits added to DC00 */

*p++ = (Py_UNICODE)(0xDC00 + (ch & ~0xFC00));

break;

default:

/* Other sizes are only needed for UCS-4 */

errmsg = "unsupported Unicode code range";

goto utf8Error;

break;

}

s += n;

continue;

utf8Error:

if (utf8_decoding_error(&s, &p, errors, errmsg))

goto onError;

}

/* Adjust length */

if (_PyUnicode_Resize(unicode, p - unicode->str))

goto onError;

return (PyObject *)unicode;

onError:

Py_DECREF(unicode);

return NULL;

}

#if 0

int utf8_encoding_error(const Py_UNICODE **source,

char **dest,

const char *errors,

const char *details)

{

if ((errors == NULL) ||

(strcmp(errors,"strict") == 0)) {

PyErr_Format(PyExc_UnicodeError,

"UTF-8 encoding error: %.400s",

details);

return -1;

}

else if (strcmp(errors,"ignore") == 0) {

return 0;

}

else if (strcmp(errors,"replace") == 0) {

**dest = '?';

(*dest)++;

return 0;

}

else {

PyErr_Format(PyExc_ValueError,

"UTF-8 encoding error; "

"unknown error handling code: %.400s",

errors);

return -1;

}

}

PyObject *PyUnicode_EncodeUTF8(const Py_UNICODE *s,

int size,

const char *errors)

{

PyObject *v;

char *p;

char *q;

Py_UCS4 ch2;

unsigned int cbAllocated = 3 * size;

unsigned int cbWritten = 0;

int i = 0;

v = PyString_FromStringAndSize(NULL, cbAllocated);

if (v == NULL)

return NULL;

if (size == 0)

return v;

p = q = PyString_AS_STRING(v);

while (i < size) {

Py_UCS4 ch = s[i++];

if (ch < 0x80) {

*p++ = (char) ch;

cbWritten++;

}

else if (ch < 0x0800) {

*p++ = 0xc0 | (ch >> 6);

*p++ = 0x80 | (ch & 0x3f);

cbWritten += 2;

}

else {

/* Check for high surrogate */

if (0xD800 <= ch && ch <= 0xDBFF) {

if (i != size) {

ch2 = s[i];

if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {

if (cbWritten >= (cbAllocated - 4)) {

/* Provide enough room for some more

cbAllocated += 4*10;

if (_PyString_Resize(&v, cbAllocated))

goto onError;

}

/* combine the two values */

ch = ((ch - 0xD800)<<10 | (ch2-0xDC00))+0x10000;

*p++ = (char)((ch >> 18) | 0xf0);

*p++ = (char)(0x80 | ((ch >> 12) & 0x3f));

i++;

cbWritten += 4;

}

}

}

else {

*p++ = (char)(0xe0 | (ch >> 12));

cbWritten += 3;

}

*p++ = (char)(0x80 | ((ch >> 6) & 0x3f));

*p++ = (char)(0x80 | (ch & 0x3f));

}

}

*p = '\0';

if (_PyString_Resize(&v, p - q))

goto onError;

return v;

onError:

return NULL;

}

PyObject *PyUnicode_AsUTF8String(PyObject *unicode)

{

if (!PyUnicode_Check(unicode)) {

PyErr_BadArgument();

return NULL;

}

return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),

PyUnicode_GET_SIZE(unicode),

NULL);

}

int utf16_decoding_error(const Py_UNICODE **source,

Py_UNICODE **dest,

const char *errors,

const char *details)

{

if ((errors == NULL) ||

(strcmp(errors,"strict") == 0)) {

PyErr_Format(PyExc_UnicodeError,

"UTF-16 decoding error: %.400s",

details);

return -1;

}

else if (strcmp(errors,"ignore") == 0) {

return 0;

}

else if (strcmp(errors,"replace") == 0) {

if (dest) {

**dest = Py_UNICODE_REPLACEMENT_CHARACTER;

(*dest)++;

}

return 0;

}

else {

PyErr_Format(PyExc_ValueError,

"UTF-16 decoding error; "

"unknown error handling code: %.400s",

errors);

return -1;

}

}

PyObject *PyUnicode_DecodeUTF16(const char *s,

int size,

const char *errors,

int *byteorder)

{

PyUnicodeObject *unicode;

Py_UNICODE *p;

const Py_UNICODE *q, *e;

int bo = 0;

const char *errmsg = "";

/* size should be an even number */

if (size % sizeof(Py_UNICODE) != 0) {

if (utf16_decoding_error(NULL, NULL, errors, "truncated data"))

return NULL;

/* The remaining input chars are ignored if we fall through

}

/* Note: size will always be longer than the resulting Unicode

unicode = _PyUnicode_New(size);

if (!unicode)

return NULL;

if (size == 0)

return (PyObject *)unicode;

/* Unpack UTF-16 encoded data */

p = unicode->str;

q = (Py_UNICODE *)s;

e = q + (size / sizeof(Py_UNICODE));

if (byteorder)

bo = *byteorder;

while (q < e) {

register Py_UNICODE ch = *q++;

/* Check for BOM marks (U+FEFF) in the input and adjust

#ifdef BYTEORDER_IS_LITTLE_ENDIAN

if (ch == 0xFEFF) {

bo = -1;

continue;

} else if (ch == 0xFFFE) {

bo = 1;

continue;

}

if (bo == 1)

ch = (ch >> 8) | (ch << 8);

#else

if (ch == 0xFEFF) {

bo = 1;

continue;

} else if (ch == 0xFFFE) {

bo = -1;

continue;

}

if (bo == -1)

ch = (ch >> 8) | (ch << 8);

if (ch < 0xD800 || ch > 0xDFFF) {

*p++ = ch;

continue;

}

/* UTF-16 code pair: */

if (q >= e) {

errmsg = "unexpected end of data";

goto utf16Error;

}

if (0xDC00 <= *q && *q <= 0xDFFF) {

q++;

if (0xD800 <= *q && *q <= 0xDBFF) {

/* This is valid data (a UTF-16 surrogate pair), but

errmsg = "code pairs are not supported";

goto utf16Error;

}

else

continue;

}