#define PY_SSIZE_T_CLEAN

#include "Python.h"

#include <ctype.h>

#ifdef COUNT_ALLOCS

int null_strings, one_strings;

static PyStringObject *characters[UCHAR_MAX + 1];

static PyStringObject *nullstring;

static PyObject *interned;

PyObject *

PyString_FromStringAndSize(const char *str, Py_ssize_t size)

{

register PyStringObject *op;

if (size < 0) {

PyErr_SetString(PyExc_SystemError,

"Negative size passed to PyString_FromStringAndSize");

return NULL;

}

if (size == 0 && (op = nullstring) != NULL) {

#ifdef COUNT_ALLOCS

null_strings++;

Py_INCREF(op);

return (PyObject *)op;

}

if (size == 1 && str != NULL &&

(op = characters[*str & UCHAR_MAX]) != NULL)

{

#ifdef COUNT_ALLOCS

one_strings++;

Py_INCREF(op);

return (PyObject *)op;

}

if (size > PY_SSIZE_T_MAX - sizeof(PyStringObject)) {

PyErr_SetString(PyExc_OverflowError, "string is too large");

return NULL;

}

/* Inline PyObject_NewVar */

op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size);

if (op == NULL)

return PyErr_NoMemory();

PyObject_INIT_VAR(op, &PyString_Type, size);

op->ob_shash = -1;

op->ob_sstate = SSTATE_NOT_INTERNED;

if (str != NULL)

Py_MEMCPY(op->ob_sval, str, size);

op->ob_sval[size] = '\0';

/* share short strings */

if (size == 0) {

PyObject *t = (PyObject *)op;

PyString_InternInPlace(&t);

op = (PyStringObject *)t;

nullstring = op;

Py_INCREF(op);

} else if (size == 1 && str != NULL) {

PyObject *t = (PyObject *)op;

PyString_InternInPlace(&t);

op = (PyStringObject *)t;

characters[*str & UCHAR_MAX] = op;

Py_INCREF(op);

}

return (PyObject *) op;

}

PyObject *

PyString_FromString(const char *str)

{

register size_t size;

register PyStringObject *op;

assert(str != NULL);

size = strlen(str);

if (size > PY_SSIZE_T_MAX - sizeof(PyStringObject)) {

PyErr_SetString(PyExc_OverflowError,

"string is too long for a Python string");

return NULL;

}

if (size == 0 && (op = nullstring) != NULL) {

#ifdef COUNT_ALLOCS

null_strings++;

Py_INCREF(op);

return (PyObject *)op;

}

if (size == 1 && (op = characters[*str & UCHAR_MAX]) != NULL) {

#ifdef COUNT_ALLOCS

one_strings++;

Py_INCREF(op);

return (PyObject *)op;

}

/* Inline PyObject_NewVar */

op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size);

if (op == NULL)

return PyErr_NoMemory();

PyObject_INIT_VAR(op, &PyString_Type, size);

op->ob_shash = -1;

op->ob_sstate = SSTATE_NOT_INTERNED;

Py_MEMCPY(op->ob_sval, str, size+1);

/* share short strings */

if (size == 0) {

PyObject *t = (PyObject *)op;

PyString_InternInPlace(&t);

op = (PyStringObject *)t;

nullstring = op;

Py_INCREF(op);

} else if (size == 1) {

PyObject *t = (PyObject *)op;

PyString_InternInPlace(&t);

op = (PyStringObject *)t;

characters[*str & UCHAR_MAX] = op;

Py_INCREF(op);

}

return (PyObject *) op;

}

PyObject *

PyString_FromFormatV(const char *format, va_list vargs)

{

va_list count;

Py_ssize_t n = 0;

const char* f;

char *s;

PyObject* string;

#ifdef VA_LIST_IS_ARRAY

Py_MEMCPY(count, vargs, sizeof(va_list));

#ifdef __va_copy

__va_copy(count, vargs);

count = vargs;

/* step 1: figure out how large a buffer we need */

for (f = format; *f; f++) {

if (*f == '%') {

const char* p = f;

while (*++f && *f != '%' && !isalpha(Py_CHARMASK(*f)))

;

/* skip the 'l' or 'z' in {%ld, %zd, %lu, %zu} since

if ((*f == 'l' || *f == 'z') &&

(f[1] == 'd' || f[1] == 'u'))

++f;

switch (*f) {

case 'c':

(void)va_arg(count, int);

/* fall through... */

case '%':

n++;

break;

case 'd': case 'u': case 'i': case 'x':

(void) va_arg(count, int);

/* 20 bytes is enough to hold a 64-bit

n += 20;

break;

case 's':

s = va_arg(count, char*);

n += strlen(s);

break;

case 'p':

(void) va_arg(count, int);

/* maximum 64-bit pointer representation:

n += 19;

break;

default:

/* if we stumble upon an unknown

n += strlen(p);

goto expand;

}

} else

n++;

}

expand:

/* step 2: fill the buffer */

/* Since we've analyzed how much space we need for the worst case,

string = PyString_FromStringAndSize(NULL, n);

if (!string)

return NULL;

s = PyString_AsString(string);

for (f = format; *f; f++) {

if (*f == '%') {

const char* p = f++;

Py_ssize_t i;

int longflag = 0;

int size_tflag = 0;

/* parse the width.precision part (we're only

n = 0;

while (isdigit(Py_CHARMASK(*f)))

n = (n*10) + *f++ - '0';

if (*f == '.') {

f++;

n = 0;

while (isdigit(Py_CHARMASK(*f)))

n = (n*10) + *f++ - '0';

}

while (*f && *f != '%' && !isalpha(Py_CHARMASK(*f)))

f++;

/* handle the long flag, but only for %ld and %lu.

if (*f == 'l' && (f[1] == 'd' || f[1] == 'u')) {

longflag = 1;

++f;

}

/* handle the size_t flag. */

if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) {

size_tflag = 1;

++f;

}

switch (*f) {

case 'c':

*s++ = va_arg(vargs, int);

break;

case 'd':

if (longflag)

sprintf(s, "%ld", va_arg(vargs, long));

else if (size_tflag)

sprintf(s, "%" PY_FORMAT_SIZE_T "d",

va_arg(vargs, Py_ssize_t));

else

sprintf(s, "%d", va_arg(vargs, int));

s += strlen(s);

break;

case 'u':

if (longflag)

sprintf(s, "%lu",

va_arg(vargs, unsigned long));

else if (size_tflag)

sprintf(s, "%" PY_FORMAT_SIZE_T "u",

va_arg(vargs, size_t));

else

sprintf(s, "%u",

va_arg(vargs, unsigned int));

s += strlen(s);

break;

case 'i':

sprintf(s, "%i", va_arg(vargs, int));

s += strlen(s);

break;

case 'x':

sprintf(s, "%x", va_arg(vargs, int));

s += strlen(s);

break;

case 's':

p = va_arg(vargs, char*);

i = strlen(p);

if (n > 0 && i > n)

i = n;

Py_MEMCPY(s, p, i);

s += i;

break;

case 'p':

sprintf(s, "%p", va_arg(vargs, void*));

/* %p is ill-defined: ensure leading 0x. */

if (s[1] == 'X')

s[1] = 'x';

else if (s[1] != 'x') {

memmove(s+2, s, strlen(s)+1);

s[0] = '0';

s[1] = 'x';

}

s += strlen(s);

break;

case '%':

*s++ = '%';

break;

default:

strcpy(s, p);

s += strlen(s);

goto end;

}

} else

*s++ = *f;

}

end:

_PyString_Resize(&string, s - PyString_AS_STRING(string));

return string;

}

PyObject *

PyString_FromFormat(const char *format, ...)

{

PyObject* ret;

va_list vargs;

#ifdef HAVE_STDARG_PROTOTYPES

va_start(vargs, format);

va_start(vargs);

ret = PyString_FromFormatV(format, vargs);

va_end(vargs);

return ret;

}

PyObject *PyString_Decode(const char *s,

Py_ssize_t size,

const char *encoding,

const char *errors)

{

PyObject *v, *str;

str = PyString_FromStringAndSize(s, size);

if (str == NULL)

return NULL;

v = PyString_AsDecodedString(str, encoding, errors);

Py_DECREF(str);

return v;

}

PyObject *PyString_AsDecodedObject(PyObject *str,

const char *encoding,

const char *errors)

{

PyObject *v;

if (!PyString_Check(str)) {

PyErr_BadArgument();

goto onError;

}

if (encoding == NULL) {

#ifdef Py_USING_UNICODE

encoding = PyUnicode_GetDefaultEncoding();

PyErr_SetString(PyExc_ValueError, "no encoding specified");

goto onError;

}

/* Decode via the codec registry */

v = PyCodec_Decode(str, encoding, errors);

if (v == NULL)

goto onError;

return v;

onError:

return NULL;

}

PyObject *PyString_AsDecodedString(PyObject *str,

const char *encoding,

const char *errors)

{

PyObject *v;

v = PyString_AsDecodedObject(str, encoding, errors);

if (v == NULL)

goto onError;

#ifdef Py_USING_UNICODE

/* Convert Unicode to a string using the default encoding */

if (PyUnicode_Check(v)) {

PyObject *temp = v;

v = PyUnicode_AsEncodedString(v, NULL, NULL);

Py_DECREF(temp);

if (v == NULL)

goto onError;

}

if (!PyString_Check(v)) {

PyErr_Format(PyExc_TypeError,

"decoder did not return a string object (type=%.400s)",

Py_TYPE(v)->tp_name);

Py_DECREF(v);

goto onError;

}

return v;

onError:

return NULL;

}

PyObject *PyString_Encode(const char *s,

Py_ssize_t size,

const char *encoding,

const char *errors)

{

PyObject *v, *str;

str = PyString_FromStringAndSize(s, size);

if (str == NULL)

return NULL;

v = PyString_AsEncodedString(str, encoding, errors);

Py_DECREF(str);

return v;

}

PyObject *PyString_AsEncodedObject(PyObject *str,

const char *encoding,

const char *errors)

{

PyObject *v;

if (!PyString_Check(str)) {

PyErr_BadArgument();

goto onError;

}

if (encoding == NULL) {

#ifdef Py_USING_UNICODE

encoding = PyUnicode_GetDefaultEncoding();

PyErr_SetString(PyExc_ValueError, "no encoding specified");

goto onError;

}

/* Encode via the codec registry */

v = PyCodec_Encode(str, encoding, errors);

if (v == NULL)

goto onError;

return v;

onError:

return NULL;

}

PyObject *PyString_AsEncodedString(PyObject *str,

const char *encoding,

const char *errors)

{

PyObject *v;

v = PyString_AsEncodedObject(str, encoding, errors);

if (v == NULL)

goto onError;

#ifdef Py_USING_UNICODE

/* Convert Unicode to a string using the default encoding */

if (PyUnicode_Check(v)) {

PyObject *temp = v;

v = PyUnicode_AsEncodedString(v, NULL, NULL);

Py_DECREF(temp);

if (v == NULL)

goto onError;

}

if (!PyString_Check(v)) {

PyErr_Format(PyExc_TypeError,

"encoder did not return a string object (type=%.400s)",

Py_TYPE(v)->tp_name);

Py_DECREF(v);

goto onError;

}

return v;

onError:

return NULL;

}

static void

string_dealloc(PyObject *op)

{

switch (PyString_CHECK_INTERNED(op)) {

case SSTATE_NOT_INTERNED:

break;

case SSTATE_INTERNED_MORTAL:

/* revive dead object temporarily for DelItem */

Py_REFCNT(op) = 3;

if (PyDict_DelItem(interned, op) != 0)

Py_FatalError(

"deletion of interned string failed");

break;

case SSTATE_INTERNED_IMMORTAL:

Py_FatalError("Immortal interned string died.");

default:

Py_FatalError("Inconsistent interned string state.");

}

Py_TYPE(op)->tp_free(op);

}

PyObject *PyString_DecodeEscape(const char *s,

const char *recode_encoding)

{

int c;

char *p, *buf;

const char *end;

PyObject *v;

Py_ssize_t newlen = recode_encoding ? 4*len:len;

v = PyString_FromStringAndSize((char *)NULL, newlen);

if (v == NULL)

return NULL;

p = buf = PyString_AsString(v);

end = s + len;

while (s < end) {

if (*s != '\\') {

non_esc:

#ifdef Py_USING_UNICODE

if (recode_encoding && (*s & 0x80)) {

PyObject *u, *w;

char *r;

const char* t;

Py_ssize_t rn;

t = s;

/* Decode non-ASCII bytes as UTF-8. */

while (t < end && (*t & 0x80)) t++;

u = PyUnicode_DecodeUTF8(s, t - s, errors);

if(!u) goto failed;

/* Recode them in target encoding. */

w = PyUnicode_AsEncodedString(

u, recode_encoding, errors);

Py_DECREF(u);

if (!w) goto failed;

/* Append bytes to output buffer. */

assert(PyString_Check(w));

r = PyString_AS_STRING(w);

rn = PyString_GET_SIZE(w);

Py_MEMCPY(p, r, rn);

p += rn;

Py_DECREF(w);

s = t;

} else {

*p++ = *s++;

}

*p++ = *s++;

continue;

}

s++;

if (s==end) {

PyErr_SetString(PyExc_ValueError,

"Trailing \\ in string");

goto failed;

}

switch (*s++) {

/* XXX This assumes ASCII! */

case '\n': break;

case '\\': *p++ = '\\'; break;

case '\'': *p++ = '\''; break;

case '\"': *p++ = '\"'; break;

case 'b': *p++ = '\b'; break;

case 'f': *p++ = '\014'; break; /* FF */

case 't': *p++ = '\t'; break;

case 'n': *p++ = '\n'; break;

case 'r': *p++ = '\r'; break;

case 'v': *p++ = '\013'; break; /* VT */

case 'a': *p++ = '\007'; break; /* BEL, not classic C */

case '0': case '1': case '2': case '3':

case '4': case '5': case '6': case '7':

c = s[-1] - '0';

if (s < end && '0' <= *s && *s <= '7') {

c = (c<<3) + *s++ - '0';

if (s < end && '0' <= *s && *s <= '7')

c = (c<<3) + *s++ - '0';

}

*p++ = c;

break;

case 'x':

if (s+1 < end &&

isxdigit(Py_CHARMASK(s[0])) &&

isxdigit(Py_CHARMASK(s[1])))

{

unsigned int x = 0;

c = Py_CHARMASK(*s);

s++;

if (isdigit(c))

x = c - '0';

else if (islower(c))

x = 10 + c - 'a';

else

x = 10 + c - 'A';

x = x << 4;

c = Py_CHARMASK(*s);

s++;

if (isdigit(c))

x += c - '0';

else if (islower(c))

x += 10 + c - 'a';

else

x += 10 + c - 'A';

*p++ = x;

break;

}

if (!errors || strcmp(errors, "strict") == 0) {

PyErr_SetString(PyExc_ValueError,

"invalid \\x escape");

goto failed;

}

if (strcmp(errors, "replace") == 0) {

*p++ = '?';

} else if (strcmp(errors, "ignore") == 0)

/* do nothing */;

else {

PyErr_Format(PyExc_ValueError,

"decoding error; "

"unknown error handling code: %.400s",

errors);

goto failed;

}

#ifndef Py_USING_UNICODE

case 'u':

case 'U':

case 'N':

if (unicode) {

PyErr_SetString(PyExc_ValueError,

"Unicode escapes not legal "

"when Unicode disabled");

goto failed;

}

default:

*p++ = '\\';

s--;

goto non_esc; /* an arbitry number of unescaped

}

}

if (p-buf < newlen)

_PyString_Resize(&v, p - buf);

return v;

failed:

Py_DECREF(v);

return NULL;

}

static Py_ssize_t

string_getsize(register PyObject *op)

{

char *s;

Py_ssize_t len;

if (PyString_AsStringAndSize(op, &s, &len))

return -1;

return len;

}

static /*const*/ char *

string_getbuffer(register PyObject *op)

{

char *s;

Py_ssize_t len;

if (PyString_AsStringAndSize(op, &s, &len))

return NULL;

return s;

}

PyString_Size(register PyObject *op)

{

if (!PyString_Check(op))

return string_getsize(op);

return Py_SIZE(op);

}

/*const*/ char *

PyString_AsString(register PyObject *op)

{

if (!PyString_Check(op))

return string_getbuffer(op);

return ((PyStringObject *)op) -> ob_sval;

}

PyString_AsStringAndSize(register PyObject *obj,

register char **s,

register Py_ssize_t *len)

{

if (s == NULL) {

PyErr_BadInternalCall();

return -1;

}

if (!PyString_Check(obj)) {

#ifdef Py_USING_UNICODE

if (PyUnicode_Check(obj)) {

obj = _PyUnicode_AsDefaultEncodedString(obj, NULL);

if (obj == NULL)

return -1;

}

else

{

PyErr_Format(PyExc_TypeError,

"expected string or Unicode object, "

"%.200s found", Py_TYPE(obj)->tp_name);

return -1;

}

}

*s = PyString_AS_STRING(obj);

if (len != NULL)

*len = PyString_GET_SIZE(obj);

else if (strlen(*s) != (size_t)PyString_GET_SIZE(obj)) {

PyErr_SetString(PyExc_TypeError,

"expected string without null bytes");

return -1;

}

return 0;

}

#include "stringlib/stringdefs.h"

#include "stringlib/fastsearch.h"

#include "stringlib/count.h"

#include "stringlib/find.h"

#include "stringlib/partition.h"

#define _Py_InsertThousandsGrouping _PyString_InsertThousandsGrouping

#include "stringlib/localeutil.h"

static int

string_print(PyStringObject *op, FILE *fp, int flags)

{

Py_ssize_t i, str_len;

char c;

int quote;

/* XXX Ought to check for interrupts when writing long strings */

if (! PyString_CheckExact(op)) {

int ret;

/* A str subclass may have its own __str__ method. */

op = (PyStringObject *) PyObject_Str((PyObject *)op);

if (op == NULL)

return -1;

ret = string_print(op, fp, flags);

Py_DECREF(op);

return ret;

}

if (flags & Py_PRINT_RAW) {

char *data = op->ob_sval;

Py_ssize_t size = Py_SIZE(op);

Py_BEGIN_ALLOW_THREADS

while (size > INT_MAX) {

/* Very long strings cannot be written atomically.

const int chunk_size = INT_MAX & ~0x3FFF;

fwrite(data, 1, chunk_size, fp);

data += chunk_size;

size -= chunk_size;

}

#ifdef __VMS

if (size) fwrite(data, (int)size, 1, fp);

fwrite(data, 1, (int)size, fp);

Py_END_ALLOW_THREADS

return 0;

}

/* figure out which quote to use; single is preferred */

quote = '\'';

if (memchr(op->ob_sval, '\'', Py_SIZE(op)) &&

!memchr(op->ob_sval, '"', Py_SIZE(op)))

quote = '"';

str_len = Py_SIZE(op);

Py_BEGIN_ALLOW_THREADS

fputc(quote, fp);

for (i = 0; i < str_len; i++) {

/* Since strings are immutable and the caller should have a

c = op->ob_sval[i];

if (c == quote || c == '\\')

fprintf(fp, "\\%c", c);

else if (c == '\t')

fprintf(fp, "\\t");

else if (c == '\n')

fprintf(fp, "\\n");

else if (c == '\r')

fprintf(fp, "\\r");

else if (c < ' ' || c >= 0x7f)

fprintf(fp, "\\x%02x", c & 0xff);

else

fputc(c, fp);

}

fputc(quote, fp);

Py_END_ALLOW_THREADS

return 0;

}

PyObject *

PyString_Repr(PyObject *obj, int smartquotes)

{

register PyStringObject* op = (PyStringObject*) obj;

size_t newsize = 2 + 4 * Py_SIZE(op);

PyObject *v;

if (newsize > PY_SSIZE_T_MAX || newsize / 4 != Py_SIZE(op)) {

PyErr_SetString(PyExc_OverflowError,

"string is too large to make repr");

return NULL;

}

v = PyString_FromStringAndSize((char *)NULL, newsize);

if (v == NULL) {

return NULL;

}

else {

register Py_ssize_t i;

register char c;

register char *p;

int quote;

/* figure out which quote to use; single is preferred */

quote = '\'';

if (smartquotes &&

memchr(op->ob_sval, '\'', Py_SIZE(op)) &&

!memchr(op->ob_sval, '"', Py_SIZE(op)))

quote = '"';

p = PyString_AS_STRING(v);

*p++ = quote;

for (i = 0; i < Py_SIZE(op); i++) {

/* There's at least enough room for a hex escape

assert(newsize - (p - PyString_AS_STRING(v)) >= 5);

c = op->ob_sval[i];

if (c == quote || c == '\\')

*p++ = '\\', *p++ = c;

else if (c == '\t')

*p++ = '\\', *p++ = 't';

else if (c == '\n')

*p++ = '\\', *p++ = 'n';

else if (c == '\r')

*p++ = '\\', *p++ = 'r';

else if (c < ' ' || c >= 0x7f) {

/* For performance, we don't want to call

sprintf(p, "\\x%02x", c & 0xff);

p += 4;

}

else

*p++ = c;

}

assert(newsize - (p - PyString_AS_STRING(v)) >= 1);

*p++ = quote;

*p = '\0';

_PyString_Resize(

&v, (p - PyString_AS_STRING(v)));

return v;

}

}

static PyObject *

string_repr(PyObject *op)

{

return PyString_Repr(op, 1);

}

static PyObject *

string_str(PyObject *s)

{

assert(PyString_Check(s));

if (PyString_CheckExact(s)) {

Py_INCREF(s);

return s;

}

else {

/* Subtype -- return genuine string with the same value. */

PyStringObject *t = (PyStringObject *) s;

return PyString_FromStringAndSize(t->ob_sval, Py_SIZE(t));

}

}

static Py_ssize_t

string_length(PyStringObject *a)

{

return Py_SIZE(a);

}

static PyObject *

string_concat(register PyStringObject *a, register PyObject *bb)

{

register Py_ssize_t size;

register PyStringObject *op;

if (!PyString_Check(bb)) {

#ifdef Py_USING_UNICODE

if (PyUnicode_Check(bb))

return PyUnicode_Concat((PyObject *)a, bb);

if (PyByteArray_Check(bb))

return PyByteArray_Concat((PyObject *)a, bb);

PyErr_Format(PyExc_TypeError,

"cannot concatenate 'str' and '%.200s' objects",

Py_TYPE(bb)->tp_name);

return NULL;

}

#define b ((PyStringObject *)bb)

/* Optimize cases with empty left or right operand */

if ((Py_SIZE(a) == 0 || Py_SIZE(b) == 0) &&

PyString_CheckExact(a) && PyString_CheckExact(b)) {

if (Py_SIZE(a) == 0) {

Py_INCREF(bb);

return bb;

}

Py_INCREF(a);

return (PyObject *)a;

}

size = Py_SIZE(a) + Py_SIZE(b);

/* Check that string sizes are not negative, to prevent an

if (Py_SIZE(a) < 0 || Py_SIZE(b) < 0 ||

Py_SIZE(a) > PY_SSIZE_T_MAX - Py_SIZE(b)) {

PyErr_SetString(PyExc_OverflowError,

"strings are too large to concat");

return NULL;

}

/* Inline PyObject_NewVar */

if (size > PY_SSIZE_T_MAX - sizeof(PyStringObject)) {

PyErr_SetString(PyExc_OverflowError,

"strings are too large to concat");

return NULL;