#ifndef Py_BUILD_CORE_BUILTIN

# define Py_BUILD_CORE_MODULE 1

#include "Python.h"

#include "multibytecodec.h"

#include "clinic/multibytecodec.c.h"

#include "pycore_tuple.h" // _PyTuple_FromPairSteal

#include <stddef.h> // offsetof()

#define MODULE_NAME "_multibytecodec"

typedef struct {

PyTypeObject *encoder_type;

PyTypeObject *decoder_type;

PyTypeObject *reader_type;

PyTypeObject *writer_type;

PyTypeObject *multibytecodec_type;

PyObject *str_write;

} module_state;

static module_state *

get_module_state(PyObject *module)

{

module_state *state = PyModule_GetState(module);

assert(state != NULL);

return state;

}

static struct PyModuleDef _multibytecodecmodule;

static module_state *

find_state_by_def(PyTypeObject *type)

{

PyObject *module = PyType_GetModuleByDef(type, &_multibytecodecmodule);

assert(module != NULL);

return get_module_state(module);

}

#define clinic_get_state() find_state_by_def(type)

#undef clinic_get_state

#define _MultibyteCodec_CAST(op) ((MultibyteCodec *)(op))

#define _MultibyteCodecObject_CAST(op) ((MultibyteCodecObject *)(op))

#define _MultibyteStatefulCodecContext_CAST(op) \

((MultibyteStatefulCodecContext *)(op))

#define _MultibyteStatefulEncoderContext_CAST(op) \

((MultibyteStatefulEncoderContext *)(op))

#define _MultibyteStatefulDecoderContext_CAST(op) \

((MultibyteStatefulDecoderContext *)(op))

#define _MultibyteIncrementalEncoderObject_CAST(op) \

((MultibyteIncrementalEncoderObject *)(op))

#define _MultibyteIncrementalDecoderObject_CAST(op) \

((MultibyteIncrementalDecoderObject *)(op))

#define _MultibyteStreamReaderObject_CAST(op) \

((MultibyteStreamReaderObject *)(op))

#define _MultibyteStreamWriterObject_CAST(op) \

((MultibyteStreamWriterObject *)(op))

typedef struct {

PyObject *inobj;

Py_ssize_t inpos, inlen;

unsigned char *outbuf, *outbuf_end;

PyObject *excobj, *outobj;

} MultibyteEncodeBuffer;

typedef struct {

const unsigned char *inbuf, *inbuf_top, *inbuf_end;

PyObject *excobj;

_PyUnicodeWriter writer;

} MultibyteDecodeBuffer;

static char *incnewkwarglist[] = {"errors", NULL};

static char *streamkwarglist[] = {"stream", "errors", NULL};

static PyObject *multibytecodec_encode(const MultibyteCodec *,

MultibyteCodec_State *, PyObject *, Py_ssize_t *,

PyObject *, int);

#define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */

static PyObject *

make_tuple(PyObject *object, Py_ssize_t len)

{

if (object == NULL) {

return NULL;

}

PyObject* len_obj = PyLong_FromSsize_t(len);

if (len_obj == NULL) {

Py_DECREF(object);

return NULL;

}

return _PyTuple_FromPairSteal(object, len_obj);

}

static PyObject *

internal_error_callback(const char *errors)

{

if (errors == NULL || strcmp(errors, "strict") == 0)

return ERROR_STRICT;

else if (strcmp(errors, "ignore") == 0)

return ERROR_IGNORE;

else if (strcmp(errors, "replace") == 0)

return ERROR_REPLACE;

else

return PyUnicode_FromString(errors);

}

static PyObject *

call_error_callback(PyObject *errors, PyObject *exc)

{

PyObject *cb, *r;

const char *str;

assert(PyUnicode_Check(errors));

str = PyUnicode_AsUTF8(errors);

if (str == NULL)

return NULL;

cb = PyCodec_LookupError(str);

if (cb == NULL)

return NULL;

r = PyObject_CallOneArg(cb, exc);

Py_DECREF(cb);

return r;

}

static PyObject *

codecctx_errors_get(PyObject *op, void *Py_UNUSED(closure))

{

const char *errors;

MultibyteStatefulCodecContext *self = _MultibyteStatefulCodecContext_CAST(op);

if (self->errors == ERROR_STRICT)

errors = "strict";

else if (self->errors == ERROR_IGNORE)

errors = "ignore";

else if (self->errors == ERROR_REPLACE)

errors = "replace";

else {

return Py_NewRef(self->errors);

}

return PyUnicode_FromString(errors);

}

static int

codecctx_errors_set(PyObject *op, PyObject *value, void *Py_UNUSED(closure))

{

PyObject *cb;

const char *str;

MultibyteStatefulCodecContext *self = _MultibyteStatefulCodecContext_CAST(op);

if (value == NULL) {

PyErr_SetString(PyExc_AttributeError, "cannot delete attribute");

return -1;

}

if (!PyUnicode_Check(value)) {

PyErr_SetString(PyExc_TypeError, "errors must be a string");

return -1;

}

str = PyUnicode_AsUTF8(value);

if (str == NULL)

return -1;

cb = internal_error_callback(str);

if (cb == NULL)

return -1;

ERROR_DECREF(self->errors);

self->errors = cb;

return 0;

}

static PyGetSetDef codecctx_getsets[] = {

{"errors", codecctx_errors_get, codecctx_errors_set,

PyDoc_STR("how to treat errors")},

{NULL,}

};

static int

expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)

{

Py_ssize_t orgpos, orgsize, incsize;

orgpos = (Py_ssize_t)((char *)buf->outbuf -

PyBytes_AS_STRING(buf->outobj));

orgsize = PyBytes_GET_SIZE(buf->outobj);

incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);

if (orgsize > PY_SSIZE_T_MAX - incsize) {

PyErr_NoMemory();

return -1;

}

if (_PyBytes_Resize(&buf->outobj, orgsize + incsize) == -1)

return -1;

buf->outbuf = (unsigned char *)PyBytes_AS_STRING(buf->outobj) +orgpos;

buf->outbuf_end = (unsigned char *)PyBytes_AS_STRING(buf->outobj)

+ PyBytes_GET_SIZE(buf->outobj);

return 0;

}

#define REQUIRE_ENCODEBUFFER(buf, s) do { \

if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \

if (expand_encodebuffer(buf, s) == -1) \

goto errorexit; \

} while(0)

static int

multibytecodec_encerror(const MultibyteCodec *codec,

MultibyteCodec_State *state,

MultibyteEncodeBuffer *buf,

PyObject *errors, Py_ssize_t e)

{

PyObject *retobj = NULL, *retstr = NULL, *tobj;

Py_ssize_t retstrsize, newpos;

Py_ssize_t esize, start, end;

const char *reason;

if (e > 0) {

reason = "illegal multibyte sequence";

esize = e;

}

else {

switch (e) {

case MBERR_TOOSMALL:

REQUIRE_ENCODEBUFFER(buf, -1);

return 0; /* retry it */

case MBERR_TOOFEW:

reason = "incomplete multibyte sequence";

esize = (Py_ssize_t)buf->inpos;

break;

case MBERR_INTERNAL:

PyErr_SetString(PyExc_RuntimeError,

"internal codec error");

return -1;

default:

PyErr_SetString(PyExc_RuntimeError,

"unknown runtime error");

return -1;

}

}

if (errors == ERROR_REPLACE) {

PyObject *replchar;

Py_ssize_t r;

Py_ssize_t inpos;

int kind;

const void *data;

replchar = PyUnicode_FromOrdinal('?');

if (replchar == NULL)

goto errorexit;

kind = PyUnicode_KIND(replchar);

data = PyUnicode_DATA(replchar);

inpos = 0;

for (;;) {

Py_ssize_t outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);

r = codec->encode(state, codec,

kind, data, &inpos, 1,

&buf->outbuf, outleft, 0);

if (r == MBERR_TOOSMALL) {

REQUIRE_ENCODEBUFFER(buf, -1);

continue;

}

else

break;

}

Py_DECREF(replchar);

if (r != 0) {

REQUIRE_ENCODEBUFFER(buf, 1);

*buf->outbuf++ = '?';

}

}

if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {

buf->inpos += esize;

return 0;

}

start = (Py_ssize_t)buf->inpos;

end = start + esize;

/* use cached exception object if available */

if (buf->excobj == NULL) {

buf->excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError,

"sOnns",

codec->encoding, buf->inobj,

start, end, reason);

if (buf->excobj == NULL)

goto errorexit;

}

else

if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 ||

PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 ||

PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0)

goto errorexit;

if (errors == ERROR_STRICT) {

PyCodec_StrictErrors(buf->excobj);

goto errorexit;

}

retobj = call_error_callback(errors, buf->excobj);

if (retobj == NULL)

goto errorexit;

if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||

(!PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) && !PyBytes_Check(tobj)) ||

!PyLong_Check(PyTuple_GET_ITEM(retobj, 1))) {

PyErr_SetString(PyExc_TypeError,

"encoding error handler must return "

"(str, int) tuple");

goto errorexit;

}

if (PyUnicode_Check(tobj)) {

Py_ssize_t inpos;

retstr = multibytecodec_encode(codec, state, tobj,

&inpos, ERROR_STRICT,

MBENC_FLUSH);

if (retstr == NULL)

goto errorexit;

}

else {

retstr = Py_NewRef(tobj);

}

assert(PyBytes_Check(retstr));

retstrsize = PyBytes_GET_SIZE(retstr);

if (retstrsize > 0) {

REQUIRE_ENCODEBUFFER(buf, retstrsize);

memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize);

buf->outbuf += retstrsize;

}

newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));

if (newpos < 0 && !PyErr_Occurred())

newpos += (Py_ssize_t)buf->inlen;

if (newpos < 0 || newpos > buf->inlen) {

PyErr_Clear();

PyErr_Format(PyExc_IndexError,

"position %zd from error handler out of bounds",

newpos);

goto errorexit;

}

buf->inpos = newpos;

Py_DECREF(retobj);

Py_DECREF(retstr);

return 0;

errorexit:

Py_XDECREF(retobj);

Py_XDECREF(retstr);

return -1;

}

static int

multibytecodec_decerror(const MultibyteCodec *codec,

MultibyteCodec_State *state,

MultibyteDecodeBuffer *buf,

PyObject *errors, Py_ssize_t e)

{

PyObject *retobj = NULL, *retuni = NULL;

Py_ssize_t newpos;

const char *reason;

Py_ssize_t esize, start, end;

if (e > 0) {

reason = "illegal multibyte sequence";

esize = e;

}

else {

switch (e) {

case MBERR_TOOSMALL:

return 0; /* retry it */

case MBERR_TOOFEW:

reason = "incomplete multibyte sequence";

esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);

break;

case MBERR_INTERNAL:

PyErr_SetString(PyExc_RuntimeError,

"internal codec error");

return -1;

case MBERR_EXCEPTION:

return -1;

default:

PyErr_SetString(PyExc_RuntimeError,

"unknown runtime error");

return -1;

}

}

if (errors == ERROR_REPLACE) {

if (_PyUnicodeWriter_WriteChar(&buf->writer,

Py_UNICODE_REPLACEMENT_CHARACTER) < 0)

goto errorexit;

}

if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {

buf->inbuf += esize;

return 0;

}

start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);

end = start + esize;

/* use cached exception object if available */

if (buf->excobj == NULL) {

buf->excobj = PyUnicodeDecodeError_Create(codec->encoding,

(const char *)buf->inbuf_top,

(Py_ssize_t)(buf->inbuf_end - buf->inbuf_top),

start, end, reason);

if (buf->excobj == NULL)

goto errorexit;

}

else

if (PyUnicodeDecodeError_SetStart(buf->excobj, start) ||

PyUnicodeDecodeError_SetEnd(buf->excobj, end) ||

PyUnicodeDecodeError_SetReason(buf->excobj, reason))

goto errorexit;

if (errors == ERROR_STRICT) {

PyCodec_StrictErrors(buf->excobj);

goto errorexit;

}

retobj = call_error_callback(errors, buf->excobj);

if (retobj == NULL)

goto errorexit;

if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||

!PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||

!PyLong_Check(PyTuple_GET_ITEM(retobj, 1))) {

PyErr_SetString(PyExc_TypeError,

"decoding error handler must return "

"(str, int) tuple");

goto errorexit;

}

if (_PyUnicodeWriter_WriteStr(&buf->writer, retuni) < 0)

goto errorexit;

newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));

if (newpos < 0 && !PyErr_Occurred())

newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);

if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {

PyErr_Clear();

PyErr_Format(PyExc_IndexError,

"position %zd from error handler out of bounds",

newpos);

goto errorexit;

}

buf->inbuf = buf->inbuf_top + newpos;

Py_DECREF(retobj);

return 0;

errorexit:

Py_XDECREF(retobj);

return -1;

}

static PyObject *

multibytecodec_encode(const MultibyteCodec *codec,

MultibyteCodec_State *state,

PyObject *text, Py_ssize_t *inpos_t,

PyObject *errors, int flags)

{

MultibyteEncodeBuffer buf;

Py_ssize_t finalsize, r = 0;

Py_ssize_t datalen;

int kind;

const void *data;

datalen = PyUnicode_GET_LENGTH(text);

if (datalen == 0 && !(flags & MBENC_RESET))

return PyBytes_FromStringAndSize(NULL, 0);

buf.excobj = NULL;

buf.outobj = NULL;

buf.inobj = text; /* borrowed reference */

buf.inpos = 0;

buf.inlen = datalen;

kind = PyUnicode_KIND(buf.inobj);

data = PyUnicode_DATA(buf.inobj);

if (datalen > (PY_SSIZE_T_MAX - 16) / 2) {

PyErr_NoMemory();

goto errorexit;

}

buf.outobj = PyBytes_FromStringAndSize(NULL, datalen * 2 + 16);

if (buf.outobj == NULL)

goto errorexit;

buf.outbuf = (unsigned char *)PyBytes_AS_STRING(buf.outobj);

buf.outbuf_end = buf.outbuf + PyBytes_GET_SIZE(buf.outobj);

while (buf.inpos < buf.inlen) {

/* we don't reuse inleft and outleft here.

Py_ssize_t outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);

r = codec->encode(state, codec,

kind, data,

&buf.inpos, buf.inlen,

&buf.outbuf, outleft, flags);

if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))

break;

else if (multibytecodec_encerror(codec, state, &buf, errors,r))

goto errorexit;

else if (r == MBERR_TOOFEW)

break;

}

if (codec->encreset != NULL && (flags & MBENC_RESET))

for (;;) {

Py_ssize_t outleft;

outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);

r = codec->encreset(state, codec, &buf.outbuf,

outleft);

if (r == 0)

break;

else if (multibytecodec_encerror(codec, state,

&buf, errors, r))

goto errorexit;

}

finalsize = (Py_ssize_t)((char *)buf.outbuf -

PyBytes_AS_STRING(buf.outobj));

if (finalsize != PyBytes_GET_SIZE(buf.outobj))

if (_PyBytes_Resize(&buf.outobj, finalsize) == -1)

goto errorexit;

if (inpos_t)

*inpos_t = buf.inpos;

Py_XDECREF(buf.excobj);

return buf.outobj;

errorexit:

Py_XDECREF(buf.excobj);

Py_XDECREF(buf.outobj);

return NULL;

}

static PyObject *

_multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self,

PyObject *input,

const char *errors)

{

MultibyteCodec_State state;

PyObject *errorcb, *r, *ucvt;

Py_ssize_t datalen;

if (PyUnicode_Check(input))

ucvt = NULL;

else {

input = ucvt = PyObject_Str(input);

if (input == NULL)

return NULL;

else if (!PyUnicode_Check(input)) {

PyErr_SetString(PyExc_TypeError,

"couldn't convert the object to unicode.");

Py_DECREF(ucvt);

return NULL;

}

}

datalen = PyUnicode_GET_LENGTH(input);

errorcb = internal_error_callback(errors);

if (errorcb == NULL) {

Py_XDECREF(ucvt);

return NULL;

}

if (self->codec->encinit != NULL &&

self->codec->encinit(&state, self->codec) != 0)

goto errorexit;

r = multibytecodec_encode(self->codec, &state,

input, NULL, errorcb,

MBENC_FLUSH | MBENC_RESET);

if (r == NULL)

goto errorexit;

ERROR_DECREF(errorcb);

Py_XDECREF(ucvt);

return make_tuple(r, datalen);

errorexit:

ERROR_DECREF(errorcb);

Py_XDECREF(ucvt);

return NULL;

}

static PyObject *

_multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self,

Py_buffer *input,

const char *errors)

{

MultibyteCodec_State state;

MultibyteDecodeBuffer buf;

PyObject *errorcb, *res;

const char *data;

Py_ssize_t datalen;

data = input->buf;

datalen = input->len;

errorcb = internal_error_callback(errors);

if (errorcb == NULL) {

return NULL;

}

if (datalen == 0) {

ERROR_DECREF(errorcb);

return make_tuple(Py_GetConstant(Py_CONSTANT_EMPTY_STR), 0);

}

_PyUnicodeWriter_Init(&buf.writer);

buf.writer.min_length = datalen;

buf.excobj = NULL;

buf.inbuf = buf.inbuf_top = (unsigned char *)data;

buf.inbuf_end = buf.inbuf_top + datalen;

if (self->codec->decinit != NULL &&

self->codec->decinit(&state, self->codec) != 0)

goto errorexit;

while (buf.inbuf < buf.inbuf_end) {

Py_ssize_t inleft, r;

inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);

r = self->codec->decode(&state, self->codec,

&buf.inbuf, inleft, &buf.writer);

if (r == 0)

break;

else if (multibytecodec_decerror(self->codec, &state,

&buf, errorcb, r))

goto errorexit;

}

res = _PyUnicodeWriter_Finish(&buf.writer);

if (res == NULL)

goto errorexit;

Py_XDECREF(buf.excobj);

ERROR_DECREF(errorcb);

return make_tuple(res, datalen);

errorexit:

ERROR_DECREF(errorcb);

Py_XDECREF(buf.excobj);

_PyUnicodeWriter_Dealloc(&buf.writer);

return NULL;

}

static struct PyMethodDef multibytecodec_methods[] = {

_MULTIBYTECODEC_MULTIBYTECODEC_ENCODE_METHODDEF

_MULTIBYTECODEC_MULTIBYTECODEC_DECODE_METHODDEF

{NULL, NULL},

};

static int

multibytecodec_clear(PyObject *op)

{

MultibyteCodecObject *self = _MultibyteCodecObject_CAST(op);

Py_CLEAR(self->cjk_module);

return 0;

}

static int

multibytecodec_traverse(PyObject *op, visitproc visit, void *arg)

{

MultibyteCodecObject *self = _MultibyteCodecObject_CAST(op);

Py_VISIT(Py_TYPE(self));

Py_VISIT(self->cjk_module);

return 0;

}

static void

multibytecodec_dealloc(PyObject *self)

{

PyObject_GC_UnTrack(self);

PyTypeObject *tp = Py_TYPE(self);

(void)multibytecodec_clear(self);

tp->tp_free(self);

Py_DECREF(tp);

}

static PyType_Slot multibytecodec_slots[] = {

{Py_tp_dealloc, multibytecodec_dealloc},

{Py_tp_getattro, PyObject_GenericGetAttr},

{Py_tp_methods, multibytecodec_methods},

{Py_tp_traverse, multibytecodec_traverse},

{Py_tp_clear, multibytecodec_clear},

{0, NULL},

};

static PyType_Spec multibytecodec_spec = {

.name = MODULE_NAME ".MultibyteCodec",

.basicsize = sizeof(MultibyteCodecObject),

.flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |

Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),

.slots = multibytecodec_slots,

};

#define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o))

#define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o))

static PyObject *

encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,

PyObject *unistr, int final)

{

PyObject *ucvt, *r = NULL;

PyObject *inbuf = NULL;

Py_ssize_t inpos, datalen;

PyObject *origpending = NULL;

if (PyUnicode_Check(unistr))

ucvt = NULL;

else {

unistr = ucvt = PyObject_Str(unistr);

if (unistr == NULL)

return NULL;

else if (!PyUnicode_Check(unistr)) {

PyErr_SetString(PyExc_TypeError,

"couldn't convert the object to str.");

Py_DECREF(ucvt);

return NULL;

}

}

if (ctx->pending) {

PyObject *inbuf_tmp;

origpending = Py_NewRef(ctx->pending);

inbuf_tmp = Py_NewRef(ctx->pending);

PyUnicode_Append(&inbuf_tmp, unistr);

if (inbuf_tmp == NULL)

goto errorexit;

Py_CLEAR(ctx->pending);

inbuf = inbuf_tmp;

}

else {

origpending = NULL;

inbuf = Py_NewRef(unistr);

}

inpos = 0;

datalen = PyUnicode_GET_LENGTH(inbuf);

r = multibytecodec_encode(ctx->codec, &ctx->state,

inbuf, &inpos,

ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0);

if (r == NULL) {

/* recover the original pending buffer */

Py_XSETREF(ctx->pending, origpending);

origpending = NULL;

goto errorexit;

}

Py_XDECREF(origpending);

if (inpos < datalen) {

if (datalen - inpos > MAXENCPENDING) {

/* normal codecs can't reach here */

PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError,

"sOnns",

ctx->codec->encoding,

inbuf,

inpos, datalen,

"pending buffer overflow");

if (excobj == NULL) goto errorexit;

PyErr_SetObject(PyExc_UnicodeEncodeError, excobj);

Py_DECREF(excobj);

goto errorexit;

}

ctx->pending = PyUnicode_Substring(inbuf, inpos, datalen);

if (ctx->pending == NULL) {

/* normal codecs can't reach here */

goto errorexit;

}

}

Py_DECREF(inbuf);

Py_XDECREF(ucvt);

return r;

errorexit:

Py_XDECREF(r);

Py_XDECREF(ucvt);

Py_XDECREF(origpending);

Py_XDECREF(inbuf);

return NULL;

}

static int

decoder_append_pending(MultibyteStatefulDecoderContext *ctx,

MultibyteDecodeBuffer *buf)

{

Py_ssize_t npendings;

npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);

if (npendings + ctx->pendingsize > MAXDECPENDING ||

npendings > PY_SSIZE_T_MAX - ctx->pendingsize) {

Py_ssize_t bufsize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);

PyObject *excobj = PyUnicodeDecodeError_Create(ctx->codec->encoding,

(const char *)buf->inbuf_top,

bufsize,

0,

bufsize,

"pending buffer overflow");

if (excobj == NULL) return -1;

PyErr_SetObject(PyExc_UnicodeDecodeError, excobj);

Py_DECREF(excobj);

return -1;

}

memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings);

ctx->pendingsize += npendings;

return 0;

}

static int

decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data,

Py_ssize_t size)

{

buf->inbuf = buf->inbuf_top = (const unsigned char *)data;

buf->inbuf_end = buf->inbuf_top + size;

buf->writer.min_length += size;

return 0;

}

static int

decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx,

MultibyteDecodeBuffer *buf)

{

while (buf->inbuf < buf->inbuf_end) {

Py_ssize_t inleft;

Py_ssize_t r;

inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);

r = ctx->codec->decode(&ctx->state, ctx->codec,

&buf->inbuf, inleft, &buf->writer);

if (r == 0 || r == MBERR_TOOFEW)

break;

else if (multibytecodec_decerror(ctx->codec, &ctx->state,

buf, ctx->errors, r))

return -1;

}

return 0;

}

static PyObject *

_multibytecodec_MultibyteIncrementalEncoder_encode_impl(MultibyteIncrementalEncoderObject *self,

PyObject *input,

int final)

{

return encoder_encode_stateful(STATEFUL_ECTX(self), input, final);

}

static PyObject *

_multibytecodec_MultibyteIncrementalEncoder_getstate_impl(MultibyteIncrementalEncoderObject *self)

{

/* state made up of 1 byte for buffer size, up to MAXENCPENDING*4 bytes

unsigned char statebytes[1 + MAXENCPENDING*4 + sizeof(self->state.c)];

Py_ssize_t statesize;

const char *pendingbuffer = NULL;

Py_ssize_t pendingsize;

if (self->pending != NULL) {

pendingbuffer = PyUnicode_AsUTF8AndSize(self->pending, &pendingsize);

if (pendingbuffer == NULL) {

return NULL;

}

if (pendingsize > MAXENCPENDING*4) {

PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError,

"sOnns",

self->codec->encoding,

self->pending,

0, PyUnicode_GET_LENGTH(self->pending),

"pending buffer too large");

if (excobj == NULL) {

return NULL;

}

PyErr_SetObject(PyExc_UnicodeEncodeError, excobj);

Py_DECREF(excobj);

return NULL;

}

statebytes[0] = (unsigned char)pendingsize;

memcpy(statebytes + 1, pendingbuffer, pendingsize);

statesize = 1 + pendingsize;

} else {

statebytes[0] = 0;

statesize = 1;

}

memcpy(statebytes+statesize, self->state.c,

sizeof(self->state.c));

statesize += sizeof(self->state.c);

return (PyObject *)_PyLong_FromByteArray(statebytes, statesize,

1 /* little-endian */ ,

0 /* unsigned */ );