Skip to content

bpo-33578: Fix getstate/setstate for CJK decoder #10290

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
methane merged 1 commit into from
Nov 2, 2018
Merged

bpo-33578: Fix getstate/setstate for CJK decoder #10290

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Lib/test/test_multibytecodec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,10 @@ def test_state_methods(self):
pending4, _ = decoder.getstate()
self.assertEqual(pending4, b'')

# Ensure state values are preserved correctly
decoder.setstate((b'abc', 123456789))
self.assertEqual(decoder.getstate(), (b'abc', 123456789))

def test_setstate_validates_input(self):
decoder = codecs.getincrementaldecoder('euc_jp')()
self.assertRaises(TypeError, decoder.setstate, 123)
Expand Down
27 changes: 22 additions & 5 deletions Modules/cjkcodecs/multibytecodec.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1218,14 +1218,24 @@ _multibytecodec_MultibyteIncrementalDecoder_getstate_impl(MultibyteIncrementalDe
/*[clinic end generated code: output=255009c4713b7f82 input=4006aa49bddbaa75]*/
{
PyObject *buffer;
PyObject *statelong;

buffer = PyBytes_FromStringAndSize((const char *)self->pending,
self->pendingsize);
if (buffer == NULL) {
return NULL;
}

return make_tuple(buffer, (Py_ssize_t)*self->state.c);
statelong = (PyObject *)_PyLong_FromByteArray(self->state.c,
sizeof(self->state.c),
1 /* little-endian */ ,
0 /* unsigned */ );
if (statelong == NULL) {
Py_DECREF(buffer);
return NULL;
}

return Py_BuildValue("NN", buffer, statelong);
}

/*[clinic input]
Expand All @@ -1240,16 +1250,23 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe
/*[clinic end generated code: output=106b2fbca3e2dcc2 input=e5d794e8baba1a47]*/
{
PyObject *buffer;
PyLongObject *statelong;
Py_ssize_t buffersize;
char *bufferstr;
unsigned long long flag;
unsigned char statebytes[8];

if (!PyArg_ParseTuple(state, "SK;setstate(): illegal state argument",
&buffer, &flag))
if (!PyArg_ParseTuple(state, "SO!;setstate(): illegal state argument",
&buffer, &PyLong_Type, &statelong))
{
return NULL;
}

if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes),
1 /* little-endian */ ,
0 /* unsigned */ ) < 0) {
return NULL;
}

buffersize = PyBytes_Size(buffer);
if (buffersize == -1) {
return NULL;
Expand All @@ -1266,7 +1283,7 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe
}
self->pendingsize = buffersize;
memcpy(self->pending, bufferstr, self->pendingsize);
memcpy(self->state.c, (unsigned char *)&flag, sizeof(flag));
memcpy(self->state.c, statebytes, sizeof(statebytes));

Py_RETURN_NONE;
}
Expand Down