Skip to content

Commit ac22f6a

Browse files
libcthornemiss-islington
libcthorne
authored and
miss-islington
committed
bpo-33578: Add getstate/setstate for CJK codec (GH-6984)
This implements getstate and setstate for the cjkcodecs multibyte incremental encoders/decoders, primarily to fix issues with seek/tell. The encoder getstate/setstate is slightly tricky as the "state" is pending bytes + MultibyteCodec_State but only an integer can be returned. The approach I've taken is to encode this data into a long, similar to how .tell() encodes a "cookie_type" as a long. https://bugs.python.org/issue33578
1 parent 4b5e62d commit ac22f6a

8 files changed

Lines changed: 416 additions & 22 deletions

File tree

‎Lib/test/test_io.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2971,6 +2971,34 @@ def test_seek_and_tell_with_data(data, min_pos=0):
29712971
finally:
29722972
StatefulIncrementalDecoder.codecEnabled = 0
29732973

2974+
def test_multibyte_seek_and_tell(self):
2975+
f = self.open(support.TESTFN, "w", encoding="euc_jp")
2976+
f.write("AB\n\u3046\u3048\n")
2977+
f.close()
2978+
2979+
f = self.open(support.TESTFN, "r", encoding="euc_jp")
2980+
self.assertEqual(f.readline(), "AB\n")
2981+
p0 = f.tell()
2982+
self.assertEqual(f.readline(), "\u3046\u3048\n")
2983+
p1 = f.tell()
2984+
f.seek(p0)
2985+
self.assertEqual(f.readline(), "\u3046\u3048\n")
2986+
self.assertEqual(f.tell(), p1)
2987+
f.close()
2988+
2989+
def test_seek_with_encoder_state(self):
2990+
f = self.open(support.TESTFN, "w", encoding="euc_jis_2004")
2991+
f.write("\u00e6\u0300")
2992+
p0 = f.tell()
2993+
f.write("\u00e6")
2994+
f.seek(p0)
2995+
f.write("\u0300")
2996+
f.close()
2997+
2998+
f = self.open(support.TESTFN, "r", encoding="euc_jis_2004")
2999+
self.assertEqual(f.readline(), "\u00e6\u0300\u0300")
3000+
f.close()
3001+
29743002
def test_encoded_writes(self):
29753003
data = "1234567890"
29763004
tests = ("utf-16",

‎Lib/test/test_multibytecodec.py‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,88 @@ def test_stateful_keep_buffer(self):
117117
self.assertRaises(UnicodeEncodeError, encoder.encode, '\u0123')
118118
self.assertEqual(encoder.encode('', True), b'\xa9\xdc')
119119

120+
def test_state_methods_with_buffer_state(self):
121+
# euc_jis_2004 stores state as a buffer of pending bytes
122+
encoder = codecs.getincrementalencoder('euc_jis_2004')()
123+
124+
initial_state = encoder.getstate()
125+
self.assertEqual(encoder.encode('\u00e6\u0300'), b'\xab\xc4')
126+
encoder.setstate(initial_state)
127+
self.assertEqual(encoder.encode('\u00e6\u0300'), b'\xab\xc4')
128+
129+
self.assertEqual(encoder.encode('\u00e6'), b'')
130+
partial_state = encoder.getstate()
131+
self.assertEqual(encoder.encode('\u0300'), b'\xab\xc4')
132+
encoder.setstate(partial_state)
133+
self.assertEqual(encoder.encode('\u0300'), b'\xab\xc4')
134+
135+
def test_state_methods_with_non_buffer_state(self):
136+
# iso2022_jp stores state without using a buffer
137+
encoder = codecs.getincrementalencoder('iso2022_jp')()
138+
139+
self.assertEqual(encoder.encode('z'), b'z')
140+
en_state = encoder.getstate()
141+
142+
self.assertEqual(encoder.encode('\u3042'), b'\x1b\x24\x42\x24\x22')
143+
jp_state = encoder.getstate()
144+
self.assertEqual(encoder.encode('z'), b'\x1b\x28\x42z')
145+
146+
encoder.setstate(jp_state)
147+
self.assertEqual(encoder.encode('\u3042'), b'\x24\x22')
148+
149+
encoder.setstate(en_state)
150+
self.assertEqual(encoder.encode('z'), b'z')
151+
152+
def test_getstate_returns_expected_value(self):
153+
# Note: getstate is implemented such that these state values
154+
# are expected to be the same across all builds of Python,
155+
# regardless of x32/64 bit, endianness and compiler.
156+
157+
# euc_jis_2004 stores state as a buffer of pending bytes
158+
buffer_state_encoder = codecs.getincrementalencoder('euc_jis_2004')()
159+
self.assertEqual(buffer_state_encoder.getstate(), 0)
160+
buffer_state_encoder.encode('\u00e6')
161+
self.assertEqual(buffer_state_encoder.getstate(),
162+
int.from_bytes(
163+
b"\x02"
164+
b"\xc3\xa6"
165+
b"\x00\x00\x00\x00\x00\x00\x00\x00",
166+
'little'))
167+
buffer_state_encoder.encode('\u0300')
168+
self.assertEqual(buffer_state_encoder.getstate(), 0)
169+
170+
# iso2022_jp stores state without using a buffer
171+
non_buffer_state_encoder = codecs.getincrementalencoder('iso2022_jp')()
172+
self.assertEqual(non_buffer_state_encoder.getstate(),
173+
int.from_bytes(
174+
b"\x00"
175+
b"\x42\x42\x00\x00\x00\x00\x00\x00",
176+
'little'))
177+
non_buffer_state_encoder.encode('\u3042')
178+
self.assertEqual(non_buffer_state_encoder.getstate(),
179+
int.from_bytes(
180+
b"\x00"
181+
b"\xc2\x42\x00\x00\x00\x00\x00\x00",
182+
'little'))
183+
184+
def test_setstate_validates_input_size(self):
185+
encoder = codecs.getincrementalencoder('euc_jp')()
186+
pending_size_nine = int.from_bytes(
187+
b"\x09"
188+
b"\x00\x00\x00\x00\x00\x00\x00\x00"
189+
b"\x00\x00\x00\x00\x00\x00\x00\x00",
190+
'little')
191+
self.assertRaises(UnicodeError, encoder.setstate, pending_size_nine)
192+
193+
def test_setstate_validates_input_bytes(self):
194+
encoder = codecs.getincrementalencoder('euc_jp')()
195+
invalid_utf8 = int.from_bytes(
196+
b"\x01"
197+
b"\xff"
198+
b"\x00\x00\x00\x00\x00\x00\x00\x00",
199+
'little')
200+
self.assertRaises(UnicodeDecodeError, encoder.setstate, invalid_utf8)
201+
120202
def test_issue5640(self):
121203
encoder = codecs.getincrementalencoder('shift-jis')('backslashreplace')
122204
self.assertEqual(encoder.encode('\xff'), b'\\xff')
@@ -165,6 +247,37 @@ def test_decode_unicode(self):
165247
decoder = codecs.getincrementaldecoder(enc)()
166248
self.assertRaises(TypeError, decoder.decode, "")
167249

250+
def test_state_methods(self):
251+
decoder = codecs.getincrementaldecoder('euc_jp')()
252+
253+
# Decode a complete input sequence
254+
self.assertEqual(decoder.decode(b'\xa4\xa6'), '\u3046')
255+
pending1, _ = decoder.getstate()
256+
self.assertEqual(pending1, b'')
257+
258+
# Decode first half of a partial input sequence
259+
self.assertEqual(decoder.decode(b'\xa4'), '')
260+
pending2, flags2 = decoder.getstate()
261+
self.assertEqual(pending2, b'\xa4')
262+
263+
# Decode second half of a partial input sequence
264+
self.assertEqual(decoder.decode(b'\xa6'), '\u3046')
265+
pending3, _ = decoder.getstate()
266+
self.assertEqual(pending3, b'')
267+
268+
# Jump back and decode second half of partial input sequence again
269+
decoder.setstate((pending2, flags2))
270+
self.assertEqual(decoder.decode(b'\xa6'), '\u3046')
271+
pending4, _ = decoder.getstate()
272+
self.assertEqual(pending4, b'')
273+
274+
def test_setstate_validates_input(self):
275+
decoder = codecs.getincrementaldecoder('euc_jp')()
276+
self.assertRaises(TypeError, decoder.setstate, 123)
277+
self.assertRaises(TypeError, decoder.setstate, ("invalid", 0))
278+
self.assertRaises(TypeError, decoder.setstate, (b"1234", "invalid"))
279+
self.assertRaises(UnicodeError, decoder.setstate, (b"123456789", 0))
280+
168281
class Test_StreamReader(unittest.TestCase):
169282
def test_bug1728403(self):
170283
try:

‎Misc/ACKS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1626,6 +1626,7 @@ Nicolas M. Thiéry
16261626
James Thomas
16271627
Robin Thomas
16281628
Brian Thorne
1629+
Christopher Thorne
16291630
Stephen Thorne
16301631
Jeremy Thurgood
16311632
Eric Tiedemann

‎Misc/NEWS.d/next/Library/2018-06-08-23-55-34.bpo-33578.7oSsjG.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Implement multibyte encoder/decoder state methods

‎Modules/cjkcodecs/_codecs_cn.c‎

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,12 @@
5151
; \
5252
}
5353

54+
/*
55+
* codecs in this file use the first byte of MultibyteCodec_State.c[8]
56+
* to store a 0 or 1 state value
57+
*/
58+
#define CN_STATE_OFFSET 0
59+
5460
/*
5561
* GB2312 codec
5662
*/
@@ -329,15 +335,15 @@ DECODER(gb18030)
329335

330336
ENCODER_INIT(hz)
331337
{
332-
state->i = 0;
338+
state->c[CN_STATE_OFFSET] = 0;
333339
return 0;
334340
}
335341

336342
ENCODER_RESET(hz)
337343
{
338-
if (state->i != 0) {
344+
if (state->c[CN_STATE_OFFSET] != 0) {
339345
WRITEBYTE2('~', '}');
340-
state->i = 0;
346+
state->c[CN_STATE_OFFSET] = 0;
341347
NEXT_OUT(2);
342348
}
343349
return 0;
@@ -350,10 +356,10 @@ ENCODER(hz)
350356
DBCHAR code;
351357

352358
if (c < 0x80) {
353-
if (state->i) {
359+
if (state->c[CN_STATE_OFFSET]) {
354360
WRITEBYTE2('~', '}');
355361
NEXT_OUT(2);
356-
state->i = 0;
362+
state->c[CN_STATE_OFFSET] = 0;
357363
}
358364
WRITEBYTE1((unsigned char)c);
359365
NEXT(1, 1);
@@ -375,10 +381,10 @@ ENCODER(hz)
375381
if (code & 0x8000) /* MSB set: GBK */
376382
return 1;
377383

378-
if (state->i == 0) {
384+
if (state->c[CN_STATE_OFFSET] == 0) {
379385
WRITEBYTE4('~', '{', code >> 8, code & 0xff);
380386
NEXT(1, 4);
381-
state->i = 1;
387+
state->c[CN_STATE_OFFSET] = 1;
382388
}
383389
else {
384390
WRITEBYTE2(code >> 8, code & 0xff);
@@ -391,13 +397,13 @@ ENCODER(hz)
391397

392398
DECODER_INIT(hz)
393399
{
394-
state->i = 0;
400+
state->c[CN_STATE_OFFSET] = 0;
395401
return 0;
396402
}
397403

398404
DECODER_RESET(hz)
399405
{
400-
state->i = 0;
406+
state->c[CN_STATE_OFFSET] = 0;
401407
return 0;
402408
}
403409

@@ -411,14 +417,14 @@ DECODER(hz)
411417
unsigned char c2 = INBYTE2;
412418

413419
REQUIRE_INBUF(2);
414-
if (c2 == '~' && state->i == 0)
420+
if (c2 == '~' && state->c[CN_STATE_OFFSET] == 0)
415421
OUTCHAR('~');
416-
else if (c2 == '{' && state->i == 0)
417-
state->i = 1; /* set GB */
418-
else if (c2 == '\n' && state->i == 0)
422+
else if (c2 == '{' && state->c[CN_STATE_OFFSET] == 0)
423+
state->c[CN_STATE_OFFSET] = 1; /* set GB */
424+
else if (c2 == '\n' && state->c[CN_STATE_OFFSET] == 0)
419425
; /* line-continuation */
420-
else if (c2 == '}' && state->i == 1)
421-
state->i = 0; /* set ASCII */
426+
else if (c2 == '}' && state->c[CN_STATE_OFFSET] == 1)
427+
state->c[CN_STATE_OFFSET] = 0; /* set ASCII */
422428
else
423429
return 1;
424430
NEXT_IN(2);
@@ -428,7 +434,7 @@ DECODER(hz)
428434
if (c & 0x80)
429435
return 1;
430436

431-
if (state->i == 0) { /* ASCII mode */
437+
if (state->c[CN_STATE_OFFSET] == 0) { /* ASCII mode */
432438
OUTCHAR(c);
433439
NEXT_IN(1);
434440
}

‎Modules/cjkcodecs/clinic/multibytecodec.c.h‎

Lines changed: 89 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,50 @@ _multibytecodec_MultibyteIncrementalEncoder_encode(MultibyteIncrementalEncoderOb
115115
return return_value;
116116
}
117117

118+
PyDoc_STRVAR(_multibytecodec_MultibyteIncrementalEncoder_getstate__doc__,
119+
"getstate($self, /)\n"
120+
"--\n"
121+
"\n");
122+
123+
#define _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_GETSTATE_METHODDEF \
124+
{"getstate", (PyCFunction)_multibytecodec_MultibyteIncrementalEncoder_getstate, METH_NOARGS, _multibytecodec_MultibyteIncrementalEncoder_getstate__doc__},
125+
126+
static PyObject *
127+
_multibytecodec_MultibyteIncrementalEncoder_getstate_impl(MultibyteIncrementalEncoderObject *self);
128+
129+
static PyObject *
130+
_multibytecodec_MultibyteIncrementalEncoder_getstate(MultibyteIncrementalEncoderObject *self, PyObject *Py_UNUSED(ignored))
131+
{
132+
return _multibytecodec_MultibyteIncrementalEncoder_getstate_impl(self);
133+
}
134+
135+
PyDoc_STRVAR(_multibytecodec_MultibyteIncrementalEncoder_setstate__doc__,
136+
"setstate($self, state, /)\n"
137+
"--\n"
138+
"\n");
139+
140+
#define _MULTIBYTECODEC_MULTIBYTEINCREMENTALENCODER_SETSTATE_METHODDEF \
141+
{"setstate", (PyCFunction)_multibytecodec_MultibyteIncrementalEncoder_setstate, METH_O, _multibytecodec_MultibyteIncrementalEncoder_setstate__doc__},
142+
143+
static PyObject *
144+
_multibytecodec_MultibyteIncrementalEncoder_setstate_impl(MultibyteIncrementalEncoderObject *self,
145+
PyLongObject *statelong);
146+
147+
static PyObject *
148+
_multibytecodec_MultibyteIncrementalEncoder_setstate(MultibyteIncrementalEncoderObject *self, PyObject *arg)
149+
{
150+
PyObject *return_value = NULL;
151+
PyLongObject *statelong;
152+
153+
if (!PyArg_Parse(arg, "O!:setstate", &PyLong_Type, &statelong)) {
154+
goto exit;
155+
}
156+
return_value = _multibytecodec_MultibyteIncrementalEncoder_setstate_impl(self, statelong);
157+
158+
exit:
159+
return return_value;
160+
}
161+
118162
PyDoc_STRVAR(_multibytecodec_MultibyteIncrementalEncoder_reset__doc__,
119163
"reset($self, /)\n"
120164
"--\n"
@@ -169,6 +213,50 @@ _multibytecodec_MultibyteIncrementalDecoder_decode(MultibyteIncrementalDecoderOb
169213
return return_value;
170214
}
171215

216+
PyDoc_STRVAR(_multibytecodec_MultibyteIncrementalDecoder_getstate__doc__,
217+
"getstate($self, /)\n"
218+
"--\n"
219+
"\n");
220+
221+
#define _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_GETSTATE_METHODDEF \
222+
{"getstate", (PyCFunction)_multibytecodec_MultibyteIncrementalDecoder_getstate, METH_NOARGS, _multibytecodec_MultibyteIncrementalDecoder_getstate__doc__},
223+
224+
static PyObject *
225+
_multibytecodec_MultibyteIncrementalDecoder_getstate_impl(MultibyteIncrementalDecoderObject *self);
226+
227+
static PyObject *
228+
_multibytecodec_MultibyteIncrementalDecoder_getstate(MultibyteIncrementalDecoderObject *self, PyObject *Py_UNUSED(ignored))
229+
{
230+
return _multibytecodec_MultibyteIncrementalDecoder_getstate_impl(self);
231+
}
232+
233+
PyDoc_STRVAR(_multibytecodec_MultibyteIncrementalDecoder_setstate__doc__,
234+
"setstate($self, state, /)\n"
235+
"--\n"
236+
"\n");
237+
238+
#define _MULTIBYTECODEC_MULTIBYTEINCREMENTALDECODER_SETSTATE_METHODDEF \
239+
{"setstate", (PyCFunction)_multibytecodec_MultibyteIncrementalDecoder_setstate, METH_O, _multibytecodec_MultibyteIncrementalDecoder_setstate__doc__},
240+
241+
static PyObject *
242+
_multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDecoderObject *self,
243+
PyObject *state);
244+
245+
static PyObject *
246+
_multibytecodec_MultibyteIncrementalDecoder_setstate(MultibyteIncrementalDecoderObject *self, PyObject *arg)
247+
{
248+
PyObject *return_value = NULL;
249+
PyObject *state;
250+
251+
if (!PyArg_Parse(arg, "O!:setstate", &PyTuple_Type, &state)) {
252+
goto exit;
253+
}
254+
return_value = _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(self, state);
255+
256+
exit:
257+
return return_value;
258+
}
259+
172260
PyDoc_STRVAR(_multibytecodec_MultibyteIncrementalDecoder_reset__doc__,
173261
"reset($self, /)\n"
174262
"--\n"
@@ -330,4 +418,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__,
330418

331419
#define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \
332420
{"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__},
333-
/*[clinic end generated code: output=680f59f4cfe63c25 input=a9049054013a1b77]*/
421+
/*[clinic end generated code: output=2fa0a38494716b97 input=a9049054013a1b77]*/

0 commit comments

Comments
 (0)