Repository navigation
Expand file tree
/
Copy pathbytes_decode_replace.py
More file actions
130 lines (99 loc) · 4.36 KB
/
Copy pathbytes_decode_replace.py
File metadata and controls
130 lines (99 loc) · 4.36 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
# Test bytes.decode() with error handler 'replace'
# Check if decode method is available (requires MICROPY_CPYTHON_COMPAT)
try:
b"".decode()
except AttributeError:
print("SKIP")
raise SystemExit
# Check if error handlers are available (requires MICROPY_PY_BUILTINS_BYTES_DECODE_ERRORS)
# When feature is disabled, invalid UTF-8 raises UnicodeError even with 'replace'
# When feature is enabled, invalid UTF-8 with 'replace' returns a string
try:
result = b"\xff".decode("utf-8", "replace")
# If we get here, feature is available
except (UnicodeError, LookupError):
# Feature not available - 'replace' was ignored, strict mode was used
print("SKIP")
raise SystemExit
# Test replace mode with invalid UTF-8
print(repr(b"\xff\xfe".decode("utf-8", "replace")))
# Test strict mode (default) with invalid UTF-8
try:
b"\xff\xfe".decode("utf-8")
print("UNEXPECTED")
except UnicodeError:
print("UnicodeError")
# Test strict mode (explicit) with invalid UTF-8
try:
b"\xff\xfe".decode("utf-8", "strict")
print("UNEXPECTED")
except UnicodeError:
print("UnicodeError")
# Test with valid UTF-8
print(repr(b"hello".decode("utf-8", "replace")))
# Test valid UTF-8 with default mode
print(repr(b"hello".decode("utf-8")))
# Test mixed valid and invalid UTF-8
print(repr(b"hello\xffworld".decode("utf-8", "replace")))
# Test multiple invalid bytes
print(repr(b"\x80\x81\x82".decode("utf-8", "replace")))
# Test invalid continuation byte
print(repr(b"\xc0\x20".decode("utf-8", "replace")))
# Test incomplete sequence at end
print(repr(b"hello\xc0".decode("utf-8", "replace")))
# Test valid multi-byte UTF-8 (© symbol)
print(repr(b"\xc2\xa9".decode("utf-8", "replace")))
# Test bytearray support
print(repr(bytearray(b"\xff\xfe").decode("utf-8", "replace")))
# Test replace mode - should either work or raise NotImplementedError
try:
result = b"\xff\xfe".decode("utf-8", "replace")
print(repr(result))
except LookupError:
print("LookupError")
# Test replace with valid UTF-8
try:
result = b"hello".decode("utf-8", "replace")
print(repr(result))
except LookupError:
print("LookupError")
# Test replace with mixed content
try:
result = b"hello\xffworld".decode("utf-8", "replace")
print(repr(result))
except LookupError:
print("LookupError")
# Additional tests for continuation byte validation and incomplete sequences
# Test 3-byte UTF-8 sequence - valid (e.g., U+4E00 - 一)
print(repr(b"\xe4\xb8\x80".decode("utf-8", "replace")))
# Test 4-byte UTF-8 sequence - valid (e.g., U+1F600 - 😀)
print(repr(b"\xf0\x9f\x98\x80".decode("utf-8", "replace")))
# Test valid multi-byte sequence mixed with invalid bytes (exercises got==need path)
print(repr(b"\xff\xc2\xa9".decode("utf-8", "replace"))) # \ufffd + © after invalid \xff
print(repr(b"\xff\xe4\xb8\x80".decode("utf-8", "replace"))) # \ufffd + 一 after invalid \xff
# Test incomplete 3-byte sequence (missing 2 continuation bytes)
print(repr(b"\xe4".decode("utf-8", "replace")))
# Test incomplete 3-byte sequence (missing 1 continuation byte)
print(repr(b"\xe4\xb8".decode("utf-8", "replace")))
# Test incomplete 4-byte sequence (missing 3 continuation bytes)
print(repr(b"\xf0".decode("utf-8", "replace")))
# Test incomplete 4-byte sequence (missing 2 continuation bytes)
print(repr(b"\xf0\x9f".decode("utf-8", "replace")))
# Test incomplete 4-byte sequence (missing 1 continuation byte)
print(repr(b"\xf0\x9f\x98".decode("utf-8", "replace")))
# Test 3-byte sequence with invalid continuation byte (first byte invalid)
print(repr(b"\xe4\x20\x80".decode("utf-8", "replace")))
# Test 3-byte sequence with invalid continuation byte (second byte invalid)
print(repr(b"\xe4\xb8\x20".decode("utf-8", "replace")))
# Test 4-byte sequence with invalid continuation bytes
print(repr(b"\xf0\x20\x98\x80".decode("utf-8", "replace")))
print(repr(b"\xf0\x9f\x20\x80".decode("utf-8", "replace")))
print(repr(b"\xf0\x9f\x98\x20".decode("utf-8", "replace")))
# Test mixed valid and incomplete sequences
print(repr(b"hello\xe4world".decode("utf-8", "replace")))
print(repr(b"hello\xf0world".decode("utf-8", "replace")))
# Test multiple incomplete sequences in a row
print(repr(b"\xe4\xf0\xe4".decode("utf-8", "replace")))
# Test replacing invalid ASCII
print(repr(b"\xe5\xa5\xbd".decode("ascii", "replace"))) # valid utf8, not valid ascii
print(repr(b"a\xbb\xcc\xddef\x01".decode("ascii", "replace"))) # fully invalid