Skip to content

Commit de2f7af

Browse files
dhondtadhondta
committed
Modified __common__ code base and improved token case handling
1 parent 9d97507 commit de2f7af

7 files changed

Lines changed: 75 additions & 28 deletions

File tree

‎codext/__common__.py‎

Lines changed: 69 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ class StreamReader(Codec, codecs.StreamReader):
137137
register(getregentry, add_to_codecs)
138138

139139

140-
def add_map(ename, encmap, repl_char="?", sep="", ignore_case=False, no_error=False, binary=False, **kwargs):
140+
def add_map(ename, encmap, repl_char="?", sep="", ignore_case=None, no_error=False, binary=False, **kwargs):
141141
"""
142142
This adds a new mapping codec (that is, declarable with a simple character mapping dictionary) to the codecs module
143143
dynamically setting its encode and/or decode functions, eventually dynamically naming the encoding with a pattern
@@ -150,7 +150,7 @@ def add_map(ename, encmap, repl_char="?", sep="", ignore_case=False, no_error=Fa
150150
:param sep: string of possible character separators (hence, only single-char separators are considered) ;
151151
- while encoding, the first separator is used
152152
- while decoding, separators can be mixed in the input text
153-
:param ignore_case: ignore text case
153+
:param ignore_case: ignore text case while encoding and/or decoding
154154
:param no_error: this encoding triggers no error (hence, always in "leave" errors handling)
155155
:param binary: encoding applies to the binary string of the input text
156156
:param pattern: pattern for dynamically naming the encoding
@@ -159,6 +159,8 @@ def add_map(ename, encmap, repl_char="?", sep="", ignore_case=False, no_error=Fa
159159
NB: this will make the codec available in the built-in open(...) but will make it impossible
160160
to remove the codec later
161161
"""
162+
if ignore_case not in [None, "encode", "decode", "both"]:
163+
raise ValueError("Bad ignore_case parameter while creating encoding map")
162164
def __generic_code(mapdict, exc, decode=False):
163165
def _wrapper(param):
164166
"""
@@ -227,11 +229,9 @@ def _wrapper(param):
227229
smapdict[k] = v.translate(t)
228230
else:
229231
raise LookupError("Bad parameter for encoding '{}': '{}'".format(ename, p))
230-
if ignore_case:
231-
case = ["upper", "lower"][any(c in "".join(smapdict.keys()) for c in "abcdefghijklmnopqrstuvwxyz")]
232-
# use the first mapped group from the mapping dictionary to determine token length ; this is useful e.g. for
233-
# tokenizing a binary string when the text is to be converted as binary
234-
tlen = len(list(set(smapdict.keys()) - {""})[0])
232+
if ignore_case is not None:
233+
case_d = ["upper", "lower"][any(c in "".join(smapdict.values()) for c in "abcdefghijklmnopqrstuvwxyz")]
234+
case_e = ["upper", "lower"][any(c in "".join(smapdict.keys()) for c in "abcdefghijklmnopqrstuvwxyz")]
235235
if decode:
236236
tmp = {}
237237
# this has a meaning for encoding maps that could have clashes in encoded chars (e.g. Bacon's cipher ;
@@ -243,34 +243,80 @@ def _wrapper(param):
243243
# this allows to avoid an error with Python2 in the "for i, c in enumerate(parts)" loop
244244
if '' not in smapdict.keys():
245245
smapdict[''] = ""
246+
# determine token and result lengths
247+
tmaxlen = max(map(len, smapdict.keys()))
248+
tminlen = max(1, min(map(len, set(smapdict.keys()) - {''})))
249+
rminlen = max(1, min(map(len, set(smapdict.values()) - {''})))
246250

251+
# generic encoding/decoding function for map encodings
247252
def code(text, errors="strict"):
248-
if ignore_case:
249-
text = getattr(text, case)()
253+
icase = ignore_case == "both" or \
254+
decode and ignore_case == "decode" or \
255+
not decode and ignore_case == "encode"
256+
if icase:
257+
case = case_d if decode else case_e
250258
if no_error:
251259
errors = "leave"
252260
text = ensure_str(text)
253261
if binary and not decode:
254262
text = "".join("{:0>8}".format(bin(ord(c))[2:]) for c in text)
255-
text = [text[i:i+tlen] for i in range(0, len(text), tlen)]
256-
parts = re.split("[" + sep + "]", text) if decode and len(sep) > 0 else text
257263
r = ""
258264
lsep = "" if decode else sep if len(sep) <= 1 else sep[0]
259-
for i, c in enumerate(parts):
265+
266+
# get the value from the mapping dictionary, trying the token with its inverted case if relevant
267+
def __get_value(token, position, case_changed=False):
260268
try:
261-
r += smapdict[c] + lsep
269+
return smapdict[token] + lsep
262270
except KeyError:
263-
if errors == "strict":
264-
raise exc("'{}' codec can't {}code character '{}' in position {}"
265-
.format(ename, ["en", "de"][decode], c, i))
266-
elif errors == "leave":
267-
r += c + lsep
268-
elif errors == "replace":
269-
r += repl_char * [1, tlen][decode] + lsep
270-
elif errors == "ignore":
271-
continue
271+
if icase and not case_changed:
272+
token_inv_case = getattr(token, case)()
273+
r = __get_value(token_inv_case, position, True)
274+
if r == token_inv_case + lsep and errors == "leave":
275+
return token + lsep
276+
return r
277+
return __handle_error(token, position)
278+
279+
def __handle_error(token, position):
280+
if errors == "strict":
281+
raise exc("'{}' codec can't {}code character '{}' in position {}"
282+
.format(ename, ["en", "de"][decode], token, position))
283+
elif errors == "leave":
284+
return token + lsep
285+
elif errors == "replace":
286+
return repl_char * rminlen + lsep
287+
elif errors == "ignore":
288+
return ""
289+
else:
290+
raise ValueError("Unsupported error handling '{}'".format(errors))
291+
292+
# if a separator is defined, rely on it by splitting the input text
293+
if decode and len(sep) > 0:
294+
for i, c in enumerate(re.split("[" + sep + "]", text)):
295+
r += __get_value(c, i)
296+
# otherwise, move through the text using a cursor for tokenizing it ; this allows defining more complex
297+
# encodings with variable token lengths
298+
else:
299+
cursor, bad = 0, ""
300+
while cursor < len(text):
301+
token = text[cursor:cursor+1]
302+
for l in range(tminlen, tmaxlen + 1):
303+
token = text[cursor:cursor+l]
304+
if token in smapdict.keys() or icase and getattr(token, case)() in smapdict.keys():
305+
# do not forget to handle bad chars already collected at this point
306+
if len(bad) > 0:
307+
r += __get_value(bad, cursor - len(bad))
308+
bad = ""
309+
r += __get_value(token, cursor)
310+
cursor += l
311+
break
272312
else:
273-
raise ValueError("Unsupported error handling '{}'".format(errors))
313+
# collect bad chars and only move the cursor one char to the right
314+
bad += text[cursor]
315+
cursor += 1
316+
# if the number of bad chars is the minimum token length, consume it and start a new buffer
317+
if len(bad) == tminlen:
318+
r += __handle_error(bad, cursor - len(bad))
319+
bad = ""
274320
if binary and decode:
275321
tmp, r = "", r.replace(lsep, "")
276322
for i in range(0, len(r), 8):

‎codext/crypto/bacon.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,4 @@
2020
}
2121

2222

23-
add_map("bacon", ENCMAP, sep=" ", ignore_case=True, pattern=r"bacon(?:(?:ian)?[-_]cipher)?([\-_].{2})?$")
23+
add_map("bacon", ENCMAP, sep=" ", ignore_case="encode", pattern=r"bacon(?:(?:ian)?[-_]cipher)?([\-_].{2})?$")

‎codext/languages/braille.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,4 @@
2525

2626

2727
if PY3:
28-
add_map("braille", ENCMAP, ignore_case=True)
28+
add_map("braille", ENCMAP, ignore_case="encode")

‎codext/languages/morse.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,4 @@
2727
}
2828

2929

30-
add_map("morse", ENCMAP, "#", " ", ignore_case=True, pattern=r"^morse([-_]?.{3})?$")
30+
add_map("morse", ENCMAP, "#", " ", ignore_case="encode", pattern=r"^morse([-_]?.{3})?$")

‎codext/stegano/nokia.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,4 @@
1717
}
1818

1919

20-
add_map("nokia", ENCMAP, "?", "-_", ignore_case=True, pattern=r"^nokia[-_]?3310$")
20+
add_map("nokia", ENCMAP, "?", "-_", ignore_case="encode", pattern=r"^nokia[-_]?3310$")

‎tests/test_common.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ def test_add_map_codec(self):
4242
ENCMAP = {'': {'a': "A", 'b': "B"}, r'bad': {'a': "B", 'b': "A"}}
4343
self.assertIsNone(codext.add_map("dummy3", ENCMAP, pattern=r"^dummy3([-_]inverted)?$"))
4444
self.assertRaises(LookupError, codext.encode, "test", "dummy3_inverted")
45+
self.assertRaises(ValueError, codext.add_map, "dummy2", ENCMAP, ignore_case="BAD")
4546

4647
def test_remove_codec(self):
4748
self.assertIsNone(codext.add("dummy", dummy_encode, dummy_decode))

‎tests/test_nokia.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,4 @@ def test_codec_nokia3310(self):
2525
self.assertEqual(codecs.encode("/", "nokia3310", errors="ignore"), "")
2626
self.assertRaises(ValueError, codecs.decode, "A", "nokia3310", errors="BAD")
2727
self.assertEqual(codecs.decode("A-B-222-3-4-5", "nokia3310", "replace"), "??cdgj")
28-
self.assertEqual(codecs.decode("A-B-222-3-4-5", "nokia3310", "leave"), "abcdgj")
28+
self.assertEqual(codecs.decode("A-B-222-3-4-5", "nokia3310", "leave"), "ABcdgj")

0 commit comments

Comments
 (0)