Skip to content

Commit d2fff96

Browse files
dhondtadhondta
committed
Improved input/output text handling
1 parent c00d3aa commit d2fff96

7 files changed

Lines changed: 41 additions & 26 deletions

File tree

‎codext/__common__.py‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ class StreamReader(Codec, codecs.StreamReader):
154154
register(getregentry, add_to_codecs)
155155

156156

157-
def add_map(ename, encmap, repl_char="?", sep="", ignore_case=None, no_error=False, binary=False, **kwargs):
157+
def add_map(ename, encmap, repl_char="?", sep="", ignore_case=None, no_error=False, intype=None, outype=None, **kwargs):
158158
"""
159159
This adds a new mapping codec (that is, declarable with a simple character mapping dictionary) to the codecs module
160160
dynamically setting its encode and/or decode functions, eventually dynamically naming the encoding with a pattern
@@ -169,15 +169,21 @@ def add_map(ename, encmap, repl_char="?", sep="", ignore_case=None, no_error=Fal
169169
- while decoding, separators can be mixed in the input text
170170
:param ignore_case: ignore text case while encoding and/or decoding
171171
:param no_error: this encoding triggers no error (hence, always in "leave" errors handling)
172-
:param binary: encoding applies to the binary string of the input text
172+
:param intype: specify the input type for pre-transforming the input text
173+
:param outype: specify the output type for post-transforming the output text
173174
:param pattern: pattern for dynamically naming the encoding
174175
:param text: specify whether the codec is a text encoding
175176
:param add_to_codecs: also add the search function to the native registry
176177
NB: this will make the codec available in the built-in open(...) but will make it impossible
177178
to remove the codec later
178179
"""
180+
outype = outype or intype
179181
if ignore_case not in [None, "encode", "decode", "both"]:
180182
raise ValueError("Bad ignore_case parameter while creating encoding map")
183+
if intype not in [None, "str", "bin", "ord"]:
184+
raise ValueError("Bad input type parameter while creating encoding map")
185+
if outype not in [None, "str", "bin", "ord"]:
186+
raise ValueError("Bad output type parameter while creating encoding map")
181187
def __generic_code(exc, decode=False):
182188
def _wrapper(param):
183189
"""
@@ -293,8 +299,11 @@ def code(text, errors="strict"):
293299
if no_error:
294300
errors = "leave"
295301
text = ensure_str(text)
296-
if binary and not decode:
297-
text = "".join("{:0>8}".format(bin(ord(c))[2:]) for c in text)
302+
if not decode:
303+
if intype == "bin":
304+
text = "".join("{:0>8}".format(bin(ord(c))[2:]) for c in text)
305+
elif intype == "ord":
306+
text = "".join(str(ord(c)).zfill(3) for c in text)
298307
r = ""
299308
lsep = "" if decode else sep if len(sep) <= 1 else sep[0]
300309

@@ -336,16 +345,18 @@ def __get_value(token, position, case_changed=False):
336345
posn = cursor - len(bad)
337346
r += handle_error(ename, errors, exc, lsep, repl_char, rminlen, decode)(bad, posn)
338347
bad = ""
339-
if binary and decode:
340-
tmp, r = "", r.replace(lsep, "")
341-
for i in range(0, len(r), 8):
342-
bs = r[i:i+8]
343-
try:
344-
tmp += chr(int(bs, 2))
345-
except ValueError:
346-
if len(bs) > 0:
347-
tmp += "[" + bs + "]"
348-
r = tmp + lsep
348+
if decode:
349+
if outype in ["bin", "ord"]:
350+
tmp, r = "", r.replace(lsep, "")
351+
step = [3, 8][outype == "bin"]
352+
for i in range(0, len(r), step):
353+
s = r[i:i+step]
354+
try:
355+
tmp += chr(int(s, 2) if outype == "bin" else int(s))
356+
except ValueError:
357+
if len(s) > 0:
358+
tmp += "[" + s + "]"
359+
r = tmp + lsep
349360
return r[:len(r)-len(lsep)], len(b(text))
350361
return code
351362
if re.search(r"\([^(?:)]", kwargs.get('pattern', "")) is None:
@@ -361,14 +372,15 @@ def __get_value(token, position, case_changed=False):
361372
encexc = "{}EncodeError".format(name)
362373
exec("class {}(ValueError): pass".format(encexc), glob)
363374
# now use the generic add() function
364-
kwargs['type'] = glob['__file__'].split(os.path.sep)[-2].rstrip("s")
375+
kwargs['category'] = glob['__file__'].split(os.path.sep)[-2].rstrip("s")
365376
kwargs['examples'] = kwargs.get('examples', glob.get('__examples__'))
366377
kwargs['encmap'] = encmap
367378
kwargs['repl_char'] = repl_char
368379
kwargs['sep'] = sep
369380
kwargs['ignore_case'] = ignore_case
370381
kwargs['no_error'] = no_error
371-
kwargs['binary'] = binary
382+
kwargs['intype'] = intype
383+
kwargs['outype'] = outype
372384
try:
373385
if isinstance(encmap, dict):
374386
smapdict = {k: v for k, v in encmap.items()}

‎codext/binary/excess3.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,5 +53,5 @@ def excess3_decode(text, errors="strict"):
5353
return r, len(b(text))
5454

5555

56-
add("excess3", excess3_encode, excess3_decode, pattern=r"^(?:excess\-?3|xs\-?3|stibitz)$", text=False)
56+
add("excess3", excess3_encode, excess3_decode, pattern=r"^(?:excess\-?3|xs\-?3|stibitz)$")
5757

‎codext/others/dna.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,5 +37,5 @@
3737
ENCMAP.append({k: v[i] for k, v in SEQUENCES.items()})
3838

3939

40-
add_map("dna", ENCMAP, binary=True, pattern=r"dna[-_]?([1-8])$")
40+
add_map("dna", ENCMAP, intype="bin", pattern=r"dna[-_]?([1-8])$")
4141

‎codext/stegano/resistor.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,17 @@
1111

1212

1313
__examples__ = {
14-
'enc(resistor_color)': {'test': None},
1514
'enc(resistor|resistor_color|resistor_color_code|resistors-color-code)': {
16-
'1234': "\x1b[48;5;130m \x1b[0;00m\x1b[48;5;1m \x1b[0;00m\x1b[48;5;214m \x1b[0;00m\x1b[48;5;11m \x1b[0;00m"
15+
'Test': "\x1b[48;5;232m \x1b[0;00m\x1b[48;5;245m \x1b[0;00m\x1b[48;5;11m \x1b[0;00m\x1b[48;5;130m "
16+
"\x1b[0;00m\x1b[48;5;232m \x1b[0;00m\x1b[48;5;130m \x1b[0;00m\x1b[48;5;130m \x1b[0;00m\x1b[48;5;130m "
17+
"\x1b[0;00m\x1b[48;5;2m \x1b[0;00m\x1b[48;5;130m \x1b[0;00m\x1b[48;5;130m \x1b[0;00m\x1b[48;5;4m "
18+
"\x1b[0;00m"
1719
},
1820
}
1921

2022

2123
ENCMAP = {i: "\033[48;5;%dm \033[0;00m" % c for i, c in zip("0123456789", [232, 130, 1, 214, 11, 2, 4, 129, 245, 231])}
22-
ENCMAP[' '] = "/"
2324

2425

25-
add_map("resistor", ENCMAP, pattern=r"^resistors?(?:[-_]color(?:[-_]code)?)?$")
26+
add_map("resistor", ENCMAP, intype="ord", pattern=r"^resistors?(?:[-_]color(?:[-_]code)?)?$")
2627

‎codext/stegano/whitespace.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222

2323
ENCMAP = {r'': {'0': "\t", '1': " "}, r'[-_]inv(erted)?': {'0': " ", '1': "\t"}}
24-
add_map("whitespace", ENCMAP, binary=True, pattern=r"^whitespaces?([-_]inv(?:erted)?)?$", examples=__examples1__)
24+
add_map("whitespace", ENCMAP, intype="bin", pattern=r"^whitespaces?([-_]inv(?:erted)?)?$", examples=__examples1__)
2525

2626

2727
def wsba_encode(p):

‎tests/test_common.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,10 @@ def test_add_map_codec(self):
4545
self.assertIsNone(codext.add_map("dummy3", ENCMAP, pattern=r"^dummy3([-_]inverted)?$"))
4646
self.assertRaises(LookupError, codext.encode, "test", "dummy3_inverted")
4747
self.assertRaises(ValueError, codext.add_map, "dummy2", ENCMAP, ignore_case="BAD")
48+
self.assertRaises(ValueError, codext.add_map, "dummy2", ENCMAP, intype="BAD")
49+
self.assertRaises(ValueError, codext.add_map, "dummy2", ENCMAP, outype="BAD")
4850
ci = codext.lookup("dummy2")
49-
for k in ["binary", "encmap", "examples", "ignore_case", "no_error", "repl_char", "sep", "text", "type"]:
51+
for k in ["category", "encmap", "ignore_case", "intype", "no_error", "outype", "repl_char", "sep", "text"]:
5052
self.assertIn(k, ci.parameters.keys())
5153

5254
def test_remove_codec(self):

‎tests/test_generated.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,9 @@ def _template(self):
3939
if examples is None:
4040
self.assertRaises(LookupError, f1, "test", ename)
4141
continue
42-
# unhandled character error tests (only for non-binary-related encodings)
42+
# unhandled character error tests
4343
encmap = params.get('encmap')
44-
if encmap and not params['binary'] and not params['no_error']:
44+
if encmap and params['intype'] not in ["bin", "ord"] and not params['no_error']:
4545
if not isinstance(encmap, list):
4646
encmap = [encmap]
4747
for em in encmap:

0 commit comments

Comments
 (0)