Skip to content

Commit 0eef213

Browse files
dhondtadhondta
committed
Added add_map for adding encoding mappings
1 parent 72e1817 commit 0eef213

2 files changed

Lines changed: 166 additions & 27 deletions

File tree

‎codext/__common__.py‎

Lines changed: 164 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import sys
77
from functools import wraps
88
from importlib import import_module
9+
from inspect import currentframe
910
from six import binary_type, string_types, text_type
1011
from types import FunctionType
1112
try: # Python3
@@ -16,10 +17,14 @@
1617
from importlib import reload
1718
except ImportError:
1819
pass
20+
try: # Python 2
21+
from string import maketrans
22+
except ImportError: # Python 3
23+
maketrans = str.maketrans
1924

2025

21-
__all__ = ["add", "b", "clear", "codecs", "ensure_str", "re", "register",
22-
"remove", "reset", "s2i", "PY3"]
26+
__all__ = ["add", "add_map", "b", "clear", "codecs", "ensure_str", "maketrans", "re", "register", "remove", "reset",
27+
"s2i", "PY3"]
2328
CODECS_REGISTRY = None
2429
PY3 = sys.version[0] == "3"
2530
__codecs_registry = []
@@ -32,22 +37,19 @@
3237
s2i = lambda s: int(codecs.encode(s, "base16"), 16)
3338

3439

35-
def add(ename, encode=None, decode=None, pattern=None, text=True,
36-
add_to_codecs=False):
40+
def add(ename, encode=None, decode=None, pattern=None, text=True, add_to_codecs=False):
3741
"""
38-
This adds a new codec to the codecs module setting its encode and/or decode
39-
functions, eventually dynamically naming the encoding with a pattern and
40-
with file handling (if text is True).
42+
This adds a new codec to the codecs module setting its encode and/or decode functions, eventually dynamically naming
43+
the encoding with a pattern and with file handling (if text is True).
4144
42-
:param ename: encoding name
43-
:param encode: encoding function or None
44-
:param decode: decoding function or None
45-
:param pattern: pattern for dynamically naming the encoding
46-
:param text: specify whether the codec is a text encoding
47-
:param add_to_codecs: also add the search function to the native registry
48-
NB: this will make the codec available in the
49-
built-in open(...) but will make it impossible
50-
to remove the codec later
45+
:param ename: encoding name
46+
:param encode: encoding function or None
47+
:param decode: decoding function or None
48+
:param pattern: pattern for dynamically naming the encoding
49+
:param text: specify whether the codec is a text encoding
50+
:param add_to_codecs: also add the search function to the native registry
51+
NB: this will make the codec available in the built-in open(...) but will make it impossible
52+
to remove the codec later
5153
"""
5254
if encode and not isinstance(encode, FunctionType):
5355
raise ValueError("Bad 'encode' function")
@@ -62,12 +64,12 @@ def getregentry(encoding):
6264
fenc, fdec, name = encode, decode, encoding
6365
# prepare CodecInfo input arguments
6466
class Codec(codecs.Codec):
65-
def encode(self, input, errors='strict'):
67+
def encode(self, input, errors="strict"):
6668
if fenc is None:
6769
raise NotImplementedError
6870
return fenc(input, errors)
6971

70-
def decode(self, input, errors='strict'):
72+
def decode(self, input, errors="strict"):
7173
if fdec is None:
7274
raise NotImplementedError
7375
return fdec(input, errors)
@@ -91,17 +93,16 @@ def decode(self, input, final=False):
9193
if pattern:
9294
m = re.match(pattern, encoding)
9395
try:
94-
g = m.group(1)
96+
g = m.group(1) or ""
9597
if g.isdigit():
9698
g = int(g)
9799
fenc = fenc(g) if fenc else fenc
98100
fdec = fdec(g) if fdec else fdec
99101
except AttributeError:
100102
return # this occurs when m is None, meaning no match
101103
except IndexError:
102-
# this occurs while m is not None, but possibly no capture group
103-
# that gives at least 1 group index ; in this case, if
104-
# fenc/fdec is a decorated function, execute it with no arg
104+
# this occurs while m is not None, but possibly no capture group that gives at least 1 group index ; in
105+
# this case, if fenc/fdec is a decorated function, execute it with no arg
105106
if fenc and len(getfullargspec(fenc).args) == 1:
106107
fenc = fenc()
107108
if fdec and len(getfullargspec(fdec).args) == 1:
@@ -136,6 +137,146 @@ class StreamReader(Codec, codecs.StreamReader):
136137
register(getregentry, add_to_codecs)
137138

138139

140+
def add_map(ename, encmap, repl_char="?", sep="", ignore_case=False, no_error=False, binary=False, **kwargs):
141+
"""
142+
This adds a new mapping codec (that is, declarable with a simple character mapping dictionary) to the codecs module
143+
dynamically setting its encode and/or decode functions, eventually dynamically naming the encoding with a pattern
144+
and with file handling (if text is True).
145+
146+
:param ename: encoding name
147+
:param encmap: characters encoding map ; can be a dictionary of encoding maps (for use with the first capture
148+
group of the regex pattern)
149+
:param repl_char: replacement char (used when errors handling is set to "replace")
150+
:param sep: string of possible character separators (hence, only single-char separators are considered) ;
151+
- while encoding, the first separator is used
152+
- while decoding, separators can be mixed in the input text
153+
:param ignore_case: ignore text case
154+
:param no_error: this encoding triggers no error (hence, always in "leave" errors handling)
155+
:param binary: encoding applies to the binary string of the input text
156+
:param pattern: pattern for dynamically naming the encoding
157+
:param text: specify whether the codec is a text encoding
158+
:param add_to_codecs: also add the search function to the native registry
159+
NB: this will make the codec available in the built-in open(...) but will make it impossible
160+
to remove the codec later
161+
"""
162+
def __generic_code(mapdict, exc, decode=False):
163+
def _wrapper(param):
164+
"""
165+
The parameter for wrapping comes from the encoding regex pattern ; e.g.
166+
[no pattern] => param will be None everytime
167+
r"barbie[-_]?([1-4])$" => param could be int 1, 2, 3 or 4
168+
r"^morse(|[-_]?.{3})$" => param could be None, "-ABC" (for mapping to ".-/")
169+
170+
In order of precedence:
171+
1. when param is a key in mapdict or mapdict is a list of encoding maps (hence in the case of "barbie...",
172+
param MUST be an int, otherwise for the first case it could clash with a character of the encoding map)
173+
2. otherwise handle it as a new encoding character map "ABC" translates to ".-/" for morse
174+
"""
175+
if isinstance(mapdict, dict):
176+
smapdict = {k: v for k, v in mapdict.items()}
177+
elif isinstance(mapdict, list) and isinstance(mapdict[0], dict):
178+
smapdict = {k: v for k, v in mapdict[0].items()}
179+
else:
180+
raise ValueError("Bad mapping dictionary or list of mapping dictionaries")
181+
if param:
182+
# case 1: list or dictionary of parameter-dependent encodings
183+
if isinstance(param, int):
184+
if isinstance(mapdict, list):
185+
param -= 1
186+
if isinstance(mapdict, list) and 0 <= param < len(mapdict) or \
187+
isinstance(mapdict, dict) and param in mapdict.keys():
188+
smapdict = mapdict[param]
189+
else:
190+
raise LookupError("Bad parameter for encoding '{}': {}".format(ename, param))
191+
# case 2: encodinc characters translation
192+
else:
193+
# collect base tokens in order of appearance in the mapping dictionary
194+
base_tokens = ""
195+
for _, c in sorted(mapdict.items()):
196+
for t in c:
197+
if t not in base_tokens:
198+
base_tokens += t
199+
if param[0] in "-_" and len(param[1:]) == len(set(param[1:])) == len(base_tokens):
200+
param = param[1:]
201+
if len(param) == len(set(param)) == len(base_tokens):
202+
t = maketrans(base_tokens, param)
203+
for k, v in smapdict.items():
204+
smapdict[k] = v.translate(t)
205+
else:
206+
raise LookupError("Bad parameter for encoding '{}': {}".format(ename, param))
207+
if ignore_case:
208+
case = ["upper", "lower"][any(c in "".join(smapdict.keys()) for c in "abcdefghijklmnopqrstuvwxyz")]
209+
# use the first mapped group from the mapping dictionary to determine token length ; this is useful e.g. for
210+
# tokenizing a binary string when the text is to be converted as binary
211+
tlen = len(list(smapdict.keys())[0])
212+
if decode:
213+
tmp = {}
214+
# this has a meaning for encoding maps that could have clashes in encoded chars (e.g. Bacon's cipher ;
215+
# I => abaaa but also J => abaaa, with the following, we keep I instead of letting J overwrite it)
216+
for k, v in smapdict.items():
217+
if v not in tmp.keys():
218+
tmp[v] = k
219+
smapdict = tmp
220+
# this allows to avoid an error with Python2 in the "for i, c in enumerate(parts)" loop
221+
if '' not in smapdict.keys():
222+
smapdict[''] = ""
223+
224+
def code(text, errors="strict"):
225+
if ignore_case:
226+
text = getattr(text, case)()
227+
if no_error:
228+
errors = "leave"
229+
text = ensure_str(text)
230+
if binary and not decode:
231+
text = "".join("{:0>8}".format(bin(ord(c))[2:]) for c in text)
232+
text = [text[i:i+tlen] for i in range(0, len(text), tlen)]
233+
parts = re.split("[" + sep + "]", text) if decode and len(sep) > 0 else text
234+
r = ""
235+
lsep = "" if decode else sep if len(sep) <= 1 else sep[0]
236+
for i, c in enumerate(parts):
237+
try:
238+
r += smapdict[c] + lsep
239+
except KeyError:
240+
if errors == "strict":
241+
raise exc("'{}' codec can't {}code character '{}' in position {}"
242+
.format(ename, ["en", "de"][decode], c, i))
243+
elif errors == "leave":
244+
r += c + lsep
245+
elif errors == "replace":
246+
r += repl_char * [1, tlen][decode] + lsep
247+
elif errors == "ignore":
248+
continue
249+
else:
250+
raise ValueError("Unsupported error handling '{}'".format(errors))
251+
if binary and decode:
252+
tmp, r = "", r.replace(lsep, "")
253+
for i in range(0, len(r), 8):
254+
bs = r[i:i+8]
255+
try:
256+
tmp += chr(int(bs, 2))
257+
except ValueError:
258+
if len(bs) > 0:
259+
tmp += "[" + bs + "]"
260+
r = tmp + lsep
261+
return r[:len(r)-len(lsep)], len(text)
262+
return code
263+
if re.search(r"\([^(?:)]", kwargs.get('pattern', "")) is None:
264+
# in this case, there is no capturing group for parametrization
265+
return _wrapper(None)
266+
return _wrapper
267+
268+
name = "".join(t.capitalize() for t in re.split(r"[-_]", ename))
269+
glob = currentframe().f_back.f_globals
270+
# dynamically make dedicated exception classes
271+
decexc = "{}DecodeError".format(name)
272+
exec("class {}(ValueError): pass".format(decexc), glob)
273+
encexc = "{}EncodeError".format(name)
274+
exec("class {}(ValueError): pass".format(encexc), glob)
275+
# now use the generic add() function
276+
add(ename, __generic_code(encmap, glob[encexc]), __generic_code(encmap, glob[decexc], True), **kwargs)
277+
codecs.add_map = add_map
278+
279+
139280
def clear():
140281
"""
141282
Clear codext's local registry of search functions.
@@ -231,8 +372,7 @@ def _wrapper(*args, **kwargs):
231372
orig_register = _codecs.register
232373

233374

234-
def __add(ename, encode=None, decode=None, pattern=None, text=True,
235-
add_to_codecs=True):
375+
def __add(ename, encode=None, decode=None, pattern=None, text=True, add_to_codecs=True):
236376
add(ename, encode, decode, pattern, text, add_to_codecs)
237377
__add.__doc__ = add.__doc__
238378
codecs.add = __add

‎codext/__init__.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,7 @@
99
from .__info__ import __author__, __copyright__, __email__, __license__, __source__, __version__
1010

1111

12-
__all__ = ["add", "clear", "decode", "encode", "lookup", "open", "register",
13-
"remove", "reset"]
12+
__all__ = ["add", "add_map", "clear", "decode", "encode", "lookup", "open", "register", "remove", "reset"]
1413

1514
decode = codecs.decode
1615
encode = codecs.encode
@@ -50,7 +49,7 @@ def main():
5049
parser = argparse.ArgumentParser(description=descr, epilog=examples, formatter_class=argparse.RawTextHelpFormatter)
5150
parser.add_argument("encoding")
5251
parser.add_argument("-d", "--decode", action="store_true", help="set decode mode")
53-
parser.add_argument("-e", "--errors", default="strict", choices=["ignore", "replace", "strict"],
52+
parser.add_argument("-e", "--errors", default="strict", choices=["ignore", "leave", "replace", "strict"],
5453
help="error handling")
5554
parser.add_argument("-i", "--input-file", dest="infile", help="input file (if none, take stdin as input)")
5655
parser.add_argument("-o", "--output-file", dest="outfile", help="output file (if none, display result to stdout)")

0 commit comments

Comments
 (0)