Skip to content

Commit 33bcf8e

Browse files
dhondtadhondta
committed
Fixed line stripping while decoding
1 parent 5182fcc commit 33bcf8e

2 files changed

Lines changed: 47 additions & 22 deletions

File tree

‎codext/__common__.py‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,10 @@
2828
maketrans = str.maketrans
2929

3030

31-
__all__ = ["add", "add_map", "b", "clear", "codecs", "decode", "encode", "ensure_str", "examples", "guess",
31+
__all__ = ["add", "add_map", "b", "clear", "codecs", "decode", "encode", "ensure_str", "examples", "guess", "isb",
3232
"generate_strings_from_regex", "get_alphabet_from_mask", "handle_error", "is_native", "list_categories",
3333
"list_encodings", "lookup", "maketrans", "rank", "re", "register", "remove", "reset", "s2i", "search",
34-
"stopfunc", "BytesIO", "MASKS", "PY3", "_input"]
34+
"stopfunc", "BytesIO", "MASKS", "PY3", "_input", "_stripl"]
3535
CODECS_REGISTRY = None
3636
CODECS_CATEGORIES = ["native", "custom"]
3737
MASKS = {
@@ -81,6 +81,14 @@ def _input(infile):
8181
return c
8282

8383

84+
def _stripl(s, st_lines, st_crlf):
85+
if st_crlf:
86+
s = s.replace(b"\r\n", b"") if isb(s) else s.replace("\r\n", "")
87+
if st_lines:
88+
s = s.replace(b"\n", b"") if isb(s) else s.replace("\n", "")
89+
return s
90+
91+
8492
def add(ename, encode=None, decode=None, pattern=None, text=True, add_to_codecs=False, **kwargs):
8593
""" This adds a new codec to the codecs module setting its encode and/or decode functions, eventually dynamically
8694
naming the encoding with a pattern and with file handling.
@@ -132,6 +140,14 @@ def getregentry(encoding):
132140
fenc = fix_inout_formats(fenc)
133141
if fdec:
134142
fdec = fix_inout_formats(fdec)
143+
sl, sc = kwargs.pop('strip_lines', False), kwargs.pop('strip_crlf', False)
144+
if sl or sc:
145+
def _striplines(f):
146+
def __wrapper(input, *a, **kw):
147+
return f(_stripl(input, sc, sl), *a, **kw)
148+
return __wrapper
149+
# this fixes issues with wrapped encoded inputs
150+
fdec = _striplines(fdec)
135151

136152
class Codec(codecs.Codec):
137153
def encode(self, input, errors="strict"):
@@ -323,7 +339,9 @@ def _wrapper(param):
323339
for x in v:
324340
if x not in tmp.keys():
325341
tmp[x] = k
326-
smapdict = tmp
342+
smapdict, cs = tmp, reduce(lambda acc, x: acc + x, tmp.keys())
343+
kwargs['strip_lines'] = "\n" not in set(cs)
344+
kwargs['strip_crlf'] = "\r" not in set(cs) and "\n" not in set(cs) and "\r\n" not in cs
327345
# this allows to avoid an error with Python2 in the "for i, c in enumerate(parts)" loop
328346
if '' not in smapdict.keys():
329347
smapdict[''] = ""
@@ -1075,8 +1093,8 @@ def guess(input, stop_func=stopfunc.printables, min_depth=0, max_depth=5, codec_
10751093
input = decode(input, encoding)
10761094
if isinstance(stop_func, string_types):
10771095
stop_func = stopfunc.regex(stop_func)
1096+
result = {}
10781097
if len(input) > 0:
1079-
result = {}
10801098
try:
10811099
# breadth-first search
10821100
for d in range(max_depth):

‎codext/base/_base.py‎

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -135,14 +135,17 @@ def _encode(input, errors="strict"):
135135

136136
def decode(param=""):
137137
a = _get_charset(charset, param)
138+
sl, sc = "\n" not in a, "\n" not in a and not "\r" in a
138139
def _decode(input, errors="strict"):
140+
input = _stripl(input, sc, sl)
139141
return decode_template(input, a, errors), len(input)
140142
return _decode
141143

142144
kwargs['len_charset'] = n
143145
kwargs['printables_rate'] = 1.
144146
n = "base{}".format(n) if name is None else name
145-
add(n, encode, decode, pattern, entropy=nb, guess=[n], **kwargs)
147+
kwargs['guess'] = kwargs.get('guess', [n])
148+
add(n, encode, decode, pattern, entropy=nb, **kwargs)
146149

147150

148151
def base_generic():
@@ -155,7 +158,9 @@ def _encode(input, errors="strict"):
155158

156159
def decode(n):
157160
a = _generate_charset(int(n))
161+
sl, sc = "\n" not in a, "\n" not in a and not "\r" in a
158162
def _decode(input, errors="strict"):
163+
input = _stripl(input, sc, sl)
159164
return base_decode(input, a, errors), len(input)
160165
return _decode
161166

@@ -164,10 +169,10 @@ def _decode(input, errors="strict"):
164169
len_charset=lambda n: int(n.split("-")[0][4:]), printables_rate=1., category="base-generic", penalty=.4)
165170

166171

167-
def main(n, ref=None, alt=None):
172+
def main(n, ref=None, alt=None, inv=True):
168173
base = str(n) + ("-" + alt.lstrip("-") if alt else "")
169174
src = "The data are encoded as described for the base%(base)s alphabet in %(reference)s.\n" % \
170-
{'base': base, 'reference': "\n" + ref if len(ref) > 10 else ref} if ref else ""
175+
{'base': base, 'reference': "\n" + ref if len(ref) > 20 else ref} if ref else ""
171176
descr = """Usage: base%(base)s [OPTION]... [FILE]
172177
Base%(base)s encode or decode FILE, or standard input, to standard output.
173178
@@ -176,8 +181,7 @@ def main(n, ref=None, alt=None):
176181
Mandatory arguments to long options are mandatory for short options too.
177182
-d, --decode decode data
178183
-i, --ignore-garbage when decoding, ignore non-alphabet characters
179-
-I, --invert invert charsets from the base alphabet (e.g. lower- and uppercase)
180-
-w, --wrap=COLS wrap encoded lines after COLS character (default 76).
184+
%(inv)s -w, --wrap=COLS wrap encoded lines after COLS character (default 76).
181185
Use 0 to disable line wrapping
182186
183187
--help display this help and exit
@@ -189,26 +193,29 @@ def main(n, ref=None, alt=None):
189193
190194
Report base%(base)s translation bugs to <https://github.com/dhondta/python-codext/issues/new>
191195
Full documentation at: <https://python-codext.readthedocs.io/en/latest/enc/base.html>
192-
""" % {'base': base, 'source': src}
196+
""" % {'base': base, 'source': src,
197+
'inv': ["", " -I, --invert invert charsets from the base alphabet (e.g. lower- and uppercase)\n"][inv]}
193198

194199
def _main():
195-
parser = ArgumentParser(description=descr, formatter_class=RawTextHelpFormatter, add_help=False)
196-
parser.format_help = MethodType(lambda s: s.description, parser)
197-
parser.add_argument("file", nargs="?")
198-
parser.add_argument("-d", "--decode", action="store_true")
199-
parser.add_argument("-i", "--ignore-garbage", action="store_true")
200-
parser.add_argument("-I", "--invert", action="store_true")
201-
parser.add_argument("-w", "--wrap", type=int, default=76)
202-
parser.add_argument("--help", action="help")
203-
parser.add_argument("--version", action="version")
204-
parser.version = "CodExt " + __version__
205-
args = parser.parse_args()
200+
p = ArgumentParser(description=descr, formatter_class=RawTextHelpFormatter, add_help=False)
201+
p.format_help = MethodType(lambda s: s.description, p)
202+
p.add_argument("file", nargs="?")
203+
p.add_argument("-d", "--decode", action="store_true")
204+
p.add_argument("-i", "--ignore-garbage", action="store_true")
205+
if inv:
206+
p.add_argument("-I", "--invert", action="store_true")
207+
p.add_argument("-w", "--wrap", type=int, default=76)
208+
p.add_argument("--help", action="help")
209+
p.add_argument("--version", action="version")
210+
p.version = "CodExt " + __version__
211+
args = p.parse_args()
212+
args.invert = getattr(args, "invert", False)
206213
c, f = _input(args.file), [encode, decode][args.decode]
207214
c = c.rstrip("\r\n") if isinstance(c, str) else c.rstrip(b"\r\n")
208215
try:
209216
c = f(c, "base" + base + ["", "-inv"][args.invert], ["strict", "ignore"][args.ignore_garbage])
210217
except Exception as err:
211-
print("%sbase%d: invalid input" % (err.output, n))
218+
print("%sbase%s: invalid input" % (getattr(err, "output", ""), base))
212219
return 1
213220
for l in wrap(ensure_str(c), args.wrap):
214221
print(l)

0 commit comments

Comments
 (0)