Skip to content

Commit 00f2cba

Browse files
dhondtadhondta
committed
Applied minor improvements
1 parent c06ee58 commit 00f2cba

2 files changed

Lines changed: 18 additions & 9 deletions

File tree

‎codext/__common__.py‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1154,7 +1154,7 @@ def generate_strings_from_regex(regex, star_plus_max=STAR_PLUS_MAX, repeat_max=R
11541154
stopfunc.default = stopfunc.text
11551155

11561156
stopfunc.LANG_BACKEND = None
1157-
stopfunc.LANG_BACKENDS = [n for n in ["langid", "langdetect", "pycld2", "cld3", "textblob"] if __module_exists(n)]
1157+
stopfunc.LANG_BACKENDS = [n for n in ["pycld2", "langdetect", "langid", "cld3", "textblob"] if __module_exists(n)]
11581158
if len(stopfunc.LANG_BACKENDS) > 0:
11591159
stopfunc.LANG_BACKEND = stopfunc.LANG_BACKENDS[0]
11601160
if "cld3" in stopfunc.LANG_BACKENDS:
@@ -1223,7 +1223,6 @@ def _load_lang_backend(backend=None):
12231223
flng = "lang_%s" % LANG
12241224
if getattr(stopfunc, flng, None):
12251225
stopfunc.default = getattr(stopfunc, flng)
1226-
_load_lang_backend(stopfunc.LANG_BACKEND)
12271226
stopfunc._reload_lang = _load_lang_backend
12281227

12291228

@@ -1244,10 +1243,12 @@ def __develop(encodings):
12441243

12451244

12461245
def __guess(prev_input, input, stop_func, depth, max_depth, min_depth, codec_categories, exclude, result, found=(),
1247-
stop=True, show=False, scoring_heuristic=False, extended=False, debug=False):
1246+
stop=True, show=False, scoring_heuristic=False, extended=False, debug=False, regex=False):
12481247
""" Perform a breadth-first tree search using a ranking logic to select and prune the list of codecs. """
12491248
if depth > min_depth and stop_func(input):
1250-
if not stop and show and found not in result:
1249+
if regex:
1250+
stop = True
1251+
if not stop and (show or debug) and found not in result:
12511252
s = repr(input)
12521253
s = s[2:-1] if s.startswith("b'") and s.endswith("'") else s
12531254
s = "[+] %s: %s" % (", ".join(found), s)
@@ -1287,7 +1288,7 @@ def expand(items, descr=None, transform=None):
12871288
if debug:
12881289
print("[*] Depth %0{}d/%d: %s".format(len(str(max_depth))) % (depth+1, max_depth, encoding))
12891290
__guess(input, new_input, stop_func, depth+1, max_depth, min_depth, codec_categories, exclude, result,
1290-
found + (encoding, ), stop, show, scoring_heuristic, extended, debug)
1291+
found + (encoding, ), stop, show, scoring_heuristic, extended, debug, regex)
12911292

12921293

12931294
def __rank(prev_input, input, prev_encoding, codecs, heuristic=False, extended=False, yield_score=False):
@@ -1374,7 +1375,7 @@ def __score(prev_input, input, prev_encoding, codec, heuristic=False, extended=F
13741375
epxf = f - .1 <= expf <= f + .1
13751376
elif isinstance(expf, (tuple, list)) and len(expf) == 2:
13761377
expf = f - expf[1] <= expf[0] <= expf[1] + .1
1377-
s += .1
1378+
s += [-1., .1][expf]
13781379
# afterwards, if the input text has an entropy close to the expected one, give a bonus weighted on the
13791380
# number of input characters to take bad entropies of shorter strings into account
13801381
entr = sc.get('entropy', {})
@@ -1408,18 +1409,22 @@ def guess(input, stop_func=stopfunc.default, min_depth=0, max_depth=5, codec_cat
14081409
""" Try decoding without the knowledge of the encoding(s). """
14091410
if max_depth <= 0:
14101411
raise ValueError("Depth must be a non-null positive integer")
1412+
if min_depth > max_depth:
1413+
raise ValueError("Min depth shall be less than or equal to the max depth")
14111414
if len(found) > 0:
14121415
for encoding in found:
14131416
input = decode(input, encoding)
1417+
regex = False
14141418
if isinstance(stop_func, string_types):
14151419
stop_func = stopfunc.regex(stop_func)
1420+
regex = True
14161421
result = {}
14171422
if len(input) > 0:
14181423
try:
14191424
# breadth-first search
14201425
for d in range(max_depth):
14211426
__guess("", input, stop_func, 0, d+1, min_depth, codec_categories, exclude, result, tuple(found), stop,
1422-
show, scoring_heuristic, extended, debug)
1427+
show, scoring_heuristic, extended, debug, regex)
14231428
if stop and len(result) > 0:
14241429
return result
14251430
except KeyboardInterrupt:

‎codext/__init__.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ def main():
121121
help="while using the regex stop function, set it as case-insensitive (default: False)")
122122
guess.add_argument("-H", "--no-heuristic", action="store_true", help="DO NOT use the scoring heuristic ; slows down"
123123
" the search but may be more accurate (default: False)")
124-
if len(stopfunc.LANG_BACKENDS) == 0:
124+
if len(stopfunc.LANG_BACKENDS) > 0:
125125
_lb = stopfunc.LANG_BACKEND
126126
guess.add_argument("-l", "--lang-backend", default=_lb, choices=stopfunc.LANG_BACKENDS + ["none"],
127127
help="natural language detection backend (default: %s)" % _lb)
@@ -206,8 +206,12 @@ def main():
206206
else:
207207
print(ensure_str(c or "Could not %scode :-(" % ["en", "de"][args.command == "decode"]), end="")
208208
elif args.command == "guess":
209+
s, lb = args.stop_function, args.lang_backend
210+
if re.match(r"lang_[a-z]{2}$", s) and lb != "none" and \
211+
all(re.match(r"lang_[a-z]{2}$", x) is None for x in dir(stopfunc)):
212+
stopfunc._reload_lang(lb)
209213
r = codecs.guess(c,
210-
getattr(stopfunc, args.stop_function, ["", "(?i)"][args.icase] + args.stop_function),
214+
getattr(stopfunc, s, ["", "(?i)"][args.icase] + s),
211215
args.min_depth,
212216
args.max_depth,
213217
args.codec_categories,

0 commit comments

Comments
 (0)