Skip to content

Commit 5a0185f

Browse files
dhondtadhondta
committed
Applied minor improvements
1 parent cf4b8b9 commit 5a0185f

4 files changed

Lines changed: 97 additions & 47 deletions

File tree

‎codext/__common__.py‎

Lines changed: 81 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from itertools import chain, product
1414
from locale import getlocale
1515
from math import log
16+
from pkgutil import iter_modules
1617
from platform import system
1718
from random import randint
1819
from six import binary_type, string_types, text_type, BytesIO
@@ -39,7 +40,10 @@
3940
"DARWIN", "LANG", "LINUX", "MASKS", "PY3", "UNIX", "WINDOWS"]
4041
CODECS_REGISTRY = None
4142
CODECS_CATEGORIES = ["native", "custom"]
42-
LANG = getlocale()[0][:2].lower()
43+
try:
44+
LANG = getlocale()[0][:2].lower()
45+
except TypeError:
46+
LANG = None
4347
MASKS = {
4448
'a': printable,
4549
'b': "".join(chr(i) for i in range(256)),
@@ -64,22 +68,6 @@
6468
UNIX = DARWIN or LINUX
6569
WINDOWS = system() == "Windows"
6670

67-
LANG_BACKEND = None
68-
for lib in ["langid", "langdetect", "pycld2", "cld3", "textblob"]:
69-
try:
70-
globals()[lib] = __import__(lib)
71-
LANG_BACKEND = lib
72-
break
73-
except ImportError:
74-
pass
75-
CLD3_LANGUAGES = "af|am|ar|bg|bn|bs|ca|ce|co|cs|cy|da|de|el|en|eo|es|et|eu|fa|fi|fr|fy|ga|gd|gl|gu|ha|hi|hm|hr|ht|hu|" \
76-
"hy|id|ig|is|it|iw|ja|jv|ka|kk|km|kn|ko|ku|ky|la|lb|lo|lt|lv|mg|mi|mk|ml|mn|mr|ms|mt|my|ne|nl|no|ny|" \
77-
"pa|pl|ps|pt|ro|ru|sd|si|sk|sl|sm|sn|so|sq|sr|st|su|sv|sw|ta|te|tg|th|tr|uk|ur|uz|vi|xh|yi|yo|zh|zu" \
78-
.split("|")
79-
TEXTBLOB_LANGUAGES = "af|ar|az|be|bg|bn|ca|cs|cy|da|de|el|en|eo|es|et|eu|fa|fi|fr|ga|gl|gu|hi|hr|ht|hu|id|is|it|iw|" \
80-
"ja|ka|kn|ko|la|lt|lv|mk|ms|mt|nl|no|pl|pt|ro|ru|sk|sl|sq|sr|sv|sw|ta|te|th|tl|tr|uk|ur|vi|yi|zh" \
81-
.split("|")
82-
8371
entropy = lambda s: -sum([p * log(p, 2) for p in [float(s.count(c)) / len(s) for c in set(s)]])
8472

8573
isb = lambda s: isinstance(s, binary_type)
@@ -307,7 +295,8 @@ class StreamReader(Codec, codecs.StreamReader):
307295
ci.parameters['guess'] = kwargs.get('guess', glob.get('__guess__', [ename])) or []
308296
ci.parameters['module'] = kwargs.get('module', glob.get('__name__'))
309297
ci.parameters.setdefault("scoring", {})
310-
for attr in ["bonus_func", "entropy", "len_charset", "penalty", "printables_rate", "padding_char"]:
298+
for attr in ["bonus_func", "entropy", "expansion_factor", "len_charset", "penalty", "printables_rate",
299+
"padding_char"]:
311300
a = kwargs.pop(attr, None)
312301
if a is not None:
313302
ci.parameters['scoring'][attr] = a
@@ -1068,6 +1057,7 @@ def generate_strings_from_regex(regex, star_plus_max=STAR_PLUS_MAX, repeat_max=R
10681057

10691058

10701059
# guess feature objects
1060+
__module_exists = lambda n: n in [x[1] for x in iter_modules()]
10711061
stopfunc = ModuleType("stopfunc", """
10721062
Predefined stop functions
10731063
~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1077,6 +1067,8 @@ def generate_strings_from_regex(regex, star_plus_max=STAR_PLUS_MAX, repeat_max=R
10771067
- `flag`: searches for the pattern "[Ff][Ll1][Aa4@][Gg9]" (either UTF-8 or UTF-16)
10781068
- `lang_**`: checks if the given lang (any from the PROFILES_DIRECTORY of the langdetect module) is detected
10791069
- `printables`: checks that every output character is in the set of printables
1070+
- `regex`: takes one argument, the regular expression, for checking a string against the given pattern
1071+
- `text`: checks for printables and an entropy less than 4.6 (empirically determined)
10801072
""")
10811073
stopfunc.printables = lambda s: all(c in printable for c in ensure_str(s))
10821074
stopfunc.printables.__name__ = stopfunc.printables.__qualname__ = "printables"
@@ -1086,13 +1078,27 @@ def generate_strings_from_regex(regex, star_plus_max=STAR_PLUS_MAX, repeat_max=R
10861078
stopfunc.text.__name__ = stopfunc.text.__qualname__ = "text"
10871079
stopfunc.flag = lambda x: re.search(r"[Ff][Ll1][Aa4@][Gg96]", ensure_str(x)) is not None
10881080
stopfunc.flag.__name__ = stopfunc.flag.__qualname__ = "flag"
1089-
stopfunc.default = stopfunc.printables
1081+
stopfunc.default = stopfunc.text
1082+
1083+
stopfunc.LANG_BACKEND = None
1084+
stopfunc.LANG_BACKENDS = [n for n in ["langid", "langdetect", "pycld2", "cld3", "textblob"] if __module_exists(n)]
1085+
if len(stopfunc.LANG_BACKENDS) > 0:
1086+
stopfunc.LANG_BACKEND = stopfunc.LANG_BACKENDS[0]
1087+
if "cld3" in stopfunc.LANG_BACKENDS:
1088+
stopfunc.CLD3_LANGUAGES = "af|am|ar|bg|bn|bs|ca|ce|co|cs|cy|da|de|el|en|eo|es|et|eu|fa|fi|fr|fy|ga|gd|gl|gu|ha|" \
1089+
"hi|hm|hr|ht|hu|hy|id|ig|is|it|iw|ja|jv|ka|kk|km|kn|ko|ku|ky|la|lb|lo|lt|lv|mg|mi|mk|" \
1090+
"ml|mn|mr|ms|mt|my|ne|nl|no|ny|pa|pl|ps|pt|ro|ru|sd|si|sk|sl|sm|sn|so|sq|sr|st|su|sv|" \
1091+
"sw|ta|te|tg|th|tr|uk|ur|uz|vi|xh|yi|yo|zh|zu".split("|")
1092+
if "textblob" in stopfunc.LANG_BACKENDS:
1093+
stopfunc.TEXTBLOB_LANGUAGES = "af|ar|az|be|bg|bn|ca|cs|cy|da|de|el|en|eo|es|et|eu|fa|fi|fr|ga|gl|gu|hi|hr|ht|hu|" \
1094+
"id|is|it|iw|ja|ka|kn|ko|la|lt|lv|mk|ms|mt|nl|no|pl|pt|ro|ru|sk|sl|sq|sr|sv|sw|ta|" \
1095+
"te|th|tl|tr|uk|ur|vi|yi|zh".split("|")
10901096

10911097

10921098
def _detect(text):
1093-
_lb, t = LANG_BACKEND, ensure_str(text)
1099+
_lb, t = stopfunc.LANG_BACKEND, ensure_str(text)
10941100
if _lb is None:
1095-
raise ValueError("No language backend installed")
1101+
raise ValueError("No language backend %s" % ["selected", "installed"][len(stopfunc.LANG_BACKENDS) == 0])
10961102
return langid.classify(t)[0] if _lb == "langid" else \
10971103
langdetect.detect(t) if _lb == "langdetect" else \
10981104
pycld2.detect(t)[2][0][1] if _lb == "pycld2" else \
@@ -1110,23 +1116,42 @@ def _test(s):
11101116
return False
11111117
return _test
11121118

1113-
if LANG_BACKEND:
1114-
_lb = LANG_BACKEND
1115-
if _lb == "langid":
1116-
langid.langid.load_model()
1117-
for lang in (
1118-
langid.langid.identifier.nb_classes if _lb == "langid" else \
1119-
[p.replace("-", "") for p in os.listdir(langdetect.PROFILES_DIRECTORY)] if _lb == "langdetect" else \
1120-
list(set(x[1][:2] for x in pycld2.LANGUAGES if x[0] in pycld2.DETECTED_LANGUAGES)) if _lb == "pycld2" else \
1121-
CLD3_LANGUAGES if _lb == "cld3" else \
1122-
TEXTBLOB_LANGUAGES if _lb == "textblob" else \
1123-
[]):
1124-
n = "lang_%s" % lang
1125-
setattr(stopfunc, n, _lang(lang))
1126-
getattr(stopfunc, n).__name__ = getattr(stopfunc, n).__qualname__ = n
1127-
flng = "lang_%s" % LANG
1128-
if getattr(stopfunc, flng, None):
1129-
stopfunc.default = getattr(stopfunc, flng)
1119+
1120+
def _load_lang_backend(backend=None):
1121+
# import the requested backend library if not imported yet
1122+
if backend is None or backend in stopfunc.LANG_BACKENDS:
1123+
stopfunc.LANG_BACKEND = backend
1124+
if backend:
1125+
globals()[backend] = __import__(backend)
1126+
else:
1127+
raise ValueError("Unsupported language detection backend")
1128+
# remove language-related stop functions
1129+
for attr in dir(stopfunc):
1130+
if attr.startswith("_") or not isinstance(getattr(stopfunc, attr), FunctionType):
1131+
continue
1132+
if re.match(r"lang_[a-z]{2}$", attr):
1133+
delattr(stopfunc, attr)
1134+
# rebind applicable language-related stop functions
1135+
if stopfunc.LANG_BACKEND:
1136+
_lb = stopfunc.LANG_BACKEND
1137+
if _lb == "langid":
1138+
langid.langid.load_model()
1139+
for lang in (
1140+
langid.langid.identifier.nb_classes if _lb == "langid" else \
1141+
list(set(p[:2] for p in os.listdir(langdetect.PROFILES_DIRECTORY))) if _lb == "langdetect" else \
1142+
list(set(x[1][:2] for x in pycld2.LANGUAGES if x[0] in pycld2.DETECTED_LANGUAGES)) if _lb == "pycld2" else \
1143+
stopfunc.CLD3_LANGUAGES if _lb == "cld3" else \
1144+
stopfunc.TEXTBLOB_LANGUAGES if _lb == "textblob" else \
1145+
[]):
1146+
n = "lang_%s" % lang
1147+
setattr(stopfunc, n, _lang(lang))
1148+
getattr(stopfunc, n).__name__ = getattr(stopfunc, n).__qualname__ = n
1149+
if LANG:
1150+
flng = "lang_%s" % LANG
1151+
if getattr(stopfunc, flng, None):
1152+
stopfunc.default = getattr(stopfunc, flng)
1153+
_load_lang_backend(stopfunc.LANG_BACKEND)
1154+
stopfunc._reload_lang = _load_lang_backend
11301155

11311156

11321157
def __develop(encodings):
@@ -1202,6 +1227,8 @@ def __rank(prev_input, input, codecs, heuristic=False, extended=False, yield_sco
12021227

12031228

12041229
class _Text(object):
1230+
__slots__ = ["entropy", "lcharset", "len", "padding", "printables"]
1231+
12051232
def __init__(self, text, pad_char=None):
12061233
self.len = len(text)
12071234
self.lcharset = len(set(text))
@@ -1247,13 +1274,24 @@ def __score(prev_input, input, codec, heuristic=False, extended=False):
12471274
elif not pad and obj.padding:
12481275
s -= .1 # it could arise a padding character is encountered while not being padding => small penalty
12491276
# give a bonus when the rate of printable characters is greater or equal than expected and a penalty when
1250-
# lower only for codecs that tolerate errors (otherwise, the printables rate can be biased)
1277+
# lower only for codecs that DO NOT tolerate errors (otherwise, the printables rate can be biased)
12511278
if not ci.parameters.get('no_error', False):
12521279
pr = sc.get('printables_rate', 0)
12531280
if isinstance(pr, type(lambda: None)):
12541281
pr = float(pr(obj.printables))
12551282
if obj.printables - pr <= .05:
12561283
s += .1
1284+
expf = sc.get('expansion_factor')
1285+
if expf:
1286+
f = float(len(new_input)) / obj.len
1287+
if isinstance(expf, type(lambda: None)):
1288+
expf = expf(f)
1289+
elif isinstance(expf, (int, float)):
1290+
epxf = f - .1 <= expf <= f + .1
1291+
elif isinstance(expf, (tuple, list)) and len(expf) == 2:
1292+
expf = f - expf[1] <= expf[0] <= expf[1] + .1
1293+
if expf:
1294+
s += .1
12571295
# afterwards, if the input text has an entropy close to the expected one, give a bonus weighted on the
12581296
# number of input characters to take bad entropies of shorter strings into account
12591297
entr = sc.get('entropy', {})
@@ -1265,7 +1303,7 @@ def __score(prev_input, input, codec, heuristic=False, extended=False):
12651303
entr = entr(obj.entropy)
12661304
if entr is not None:
12671305
# use a quadratic heuristic to compute a weight for the entropy delta, aligned on (100w,.1) and (200w,1)
1268-
d_entr = min(4e-05 * obj.len**2 - .003 * obj.len, 1) * abs(entr - obj.entropy)
1306+
d_entr = min(4e-05 * obj.len**2 - .003 * obj.len, 1) * abs(entr - entropy(new_input))
12691307
if d_entr <= .5:
12701308
s += .5 - d_entr
12711309
# finally, if relevant, apply a custom bonus (e.g. when a regex pattern is matched)
@@ -1283,7 +1321,7 @@ def __score(prev_input, input, codec, heuristic=False, extended=False):
12831321

12841322

12851323
def guess(input, stop_func=stopfunc.default, min_depth=0, max_depth=5, codec_categories=None, exclude=None, found=(),
1286-
stop=True, show=False, scoring_heuristic=False, extended=False, debug=False):
1324+
stop=True, show=False, scoring_heuristic=True, extended=False, debug=False):
12871325
""" Try decoding without the knowledge of the encoding(s). """
12881326
if max_depth <= 0:
12891327
raise ValueError("Depth must be a non-null positive integer")
@@ -1309,6 +1347,8 @@ def guess(input, stop_func=stopfunc.default, min_depth=0, max_depth=5, codec_cat
13091347

13101348
def rank(input, extended=False, limit=-1, codec_categories=None, exclude=None):
13111349
""" Rank the most probable encodings based on the given input. """
1350+
if isinstance(codec_categories, string_types):
1351+
codec_categories = (codec_categories, )
13121352
codecs = list_encodings(*(codec_categories or ()))
13131353
for e in __develop(exclude):
13141354
try:

‎codext/__init__.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,12 @@ def main():
109109
def_func = lng if getattr(stopfunc, lng, None) else "text"
110110
guess.add_argument("-f", "--stop-function", default=def_func, help="result checking function (default: %s) ; "
111111
"format: printables|text|flag|lang_[bigram]|[regex]" % def_func)
112-
guess.add_argument("-H", "--heuristic", action="store_true", help="use the scoring heuristic for accelerating"
113-
" the search (default: False)")
112+
guess.add_argument("-H", "--no-heuristic", action="store_true", help="DO NOT use the scoring heuristic ; slows down"
113+
" the search but may be more accurate (default: False)")
114+
if len(stopfunc.LANG_BACKENDS) == 0:
115+
_lb = stopfunc.LANG_BACKEND
116+
guess.add_argument("-l", "--lang-backend", default=_lb, choices=stopfunc.LANG_BACKENDS + ["none"],
117+
help="natural language detection backend (default: %s)" % _lb)
114118
guess.add_argument("-s", "--do-not-stop", action="store_true",
115119
help="do not stop if a valid output is found (default: False)")
116120
guess.add_argument("-v", "--verbose", action="store_true",
@@ -201,7 +205,7 @@ def main():
201205
args.encoding,
202206
not args.do_not_stop,
203207
True, # show
204-
args.heuristic,
208+
not args.no_heuristic,
205209
args.extended,
206210
args.verbose)
207211
for i, o in enumerate(r.items()):

‎codext/languages/leetspeak.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,5 @@
2020

2121

2222
add_map("leet", ENCMAP, ignore_case="encode", no_error=True, pattern=r"(?:leet|1337|leetspeak)$",
23-
printables_rate=lambda pr: pr)
23+
entropy=lambda e: e, expansion_factor=1.)
2424

‎tests/test_common.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,10 +143,16 @@ def test_encode_multiple_rounds(self):
143143
self.assertIsNotNone(codext.encode("test", "base64[10]"))
144144

145145
def test_guess_decode(self):
146+
self.assertIsNone(codext.stopfunc._reload_lang())
146147
_l = lambda d: list(d.items())[0][1] if len(d) > 0 else None
147148
codext.add("test_codec", lambda x, e="strict": (x + "=", len(x)), lambda x, e="strict": (x[:-1], len(x)-1),
148149
"test", no_error=True, bonus_func=lambda *a: True, penalty=-.5)
149150
self.assertIn("test-codec", codext.list_encodings("test"))
151+
self.assertEqual(codext.decode("TEST=", "test"), "TEST")
152+
self.assertEqual(list(codext.guess("TEST=", codext.stopfunc.text, codec_categories="test", max_depth=2,
153+
scoring_heuristic=False).items())[0][1], "TEST")
154+
self.assertEqual(list(codext.guess("TEST=", codext.stopfunc.text, codec_categories=["test", "base"],
155+
max_depth=2).items())[0][1], "TEST")
150156
STR = "This is a test"
151157
self.assertEqual(STR, _l(codext.guess("VGhpcyBpcyBhIHRlc3Q=", "a test", max_depth=1)))
152158
self.assertEqual(STR, _l(codext.guess("CJG3Ix8bVcSRMLOqwDUg28aDsT7", "a test", found=["base62"])))
@@ -190,8 +196,7 @@ def test_guess_decode(self):
190196
b64 = codext.encode(txt, "base64")
191197
self.assertEqual(txt, _l(codext.guess(b64, "0123456789", max_depth=1, scoring_heuristic=True,
192198
codec_categories="base")))
193-
self.assertEqual(list(codext.guess("TEST=", codec_categories="test").items())[0][1], "TEST")
194-
self.assertEqual(list(codext.guess("TEST=", codec_categories=["test", "base"]).items())[0][1], "TEST")
199+
self.assertRaises(ValueError, codext.stopfunc._reload_langs, "DOES_NOT_EXIST")
195200

196201
def test_rank_input(self):
197202
codext.add("test_codec", lambda x, e="strict": (x + "=", len(x)), lambda x, e="strict": (x[:-1], len(x)-1),
@@ -201,6 +206,7 @@ def test_rank_input(self):
201206
self.assertTrue(len(codext.rank(ENC)) > 20)
202207
self.assertEqual(len(codext.rank(ENC, limit=20)), 20)
203208
self.assertEqual(codext.rank(ENC, exclude=["rot"])[0][1], "base64")
209+
self.assertEqual(codext.rank(ENC, codec_categories="base")[0][0][1], STR)
204210
self.assertEqual(codext.rank(ENC, codec_categories=["base"])[0][0][1], STR)
205211
self.assertIsNotNone(codext.rank(ENC, codec_categories=["base"], exclude=["does_not_exist"])[0][0][1], STR)
206212
self.assertIsNotNone(codext.rank("TEST=", codec_categories=["test", "base"])[0][0][1], "TEST")

0 commit comments

Comments
 (0)