Skip to content

Commit 8f23ffa

Browse files
dhondtadhondta
committed
Added stopfunc submodule (for use with the guess-decode feature)
1 parent 9cab179 commit 8f23ffa

3 files changed

Lines changed: 89 additions & 11 deletions

File tree

‎codext/__common__.py‎

Lines changed: 57 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from itertools import chain, product
1313
from six import binary_type, string_types, text_type, BytesIO
1414
from string import *
15-
from types import FunctionType
15+
from types import FunctionType, ModuleType
1616
try: # Python3
1717
from importlib import reload
1818
except ImportError:
@@ -29,7 +29,8 @@
2929

3030
__all__ = ["add", "add_map", "b", "clear", "codecs", "decode", "encode", "ensure_str", "examples", "guess",
3131
"generate_strings_from_regex", "get_alphabet_from_mask", "handle_error", "list_categories", "list_encodings",
32-
"lookup", "maketrans", "re", "register", "remove", "reset", "s2i", "search", "BytesIO", "MASKS", "PY3"]
32+
"lookup", "maketrans", "re", "register", "remove", "reset", "s2i", "search", "stopfunc", "BytesIO", "MASKS",
33+
"PY3"]
3334
CODECS_REGISTRY = None
3435
MASKS = {
3536
'a': printable,
@@ -46,8 +47,7 @@
4647
__codecs_registry = []
4748

4849

49-
entropy = lambda s: -sum([p * log(p, 2) for p in [float(s.count(c)) / len(s) for c in set(s)]])
50-
is_printable = lambda s: all(c in printable for c in ensure_str(s))
50+
entropy = lambda s: -sum([p * log(p, 2) for p in [float(s.count(c)) / len(s) for c in set(s)]])
5151

5252
isb = lambda s: isinstance(s, binary_type)
5353
iss = lambda s: isinstance(s, string_types)
@@ -833,14 +833,64 @@ def generate_strings_from_regex(regex, star_plus_max=STAR_PLUS_MAX, repeat_max=R
833833

834834

835835
# guess feature objects
836-
def __guess(input, stop_func, depth, max_depth, codecs, result, found=()):
836+
stopfunc = ModuleType("stopfunc", """
837+
Predefined stop functions
838+
~~~~~~~~~~~~~~~~~~~~~~~~~
839+
840+
This submodule contains stop functions for the guess feature of codext.
841+
842+
- `flag`: searches for the pattern "[Ff][Ll1][Aa4@][Gg9]" (either UTF-8 or UTF-16)
843+
- `lang_**`: checks if the given lang (any from the PROFILES_DIRECTORY of the langdetect module) is detected
844+
- `printables`: checks that every output character is in the set of printables
845+
""")
846+
stopfunc.printables = lambda s: all(c in printable for c in ensure_str(s))
847+
848+
try:
849+
from langdetect import detect, PROFILES_DIRECTORY
850+
for lang in [p.replace("-", "") for p in os.listdir(PROFILES_DIRECTORY)]:
851+
setattr(stopfunc, "lang_%s" % lang, lambda s, l=lang: stopfunc.printables(s) and detect(s) == l)
852+
except ImportError:
853+
pass
854+
855+
856+
__flag = lambda x: re.search(r"[Ff][Ll1][Aa4@][Gg9]", x) is not None
857+
def _flag(x):
858+
try:
859+
return __flag(ensure_str(b(x).decode("utf16")))
860+
except (UnicodeDecodeError, UnicodeEncodeError):
861+
return __flag(x)
862+
stopfunc.flag = _flag
863+
864+
865+
def __guess(input, stop_func, depth, max_depth, codecs, result, found=(), exclude=()):
837866
""" Perform a breadth-first tree search using a ranking logic to select and prune the list of codecs. """
838867
if depth > 0 and stop_func(input):
839868
result.append((input, found))
840869
return
841870
if depth >= max_depth or len(result) > 0:
842871
return
872+
# format 1: when 'exclude' is a string, take it as the only encoding to be excluded at any depth
873+
if isinstance(exclude, string_types):
874+
e = (exclude, )
875+
# format 2: when 'exclude' is a tuple, consider it as the encodings to be excluded at any depth
876+
elif isinstance(exclude, tuple):
877+
e = exclude
878+
# format 3: when 'exclude' is a list, consider it as the list of tuples of encodings to be exlucded with the order
879+
# number corresponding to the applicable depth
880+
elif isinstance(exclude, list):
881+
try:
882+
e = exclude[depth]
883+
if e is None:
884+
e = ()
885+
elif isinstance(e, string_types):
886+
e = (e, )
887+
except IndexError:
888+
e = ()
889+
else:
890+
raise ValueError("Bad exclude format %s" % exclude)
843891
for new_input, encoding in __rank(input, codecs):
892+
if encoding in e:
893+
continue
844894
__guess(new_input, stop_func, depth+1, max_depth, codecs, result, found + (encoding, ))
845895

846896

@@ -874,7 +924,7 @@ def __score(input, codec):
874924
yield score, new_input, encoding
875925

876926

877-
def guess(input, stop_func=is_printable, max_depth=5, codec_categories=None, found=()):
927+
def guess(input, stop_func=stopfunc.printables, max_depth=5, codec_categories=None, found=(), exclude=None):
878928
""" Try decoding without the knowledge of the encoding(s). """
879929
if max_depth <= 0:
880930
raise ValueError("Depth must be a non-null positive integer")
@@ -893,7 +943,7 @@ def guess(input, stop_func=is_printable, max_depth=5, codec_categories=None, fou
893943
if len(input) > 0:
894944
result = []
895945
for d in range(max_depth):
896-
__guess(input, stop_func, 0, d+1, codecs, result, tuple(found))
946+
__guess(input, stop_func, 0, d+1, codecs, result, tuple(found), exclude or [])
897947
if len(result) > 0:
898948
return result[0]
899949
return None, None

‎docs/features.md‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -295,13 +295,20 @@ An example of a 2-stages base64- then base62-encoded string:
295295
('FKU2Ng7lJbR>.IHuzLDv17eLhE6', ('barbie',))
296296
```
297297

298-
In the second example, we can see that the given encoded string is not decoded as expected. This is the case because the stop condition (default) is too broad. If we have a prior knowledge on what we should expect, we can input a simple string or a regex:
298+
In the second example, we can see that the given encoded string is not decoded as expected. This is the case because the (default) stop condition is too broad and stops if all the characters of the output are printable. If we have a prior knowledge on what we should expect, we can input a simple string or a regex:
299299

300300
```python
301301
>>> codext.guess("CJG3Ix8bVcSRMLOqwDUg28aDsT7", "test")
302302
('This is a test', ('base62', 'base64'))
303303
```
304304

305+
Instead of a string, we can also pass a function. For this purpose, standard stop functions are predefined in the `stopfunc` submodule. So, we can for instance use `stopfunc.lang_en` to stop when we find something that is English (only works if [`langdetect`](https://pypi.org/project/langdetect/) is installed). Note that working this way gives lots of false positives if the text is very short like in the example case. That's why the `codec_categories` argument is used to only consider baseX codecs. This is also demonstrated in the next examples.
306+
307+
```python
308+
>>> codext.guess("CJG3Ix8bVcSRMLOqwDUg28aDsT7", codext.stopfunc.lang_en, codec_categories="base")
309+
('This is a test', ('base62', 'base64'))
310+
```
311+
305312
If we know the first encoding, we can set this in the `found` parameter to save time:
306313

307314
```python
@@ -325,7 +332,17 @@ Another example of 2-stages encoded string:
325332
('this is a test', ('base64', 'morse'))
326333
```
327334

328-
Note that the first call takes much longer than the second one but requires no knowledge about the possible [categories](#list-codecs) of encodings.
335+
!!! warning "Computation time"
336+
337+
Note that, in the very last examples, the first call takes much longer than the second one but requires no knowledge about the possible [categories](#list-codecs) of encodings.
338+
339+
!!! note "Stop functions"
340+
341+
Currently, a few standard stop functions are provided with the `stopfunc` submodule:
342+
343+
- `flag`: searches for the pattern "`[Ff][Ll1][Aa4@][Gg9]`" (either UTF-8 or UTF-16)
344+
- `lang_**`: checks if the given lang (any from the [`PROFILES_DIRECTORY`](https://github.com/Mimino666/langdetect/tree/master/langdetect/profiles) of the [`langdetect` module](https://github.com/Mimino666/langdetect)) is detected (note that it first checks if all characters are printable)
345+
- `printables`: checks that every output character is in the set of printables
329346

330347
-----
331348

‎tests/test_common.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,9 +132,20 @@ def test_guess_decode(self):
132132
STR = "This is a test"
133133
self.assertEqual(STR, codext.guess("VGhpcyBpcyBhIHRlc3Q=", "a test", 1)[0])
134134
self.assertEqual(STR, codext.guess("CJG3Ix8bVcSRMLOqwDUg28aDsT7", "a test", found=["base62"])[0])
135-
self.assertEqual(STR, codext.guess("VGhpcyBpcyBhIHRlc3Q=", "a test", 1, "base")[0])
136-
self.assertIsNone(codext.guess("NOT THE ENCODED TEST STRING", "a test", 1)[0])
135+
if hasattr(codext.stopfunc, "lang_en"):
136+
f = codext.stopfunc.lang_en
137+
self.assertEqual(STR, codext.guess("CJG3Ix8bVcSRMLOqwDUg28aDsT7", f, found=["base62"])[0])
138+
self.assertIsNotNone(codext.guess("CJG3Ix8bVcSRMLOqwDUg28aDsT7", f, max_depth=1)[0])
139+
self.assertEqual(STR, codext.guess("VGhpcyBpcyBhIHRlc3Q=", "a test", 1, "base", exclude=["base100"])[0])
140+
self.assertEqual(STR, codext.guess("VGhpcyBpcyBhIHRlc3Q=", "a test", 1, ["base", "crypto"])[0])
141+
self.assertIsNone(codext.guess("NOT THE ENCODED TEST STRING", "a test", 1, exclude=[None])[0])
142+
self.assertIn("F1@9", codext.guess("VGVzdCBGMUA5ICE=", codext.stopfunc.flag, 1)[0])
143+
self.assertIsNone(codext.guess("VGhpcyBpcyBhIHRlc3Q=", " a test", 1, codec_categories="base",
144+
exclude="base64")[0])
145+
self.assertIsNone(codext.guess("VGhpcyBpcyBhIHRlc3Q=", " a test", 1, codec_categories="base",
146+
exclude=("base64", "atbash"))[0])
137147
self.assertRaises(ValueError, codext.guess, STR, max_depth=0)
148+
self.assertRaises(ValueError, codext.guess, STR, exclude=42)
138149
for c in ["base", "language", "native", "stegano"]:
139150
e = codext.list(c)
140151
random.shuffle(e)

0 commit comments

Comments
 (0)