Skip to content

Commit 0c5c3f0

Browse files
dhondtadhondta
committed
Improved guess feature + New release
1 parent b78a7bc commit 0c5c3f0

8 files changed

Lines changed: 61 additions & 41 deletions

File tree

‎codext/VERSION.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.8.0
1+
1.8.1

‎codext/__common__.py‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -446,6 +446,8 @@ def list_categories():
446446
for d in os.listdir(root):
447447
if os.path.isdir(os.path.join(root, d)) and not d.startswith("__"):
448448
c.append(d.rstrip("s"))
449+
# particular category, hardcoded from base/_base.py
450+
c += ["base-generic"]
449451
return c
450452

451453

@@ -844,7 +846,8 @@ def generate_strings_from_regex(regex, star_plus_max=STAR_PLUS_MAX, repeat_max=R
844846
- `printables`: checks that every output character is in the set of printables
845847
""")
846848
stopfunc.printables = lambda s: all(c in printable for c in ensure_str(s))
847-
stopfunc.regex = lambda p: lambda s: re.search(p, ensure_str(s), re.I) is not None
849+
stopfunc.regex = lambda p: lambda s: re.search(p, ensure_str(s)) is not None
850+
stopfunc.text = lambda s: stopfunc.printables(s) and entropy(s) < 4.5
848851

849852
try:
850853
from langdetect import detect, PROFILES_DIRECTORY
@@ -864,15 +867,14 @@ def _flag(x):
864867

865868

866869
def __guess(prev_input, input, stop_func, depth, max_depth, codec_categories, exclude, result, found=(), stop=True,
867-
show=False, scoring_heuristic=False, extended=False):
870+
show=False, scoring_heuristic=False, extended=False, debug=False):
868871
""" Perform a breadth-first tree search using a ranking logic to select and prune the list of codecs. """
869872
if depth > 0 and stop_func(input):
870-
if not stop and show:
871-
s = "[*] %s: %s" % (", ".join(found), ensure_str(input))
873+
if not stop and show and found not in result:
874+
s = "[+] %s: %s" % (", ".join(found), ensure_str(input))
872875
print(s if len(s) <= 80 else s[:77] + "...")
873-
result.append((input, found))
874-
return
875-
if depth >= max_depth or len(result) > 0:
876+
result[found] = input
877+
if depth >= max_depth or len(result) > 0 and stop:
876878
return
877879
# compute included and excluded codecs for this depth
878880
def expand(items, descr=None, transform=None):
@@ -898,10 +900,14 @@ def expand(items, descr=None, transform=None):
898900
# parse valid encodings, expanding included/excluded codecs
899901
c, e = expand(codec_categories, "codec_categories", list_encodings), expand(exclude, "exclude")
900902
for new_input, encoding in __rank(prev_input, input, c, scoring_heuristic, extended):
903+
if len(result) > 0 and stop:
904+
return
901905
if encoding in e:
902906
continue
907+
if debug:
908+
print("[*] Depth %d/%d ; trying %s" % (depth+1, max_depth, encoding))
903909
__guess(input, new_input, stop_func, depth+1, max_depth, codec_categories, exclude, result,
904-
found + (encoding, ), stop, show, extended)
910+
found + (encoding, ), stop, show, scoring_heuristic, extended, debug)
905911

906912

907913
def __rank(prev_input, input, codecs, heuristic=False, extended=False):
@@ -994,8 +1000,8 @@ def __score(prev_input, input, codec, heuristic=False, extended=False):
9941000
yield s, new_input, encoding
9951001

9961002

997-
def guess(input, stop_func=stopfunc.printables, max_depth=5, codec_categories=None, exclude=None, found=(), stop=True,
998-
show=False, scoring_heuristic=False, extended=False):
1003+
def guess(input, stop_func=stopfunc.printables, max_depth=5, codec_categories=None, exclude=None, result=None, found=(),
1004+
stop=True, show=False, scoring_heuristic=False, extended=False, debug=False):
9991005
""" Try decoding without the knowledge of the encoding(s). """
10001006
if max_depth <= 0:
10011007
raise ValueError("Depth must be a non-null positive integer")
@@ -1005,10 +1011,11 @@ def guess(input, stop_func=stopfunc.printables, max_depth=5, codec_categories=No
10051011
if isinstance(stop_func, string_types):
10061012
stop_func = stopfunc.regex(stop_func)
10071013
if len(input) > 0:
1008-
result = []
1014+
result = result or {}
1015+
# breadth-first search
10091016
for d in range(max_depth):
10101017
__guess("", input, stop_func, 0, d+1, codec_categories, exclude, result, tuple(found), stop, show,
1011-
scoring_heuristic, extended)
1018+
scoring_heuristic, extended, debug)
10121019
if stop and len(result) > 0:
10131020
return result
10141021
return result

‎codext/__info__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
__author__ = "Alexandre D'Hondt"
99
__copyright__ = "© 2019-{} A. D'Hondt".format(datetime.now().year)
1010
__email__ = "alexandre.dhondt@gmail.com"
11-
__license__ = "AGPLv3 (http://www.gnu.org/licenses/agpl.html)"
11+
__license__ = "GPLv3 (https://www.gnu.org/licenses/gpl-3.0.fr.html)"
1212
__source__ = "https://github.com/dhondta/python-codext"
1313

1414
with open(os.path.join(os.path.dirname(__file__), "VERSION.txt")) as f:

‎codext/__init__.py‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"""
55
from __future__ import print_function
66
from ast import literal_eval
7-
from six import b, binary_type, text_type
7+
from six import binary_type, text_type
88

99
from .__common__ import *
1010
from .__info__ import __author__, __copyright__, __email__, __license__, __source__, __version__
@@ -61,7 +61,7 @@ def main():
6161
"echo -en \"test\" | codext encode upper reverse base32 | codext decode base32 reverse lower",
6262
"echo -en \"test\" | codext encode upper reverse base32 base64 morse",
6363
"echo -en \"test\" | codext encode base64 gzip | codext guess",
64-
"echo -en \"test\" | codext encode base64 gzip | codext guess gzip",
64+
"echo -en \"test\" | codext encode base64 gzip | codext guess gzip -c base",
6565
])
6666
parser = argparse.ArgumentParser(description=descr, epilog=examples, formatter_class=argparse.RawTextHelpFormatter)
6767
sparsers = parser.add_subparsers(dest="command", help="command to be executed")
@@ -81,15 +81,19 @@ def main():
8181
guess.add_argument("encoding", nargs="*", help="list of known encodings to apply (default: none)")
8282
guess.add_argument("-c", "--codec-categories", help="codec categories to be included in the search ; "
8383
"format: string|tuple|list(strings|tuples)")
84-
guess.add_argument("-d", "--depth", default=3, type=int, help="maximum codec search depth (default: 3)")
84+
guess.add_argument("-d", "--depth", default=5, type=int, help="maximum codec search depth (default: 3)")
8585
guess.add_argument("-e", "--exclude-codecs", help="codecs to be explicitely not used ; "
8686
"format: string|tuple|list(strings|tuples)")
87+
guess.add_argument("-f", "--stop-function", default="test", help="result checking function (default: text) ; "
88+
"format: printables|text|flag|lang_[bigram]|[regex]")
8789
guess.add_argument("--extended", action="store_true",
8890
help="while using the scoring heuristic, also consider null scores (default: False)")
8991
guess.add_argument("--heuristic", action="store_true",
9092
help="use a scoring heuristic to accelerate guessing (default: False)")
9193
guess.add_argument("-s", "--do-not-stop", action="store_true",
9294
help="do not stop if a valid output is found (default: False)")
95+
guess.add_argument("-v", "--verbose", action="store_true",
96+
help="show guessing information and steps (default: False)")
9397
search = sparsers.add_parser("search", help="search for codecs")
9498
search.add_argument("pattern", nargs="+", help="encoding pattern to search")
9599
args = parser.parse_args()
@@ -121,18 +125,23 @@ def main():
121125
else:
122126
print(ensure_str(c or "Could not decode :-("), end="")
123127
elif args.command == "guess":
124-
l = [o for o in codecs.guess(c, stopfunc.printables, args.depth, __literal_eval(args.codec_categories),
125-
__literal_eval(args.exclude_codecs), args.encoding, not args.do_not_stop, True,
126-
args.heuristic, args.extended)]
127-
for i, o in enumerate(l):
128-
out, e = o
128+
sfunc = getattr(stopfunc, args.stop_function, args.stop_function)
129+
r = {}
130+
try:
131+
codecs.guess(c, sfunc, args.depth, __literal_eval(args.codec_categories),
132+
__literal_eval(args.exclude_codecs), r, args.encoding, not args.do_not_stop, True,
133+
args.heuristic, args.extended, args.verbose)
134+
except KeyboardInterrupt:
135+
pass
136+
for i, o in enumerate(r.items()):
137+
e, out = o
129138
if len(e) > 0:
130139
if args.outfile:
131140
n, ext = os.path.splitext(args.outfile)
132-
fn = args.outfile if len(l) == 1 else "%s-%d%s" % (n, i+1, ext)
141+
fn = args.outfile if len(r) == 1 else "%s-%d%s" % (n, i+1, ext)
133142
else:
134143
print("Codecs: %s" % ", ".join(e))
135-
print(ensure_str(out), end="")
136-
if len(l) == 0:
144+
print(ensure_str(out))
145+
if len(r) == 0:
137146
print("Could not decode :-(")
138147

‎codext/base/_base.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,5 +159,5 @@ def _decode(input, errors="strict"):
159159

160160
add("base", encode, decode, r"^base[-_]?([2-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(?:[-_]generic)?$",
161161
guess=["base%d-generic" % i for i in range(2, 255)], entropy=lambda e, n: log(int(n.split("-")[0][4:]), 2),
162-
len_charset=lambda n: int(n.split("-")[0][4:]), printables_rate=1.)
162+
len_charset=lambda n: int(n.split("-")[0][4:]), printables_rate=1., category="base-generic")
163163

‎codext/base/base91.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
from ._base import digits, lower, upper
1414

1515
# no __examples__ ; handled manually in tests/test_base.py
16-
__guess__ = ["base91", "base91-inv"]
16+
__guess__ = ["base91", "base91-inv"]
1717

1818

1919
B91 = {

‎docs/cli.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,11 @@ The help message describes everything to know:
99
```sh
1010
usage: codext [-h] [-i INFILE] [-o OUTFILE] [-s] {encode,decode,guess,search} ...
1111

12-
Codecs Extension (CodExt) 1.6.2
12+
Codecs Extension (CodExt) 1.8.1
1313

1414
Author : Alexandre D'Hondt (alexandre.dhondt@gmail.com)
15-
Copyright: © 2019-2020 A. D'Hondt
16-
License : AGPLv3 (http://www.gnu.org/licenses/agpl.html)
15+
Copyright: © 2019-2021 A. D'Hondt
16+
License : GPLv3 (https://www.gnu.org/licenses/gpl-3.0.fr.html)
1717
Source : https://github.com/dhondta/python-codext
1818

1919
This tool allows to encode/decode input strings/files with an extended set of codecs.
@@ -46,7 +46,7 @@ usage examples:
4646
- echo -en "test" | codext encode upper reverse base32 | codext decode base32 reverse lower
4747
- echo -en "test" | codext encode upper reverse base32 base64 morse
4848
- echo -en "test" | codext encode base64 gzip | codext guess
49-
- echo -en "test" | codext encode base64 gzip | codext guess gzip
49+
- echo -en "test" | codext encode base64 gzip | codext guess gzip -c base
5050
```
5151

5252
!!! note "Input/output"

‎tests/test_common.py‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -128,22 +128,24 @@ def test_search_codecs(self):
128128
self.assertIsNotNone(list(codext.generate_strings_from_regex(r"[^a]")))
129129

130130
def test_guess_decode(self):
131+
_l = lambda d: list(d.items())[0][1]
131132
codext.reset()
132133
STR = "This is a test"
133-
self.assertEqual(STR, codext.guess("VGhpcyBpcyBhIHRlc3Q=", "a test", 1)[0][0])
134-
self.assertEqual(STR, codext.guess("CJG3Ix8bVcSRMLOqwDUg28aDsT7", "a test", found=["base62"])[0][0])
134+
self.assertEqual(STR, _l(codext.guess("VGhpcyBpcyBhIHRlc3Q=", "a test", 1)))
135+
self.assertEqual(STR, _l(codext.guess("CJG3Ix8bVcSRMLOqwDUg28aDsT7", "a test", found=["base62"])))
135136
if hasattr(codext.stopfunc, "lang_en"):
136137
f = codext.stopfunc.lang_en
137-
self.assertEqual(STR, codext.guess("CJG3Ix8bVcSRMLOqwDUg28aDsT7", f, found=["base62"])[0][0])
138-
self.assertIsNotNone(codext.guess("CJG3Ix8bVcSRMLOqwDUg28aDsT7", f, max_depth=1)[0][0])
139-
self.assertEqual(STR, codext.guess("VGhpcyBpcyBhIHRlc3Q=", "a test", 1, "base", exclude=["base100"])[0][0])
140-
self.assertEqual(STR, codext.guess("VGhpcyBpcyBhIHRlc3Q=", "a test", 1, ["base", "crypto"])[0][0])
138+
self.assertEqual(STR, _l(codext.guess("CJG3Ix8bVcSRMLOqwDUg28aDsT7", f, found=["base62"])))
139+
self.assertIsNotNone(_l(codext.guess("CJG3Ix8bVcSRMLOqwDUg28aDsT7", f, max_depth=1)))
140+
self.assertEqual(STR, _l(codext.guess("VGhpcyBpcyBhIHRlc3Q=", "a test", 1, "base", scoring_heuristic=True,
141+
exclude=["base100"])))
142+
self.assertEqual(STR, _l(codext.guess("VGhpcyBpcyBhIHRlc3Q=", "a test", 1, ["base", "crypto"])))
141143
self.assertEqual(len(codext.guess("NOT THE ENCODED TEST STRING", "a test", 1, exclude=[None])), 0)
142-
self.assertIn("F1@9", codext.guess("VGVzdCBGMUA5ICE=", codext.stopfunc.flag, 1, stop=False, show=True)[0][0])
144+
self.assertIn("F1@9", _l(codext.guess("VGVzdCBGMUA5ICE=", codext.stopfunc.flag, 1, stop=False, show=True)))
143145
self.assertEqual(len(codext.guess("VGhpcyBpcyBhIHRlc3Q=", " a test", 1, codec_categories="base",
144146
exclude="base64")), 0)
145147
self.assertEqual(len(codext.guess("VGhpcyBpcyBhIHRlc3Q=", " a test", 1, codec_categories="base",
146-
exclude=("base64", "atbash"))), 0)
148+
scoring_heuristic=True, exclude=("base64", "atbash"))), 0)
147149
self.assertRaises(ValueError, codext.guess, STR, max_depth=0)
148150
self.assertRaises(ValueError, codext.guess, STR, exclude=42)
149151
for c in ["base", "language", "native", "stegano"]:
@@ -159,7 +161,8 @@ def test_guess_decode(self):
159161
enc = codext.encode(b(STR), encoding)
160162
if codext.decode(enc, encoding) == STR:
161163
continue
162-
for found_dec, found_encodings in codext.guess(enc, "a test", 1, [c]):
164+
for found_encodings, found_dec in codext.guess(enc, "a test", 1, [c], scoring_heuristic=True,
165+
debug=True).items():
163166
self.assertEqual(ensure_str(STR).lower(), ensure_str(found_dec).lower())
164167
if c != "base":
165168
# do not check for base as the guessed encoding name can be different, e.g.:
@@ -171,5 +174,6 @@ def test_guess_decode(self):
171174
self.assertEqual(encoding, found_encodings[0])
172175
txt = "".join(chr(i) for i in range(256))
173176
b64 = codext.encode(txt, "base64")
174-
self.assertEqual(txt, codext.guess(b64, "0123456789", max_depth=1, codec_categories="base")[0][0])
177+
self.assertEqual(txt, _l(codext.guess(b64, "0123456789", max_depth=1, scoring_heuristic=True,
178+
codec_categories="base")))
175179

0 commit comments

Comments
 (0)