Skip to content

Commit 5feb784

Browse files
dhondtadhondta
committed
Added transitivity check in guess mode
1 parent d962dc1 commit 5feb784

6 files changed

Lines changed: 33 additions & 19 deletions

File tree

‎codext/__common__.py‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ def getregentry(encoding):
209209
while True:
210210
try:
211211
g = m.group(i) or ""
212-
if g.isdigit():
212+
if g.isdigit() and not g.startswith("0") and "".join(set(g)) != "01":
213213
g = int(g)
214214
args += [g]
215215
i += 1
@@ -296,7 +296,7 @@ class StreamReader(Codec, codecs.StreamReader):
296296
ci.parameters['module'] = kwargs.get('module', glob.get('__name__'))
297297
ci.parameters.setdefault("scoring", {})
298298
for attr in ["bonus_func", "entropy", "expansion_factor", "len_charset", "penalty", "printables_rate",
299-
"padding_char"]:
299+
"padding_char", "transitive"]:
300300
a = kwargs.pop(attr, None)
301301
if a is not None:
302302
ci.parameters['scoring'][attr] = a
@@ -1205,7 +1205,8 @@ def expand(items, descr=None, transform=None):
12051205
return r if transform is None else transform(*r)
12061206
# parse valid encodings, expanding included/excluded codecs
12071207
c, e = expand(codec_categories, "codec_categories", list_encodings), __develop(expand(exclude, "exclude"))
1208-
for new_input, encoding in __rank(prev_input, input, c, scoring_heuristic, extended):
1208+
prev_enc = found[-1] if len(found) > 0 else ""
1209+
for new_input, encoding in __rank(prev_input, input, prev_enc, c, scoring_heuristic, extended):
12091210
if len(result) > 0 and stop:
12101211
return
12111212
if encoding in e:
@@ -1216,11 +1217,11 @@ def expand(items, descr=None, transform=None):
12161217
found + (encoding, ), stop, show, scoring_heuristic, extended, debug)
12171218

12181219

1219-
def __rank(prev_input, input, codecs, heuristic=False, extended=False, yield_score=False):
1220+
def __rank(prev_input, input, prev_encoding, codecs, heuristic=False, extended=False, yield_score=False):
12201221
""" Filter valid encodings and rank them by relevance. """
12211222
ranking = {}
12221223
for codec in codecs:
1223-
for score, new_input, encoding in __score(prev_input, input, codec, heuristic, extended):
1224+
for score, new_input, encoding in __score(prev_input, input, prev_encoding, codec, heuristic, extended):
12241225
ranking[encoding] = (score, new_input)
12251226
for encoding, result in sorted(ranking.items(), key=lambda x: -x[1][0]):
12261227
yield result if yield_score else result[1], encoding
@@ -1237,10 +1238,11 @@ def __init__(self, text, pad_char=None):
12371238
self.entropy = entropy(text)
12381239

12391240

1240-
def __score(prev_input, input, codec, heuristic=False, extended=False):
1241+
def __score(prev_input, input, prev_encoding, codec, heuristic=False, extended=False):
12411242
""" Score relevant encodings given an input. """
12421243
obj, ci = None, lookup(codec, False) # NB: lookup(...) won't fail as the codec value comes from list_encodings(...)
12431244
sc = ci.parameters.get('scoring', {})
1245+
no_error, transitive = ci.parameters.get('no_error', False), sc.get('transitive', False)
12441246
for encoding in ci.parameters.get('guess', [codec]):
12451247
# ignore encodings that fail to decode with their default errors handling value
12461248
try:
@@ -1250,6 +1252,12 @@ def __score(prev_input, input, codec, heuristic=False, extended=False):
12501252
# ignore encodings that give an output identical to the input (identity transformation) or to the previous input
12511253
if len(new_input) == 0 or prev_input is not None and b(input) == b(new_input) or b(prev_input) == b(new_input):
12521254
continue
1255+
# ignore encodings that transitively give the same output (identity transformation by chaining twice a same
1256+
# codec (e.g. rot-15 is equivalent to rot-3 and rot-12 or rot-6 and rot-9)
1257+
if transitive and prev_encoding:
1258+
ci_prev = lookup(prev_encoding, False)
1259+
if ci_prev.parameters['name'] == ci.parameters['name']:
1260+
continue
12531261
# compute input's characteristics only once and only if the control flow reaches this point
12541262
pad = sc.get('padding_char')
12551263
if obj is None:
@@ -1275,23 +1283,25 @@ def __score(prev_input, input, codec, heuristic=False, extended=False):
12751283
s -= .1 # it could arise a padding character is encountered while not being padding => small penalty
12761284
# give a bonus when the rate of printable characters is greater or equal than expected and a penalty when
12771285
# lower only for codecs that DO NOT tolerate errors (otherwise, the printables rate can be biased)
1278-
if not ci.parameters.get('no_error', False):
1286+
if not no_error:
12791287
pr = sc.get('printables_rate', 0)
12801288
if isinstance(pr, type(lambda: None)):
12811289
pr = float(pr(obj.printables))
12821290
if obj.printables - pr <= .05:
12831291
s += .1
1284-
expf = sc.get('expansion_factor')
1292+
expf = sc.get('expansion_factor', 1.)
12851293
if expf:
12861294
f = float(len(new_input)) / obj.len
12871295
if isinstance(expf, type(lambda: None)):
1288-
expf = expf(f)
1296+
try: # this case allows to consider the current encoding name from the current codec
1297+
expf = expf(f, encoding)
1298+
except TypeError:
1299+
expf = expf(f)
12891300
elif isinstance(expf, (int, float)):
12901301
epxf = f - .1 <= expf <= f + .1
12911302
elif isinstance(expf, (tuple, list)) and len(expf) == 2:
12921303
expf = f - expf[1] <= expf[0] <= expf[1] + .1
1293-
if expf:
1294-
s += .1
1304+
s += .1
12951305
# afterwards, if the input text has an entropy close to the expected one, give a bonus weighted on the
12961306
# number of input characters to take bad entropies of shorter strings into account
12971307
entr = sc.get('entropy', {})
@@ -1355,7 +1365,7 @@ def rank(input, extended=False, limit=-1, codec_categories=None, exclude=None):
13551365
codecs.remove(e)
13561366
except ValueError:
13571367
pass
1358-
r = list(__rank(None, input, codecs, True, extended, True))
1368+
r = list(__rank(None, input, "", codecs, True, extended, True))
13591369
return r[:limit] if len(r) > 1 else r
13601370
codecs.rank = rank
13611371

‎codext/binary/rotate.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,5 +47,6 @@ def decode(text, errors="strict"):
4747
return decode
4848

4949

50-
add("rotate", rotate_encode, rotate_decode, r"rotate(?:[-_]?bits)?[-_]?((?:(?:left|right)[-_]?)?[1-7])$")
50+
add("rotate", rotate_encode, rotate_decode, r"rotate(?:[-_]?bits)?[-_]?((?:(?:left|right)[-_]?)?[1-7])$",
51+
transitive=True)
5152

‎codext/crypto/rot.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,9 @@ def decode(text, errors="strict"):
7272
return decode
7373

7474

75-
add("rot", rot_encode, rot_decode, r"(?:caesar|rot)[-_]?([1-9]|1[0-9]|2[0-5]|47)$", aliases=["caesar"],
76-
entropy=lambda e: e, printables_rate=lambda pr: pr, examples=__examples1__, guess=__guess1__)
75+
add("rot", rot_encode, rot_decode, r"(?:caesar|rot)[-_]?([1-9]|1[0-9]|2[0-5]|47)$", aliases=["caesar"], penalty=.1,
76+
entropy=lambda e: e, printables_rate=lambda pr: pr, transitive=True, examples=__examples1__, guess=__guess1__)
7777
add("progressive-rot", prot_encode, prot_decode, r"p(?:rog(?:ressive)?-)?(?:caesar|rot)[-_]?(n?)([1-9]|1[0-9]|2[0-5])$",
78-
entropy=lambda e: e, printables_rate=lambda pr: pr, examples=__examples2__, guess=__guess2__)
78+
penalty=.1, entropy=lambda e: e, printables_rate=lambda pr: pr, transitive=True, examples=__examples2__,
79+
guess=__guess2__)
7980

‎codext/crypto/shift.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,6 @@ def encode(text, errors="strict"):
2929
return encode
3030

3131

32-
add("shift", ord_shift_encode, ord_shift_decode, r"shift[-_]?([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])$")
32+
add("shift", ord_shift_encode, ord_shift_decode, r"shift[-_]?([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])$",
33+
transitive=True)
3334

‎codext/crypto/xor.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,6 @@ def encode(text, errors="strict"):
3030
return encode
3131

3232

33-
add("xor", xor_byte_encode, xor_byte_encode, r"^xor[-_]?([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])$")
33+
add("xor", xor_byte_encode, xor_byte_encode, r"^xor[-_]?([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])$",
34+
transitive=True)
3435

‎tests/test_common.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ def test_guess_decode(self):
196196
b64 = codext.encode(txt, "base64")
197197
self.assertEqual(txt, _l(codext.guess(b64, "0123456789", max_depth=1, scoring_heuristic=True,
198198
codec_categories="base")))
199-
self.assertRaises(ValueError, codext.stopfunc._reload_langs, "DOES_NOT_EXIST")
199+
self.assertRaises(ValueError, codext.stopfunc._reload_lang, "DOES_NOT_EXIST")
200200

201201
def test_rank_input(self):
202202
codext.add("test_codec", lambda x, e="strict": (x + "=", len(x)), lambda x, e="strict": (x[:-1], len(x)-1),

0 commit comments

Comments
 (0)