Skip to content

Commit 53eabf1

Browse files
dhondtadhondta
committed
Improved guess-decode feature (scoring heuristic)
1 parent ceae1be commit 53eabf1

37 files changed

Lines changed: 166 additions & 92 deletions

‎codext/__common__.py‎

Lines changed: 91 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from importlib import import_module
1111
from inspect import currentframe
1212
from itertools import chain, product
13+
from math import log
1314
from six import binary_type, string_types, text_type, BytesIO
1415
from string import *
1516
from types import FunctionType, ModuleType
@@ -156,6 +157,11 @@ class StreamReader(Codec, codecs.StreamReader):
156157
ci.parameters['examples'] = kwargs.get('examples', glob.get('__examples__'))
157158
ci.parameters['guess'] = kwargs.get('guess', glob.get('__guess__', [ename]))
158159
ci.parameters['module'] = kwargs.get('module', glob.get('__name__'))
160+
ci.parameters.setdefault("scoring", {})
161+
for attr in ["entropy", "len_charset", "printables_rate", "padding_char"]:
162+
a = kwargs.get(attr)
163+
if a is not None:
164+
ci.parameters['scoring'][attr] = a
159165
return ci
160166

161167
getregentry.__name__ = re.sub(r"[\s\-]", "_", ename)
@@ -838,6 +844,7 @@ def generate_strings_from_regex(regex, star_plus_max=STAR_PLUS_MAX, repeat_max=R
838844
- `printables`: checks that every output character is in the set of printables
839845
""")
840846
stopfunc.printables = lambda s: all(c in printable for c in ensure_str(s))
847+
stopfunc.regex = lambda p: lambda s: re.search(p, ensure_str(s), re.I) is not None
841848

842849
try:
843850
from langdetect import detect, PROFILES_DIRECTORY
@@ -847,7 +854,7 @@ def generate_strings_from_regex(regex, star_plus_max=STAR_PLUS_MAX, repeat_max=R
847854
pass
848855

849856

850-
__flag = lambda x: re.search(r"[Ff][Ll1][Aa4@][Gg9]", x) is not None
857+
__flag = lambda x: re.search(r"[Ff][Ll1][Aa4@][Gg96]", x) is not None
851858
def _flag(x):
852859
try:
853860
return __flag(ensure_str(b(x).decode("utf16")))
@@ -856,7 +863,8 @@ def _flag(x):
856863
stopfunc.flag = _flag
857864

858865

859-
def __guess(input, stop_func, depth, max_depth, codec_categories, exclude, result, found=(), stop=True, show=False):
866+
def __guess(prev_input, input, stop_func, depth, max_depth, codec_categories, exclude, result, found=(), stop=True,
867+
show=False, extended=False):
860868
""" Perform a breadth-first tree search using a ranking logic to select and prune the list of codecs. """
861869
if depth > 0 and stop_func(input):
862870
if not stop and show:
@@ -867,7 +875,8 @@ def __guess(input, stop_func, depth, max_depth, codec_categories, exclude, resul
867875
if depth >= max_depth or len(result) > 0:
868876
return
869877
# compute included and excluded codecs for this depth
870-
def __expand(items, descr=None, transform=None):
878+
def expand(items, descr=None, transform=None):
879+
items = items or []
871880
# format 1: when string, take it as the only items at any depth
872881
if isinstance(items, string_types):
873882
r = (items, )
@@ -887,59 +896,115 @@ def __expand(items, descr=None, transform=None):
887896
raise ValueError("Bad %sformat %s" % (["%s " % descr, ""][descr is None], items))
888897
return r if transform is None else transform(*r)
889898
# parse valid encodings, expanding included/excluded codecs
890-
c, e = __expand(codec_categories, "codec_categories", list_encodings), __expand(exclude, "exclude")
891-
for new_input, encoding in __rank(input, c):
899+
c, e = expand(codec_categories, "codec_categories", list_encodings), expand(exclude, "exclude")
900+
for new_input, encoding in __rank(prev_input, input, c, extended):
892901
if encoding in e:
893902
continue
894-
__guess(new_input, stop_func, depth+1, max_depth, codec_categories, exclude, result, found + (encoding, ), stop,
895-
show)
903+
__guess(input, new_input, stop_func, depth+1, max_depth, codec_categories, exclude, result,
904+
found + (encoding, ), stop, show, extended)
896905

897906

898-
def __rank(input, codecs):
907+
def __rank(prev_input, input, codecs, extended=False):
899908
""" Filter valid encodings and rank them by relevance. """
900909
ranking = {}
901910
for codec in codecs:
902-
for score, new_input, encoding in __score(input, codec):
903-
if score is not None:
904-
ranking[encoding] = (score, new_input)
911+
for score, new_input, encoding in __score(prev_input, input, codec, extended):
912+
ranking[encoding] = (score, new_input)
905913
for encoding, result in sorted(ranking.items(), key=lambda x: -x[1][0]):
906914
yield result[1], encoding
907915

908916

909-
def __score(input, codec):
917+
class _Text(object):
918+
def __init__(self, text, pad_char=None):
919+
self.len = len(text)
920+
self.lcharset = len(set(text))
921+
self.padding = pad_char is not None and text[-1] in [pad_char, b(pad_char)]
922+
self.printables = float(len([c for c in text if (chr(c) if isinstance(c, int) else c) in printable])) / self.len
923+
self.entropy = entropy(text)
924+
925+
926+
def __score(prev_input, input, codec, extended=False):
910927
""" Score relevant encodings given an input. """
911-
for encoding in lookup(codec).parameters.get('guess', [codec]):
928+
obj, ci = None, lookup(codec) # NB: lookup(...) won't fail as the codec value comes from list_encodings(...)
929+
for encoding in ci.parameters.get('guess', [codec]):
930+
# ignore encodings that fail to decode with their default errors handling value
912931
try:
913932
new_input = decode(input, encoding)
914933
except:
915934
continue
916-
# ignore encodings that give an output identical to the input (identity transformation)
917-
if b(input) == b(new_input):
935+
# ignore encodings that give an output identical to the input (identity transformation) or to the previous input
936+
if b(input) == b(new_input) or b(prev_input) == b(new_input):
918937
continue
919-
score = 1.0
920-
#FIXME: score the input/new_input to establish priorities of the depth-first search
921-
#This could rely on a series of weighted features:
922-
#- is input's length within a given interval (e.g. (1, 65) for base64)
923-
#- is input's length within a given interval of possible maximum lengths (e.g. (64, 65) for base64)
924-
#- is input's entropy within a given interval
925-
yield score, new_input, encoding
938+
# compute input's characteristics only once and only if the control flow reaches this point
939+
pad = ci.parameters.get('scoring', {}).get('padding_char')
940+
if obj is None:
941+
obj = _Text(input, pad)
942+
# from here, the goal (e.g. if the input is Base32) is to rank candidate encodings (e.g. multiple base codecs)
943+
# so that we can put the right one as early as possible and eventually exclude bad candidates
944+
s = .0
945+
# first, apply a bonus if the length of input text's charset is exactly the same as encoding's charset ;
946+
# on the contrary, if the length of input text's charset is strictly greater, give a penalty
947+
lcs = ci.parameters.get('scoring', {}).get('len_charset', 256)
948+
if isinstance(lcs, type(lambda: None)):
949+
lcs = int(lcs(encoding))
950+
if (pad and obj.padding and lcs + 1 == obj.lcharset) or lcs == obj.lcharset:
951+
s += .3
952+
elif (pad and obj.padding and lcs + 1 < obj.lcharset) or lcs < obj.lcharset:
953+
s -= .2 # this can occur for encodings with no_error set to True
954+
# then, take padding into account, giving a bonus if padding is to be encountered and effectively present, or a
955+
# penalty when it should not be encountered but it is present
956+
if pad and obj.padding:
957+
s += .2 # when padding is encountered while it is legitimate, it could be a good indication => good bonus
958+
elif not pad and obj.padding:
959+
s -= .1 # it could arise that a padding character is encountered while not being padding => small penalty
960+
# give a bonus when the rate of printable characters is greater or equal than expected and a penalty when lower
961+
# only for codecs that tolerate errors (otherwise, the printables rate can be biased)
962+
if not ci.parameters.get('no_error', False):
963+
pr = ci.parameters.get('scoring', {}).get('printables_rate', 0)
964+
if isinstance(pr, type(lambda: None)):
965+
pr = float(pr(obj.printables))
966+
if obj.printables - pr <= .05:
967+
s += .1
968+
# afterwards, if the input text has an entropy close to the expected one, give a bonus weighted on the number of
969+
# input characters to take bad entropies of shorter strings into account
970+
entr = ci.parameters.get('entropy', {})
971+
entr = entr.get(encoding, entr.get('default')) if isinstance(entr, dict) else entr
972+
if isinstance(entr, type(lambda: None)):
973+
try: # this case allows to consider the current encoding name from the current codec
974+
entr = entr(obj.entropy, encoding)
975+
except TypeError:
976+
entr = entr(obj.entropy)
977+
if entr is not None:
978+
# use a quadratic heuristic to compute a weight for the entropy delta, aligned on (100w, .1) and (200w, 1)
979+
d_entr = min(4e-05 * obj.len**2 - .003 * obj.len, 1) * abs(entr - obj.entropy)
980+
if d_entr <= .5:
981+
s += .5 - d_entr
982+
# finally, if relevant, apply a custom bonus (e.g. when a regex pattern is matched)
983+
bonus = ci.parameters.get('scoring', {}).get('bonus_func')
984+
if bonus is not None:
985+
if isinstance(bon, type(lambda: None)):
986+
bonus = bonus(obj, ci, encoding)
987+
if bonus:
988+
s += .2
989+
# exclude negative (and eventually null) scores as they are (hopefully) not relevant
990+
if extended and s >= .0 or not extended and s > .0:
991+
yield s, new_input, encoding
926992

927993

928994
def guess(input, stop_func=stopfunc.printables, max_depth=5, codec_categories=None, exclude=None, found=(), stop=True,
929-
show=False):
995+
show=False, extended=False):
930996
""" Try decoding without the knowledge of the encoding(s). """
931997
if max_depth <= 0:
932998
raise ValueError("Depth must be a non-null positive integer")
933999
if len(found) > 0:
9341000
for encoding in found:
9351001
input = decode(input, encoding)
9361002
if isinstance(stop_func, string_types):
937-
p = stop_func
938-
stop_func = lambda s: re.search(ensure_str(p).lower(), ensure_str(s).lower()) is not None
1003+
stop_func = stopfunc.regex(stop_func)
9391004
if len(input) > 0:
9401005
result = []
9411006
for d in range(max_depth):
942-
__guess(input, stop_func, 0, d+1, codec_categories or [], exclude or [], result, tuple(found), stop, show)
1007+
__guess("", input, stop_func, 0, d+1, codec_categories, exclude, result, tuple(found), stop, show, extended)
9431008
if stop and len(result) > 0:
9441009
return result
9451010
return result

‎codext/base/_base.py‎

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,7 @@ class BaseEncodeError(BaseError):
2323

2424

2525
def _generate_charset(n):
26-
"""
27-
Generate a characters set.
26+
""" Generate a characters set.
2827
2928
:param n: size of charset
3029
"""
@@ -36,8 +35,7 @@ def _generate_charset(n):
3635

3736

3837
def _get_charset(charset, p=""):
39-
"""
40-
Characters set selection function. It allows to define charsets in many different ways.
38+
""" Characters set selection function. It allows to define charsets in many different ways.
4139
4240
:param charset: charset object, can be a string (the charset itself), a function (that chooses the right charset
4341
depending on the input parameter) or a dictionary (either by exact key or by pattern matching)
@@ -81,8 +79,7 @@ def _get_charset(charset, p=""):
8179

8280
# generic base en/decoding functions
8381
def base_encode(input, charset, errors="strict", exc=BaseEncodeError):
84-
"""
85-
Base-10 to base-N encoding.
82+
""" Base-10 to base-N encoding.
8683
8784
:param input: input (str or int) to be decoded
8885
:param charset: base-N characters set
@@ -99,8 +96,7 @@ def base_encode(input, charset, errors="strict", exc=BaseEncodeError):
9996

10097

10198
def base_decode(input, charset, errors="strict", exc=BaseDecodeError):
102-
"""
103-
Base-N to base-10 decoding.
99+
""" Base-N to base-10 decoding.
104100
105101
:param input: input to be decoded
106102
:param charset: base-N characters set
@@ -117,9 +113,8 @@ def base_decode(input, charset, errors="strict", exc=BaseDecodeError):
117113

118114

119115
# base codec factory functions
120-
def base(charset, pattern, pow2=False, encode_template=base_encode, decode_template=base_decode, name=None):
121-
"""
122-
Base-N codec factory.
116+
def base(charset, pattern, pow2=False, encode_template=base_encode, decode_template=base_decode, name=None, **kwargs):
117+
""" Base-N codec factory.
123118
124119
:param charset: charset selection function
125120
:param pattern: matching pattern for the codec name (first capturing group is used as the parameter for selecting
@@ -143,13 +138,13 @@ def _decode(input, errors="strict"):
143138
return decode_template(input, a, errors), len(input)
144139
return _decode
145140

146-
add("base{}".format(n) if name is None else name, encode, decode, pattern)
141+
kwargs['len_charset'] = n
142+
kwargs['printables_rate'] = 1.
143+
add("base{}".format(n) if name is None else name, encode, decode, pattern, entropy=nb, **kwargs)
147144

148145

149146
def base_generic():
150-
"""
151-
Base-N generic codec.
152-
"""
147+
""" Base-N generic codec. """
153148
def encode(n):
154149
a = _generate_charset(int(n))
155150
def _encode(input, errors="strict"):
@@ -163,5 +158,6 @@ def _decode(input, errors="strict"):
163158
return _decode
164159

165160
add("base", encode, decode, r"^base[-_]?([2-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(?:[-_]generic)?$",
166-
guess=["base%d-generic" % i for i in range(2, 255)])
161+
guess=["base%d-generic" % i for i in range(2, 255)], entropy=lambda e, n: log(int(n.split("-")[0][4:]), 2),
162+
len_charset=lambda n: int(n.split("-")[0][4:]), printables_rate=1.)
167163

‎codext/base/_base2n.py‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,19 @@ class Base2NEncodeError(BaseError):
2121
pass
2222

2323

24-
def base2n(charset, pattern=None, name=None):
25-
"""
26-
Base-N codec factory for N a power of 2.
24+
def base2n(charset, pattern=None, name=None, **kwargs):
25+
""" Base-N codec factory for N a power of 2.
2726
2827
:param charset: charset selection function
2928
:param pattern: matching pattern for the codec name (first capturing group is used as the parameter for selecting
3029
the charset)
3130
:param name: forced encoding name (useful e.g. for zbase32)
3231
"""
33-
base(charset, pattern, True, base2n_encode, base2n_decode, name)
32+
base(charset, pattern, True, base2n_encode, base2n_decode, name, **kwargs)
3433

3534

3635
def base2n_encode(string, charset, errors="strict", exc=Base2NEncodeError):
37-
"""
38-
8-bits characters to base-N encoding for N a power of 2.
36+
""" 8-bits characters to base-N encoding for N a power of 2.
3937
4038
:param string: string to be decoded
4139
:param charset: base-N characters set
@@ -67,8 +65,7 @@ def base2n_encode(string, charset, errors="strict", exc=Base2NEncodeError):
6765

6866

6967
def base2n_decode(string, charset, errors="strict", exc=Base2NDecodeError):
70-
"""
71-
Base-N to 8-bits characters decoding for N a power of 2.
68+
""" Base-N to 8-bits characters decoding for N a power of 2.
7269
7370
:param string: string to be decoded
7471
:param charset: base-N characters set

‎codext/base/ascii85.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,5 +24,5 @@ def ascii85_encode(input, errors='strict'):
2424
def ascii85_decode(input, errors='strict'):
2525
return base64.a85decode(b(input)), len(input)
2626

27-
add("ascii85", ascii85_encode, ascii85_decode, r"^ascii[-_]?85$")
27+
add("ascii85", ascii85_encode, ascii85_decode, r"^ascii[-_]?85$", entropy=6.36)
2828

‎codext/base/base100.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@
1111
"""
1212
from ..__common__ import *
1313

14+
1415
# no __examples__ ; handled manually in tests/test_base.py
1516

17+
1618
if PY3:
1719
class Base100DecodeError(ValueError):
1820
pass

‎codext/base/base122.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,10 @@
99
"""
1010
from ..__common__ import *
1111

12+
1213
# no __examples__ ; handled manually in tests/test_base.py
1314

15+
1416
def base122_encode(input, errors="strict"):
1517
raise NotImplementedError
1618

‎codext/base/base85.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,5 @@ def base85_encode(input, errors='strict'):
3434
def base85_decode(input, errors='strict'):
3535
return base64.b85decode(b(input)), len(input)
3636

37-
add("base85", base85_encode, base85_decode, r"^base[-_]?85$")
37+
add("base85", base85_encode, base85_decode, r"^base[-_]?85$", entropy=7.05)
3838

‎codext/base/base91.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,5 +83,5 @@ def decode(text, errors="strict"):
8383
return decode
8484

8585

86-
add("base91", base91_encode, base91_decode, r"^base[-_]?91(|[-_]inv(?:erted)?)$")
86+
add("base91", base91_encode, base91_decode, r"^base[-_]?91(|[-_]inv(?:erted)?)$", entropy=6.5)
8787

‎codext/base/baseN.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,9 @@
3838
r'(?:[-_]ext(?:ended)?)?[-_]hex$': digits + upper[:22],
3939
r'[-_]geohash': digits + "bcdefghjkmnpqrstuvwxyz",
4040
}
41-
base2n(B32, r"^base[-_]?32(|[-_]inv(?:erted)?|(?:[-_]ext(?:ended)?)?[-_]hex|[-_]geohash)$")
41+
base2n(B32, r"^base[-_]?32(|[-_]inv(?:erted)?|(?:[-_]ext(?:ended)?)?[-_]hex|[-_]geohash)$", padding_char="=")
4242
ZB32 = {'': "ybndrfg8ejkmcpqxot1uwisza345h769"}
43-
base2n(ZB32, r"^z[-_]?base[-_]?32$", name="zbase32")
43+
base2n(ZB32, r"^z[-_]?base[-_]?32$", name="zbase32", padding_char="=")
4444

4545

4646
B36 = {'': digits + upper, 'inv': upper + digits}
@@ -64,7 +64,7 @@
6464
r'[-_]inv(erted)?$': lower + upper + digits + "+/",
6565
r'[-_]?(file|url)(safe)?$': upper + lower + digits + "-_",
6666
}
67-
base2n(B64, r"^base[-_]?64(|[-_]inv(?:erted)?|[-_]?(?:file|url)(?:safe)?)$")
67+
base2n(B64, r"^base[-_]?64(|[-_]inv(?:erted)?|[-_]?(?:file|url)(?:safe)?)$", padding_char="=")
6868

6969

7070
# generic base encodings, to be added after all others as they have the precedence

0 commit comments

Comments
 (0)