Skip to content

Commit 89d4e47

Browse files
dhondtadhondta
committed
Improved CLI script
1 parent adf5e8f commit 89d4e47

2 files changed

Lines changed: 76 additions & 65 deletions

File tree

‎codext/__common__.py‎

Lines changed: 58 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -864,7 +864,7 @@ def _flag(x):
864864

865865

866866
def __guess(prev_input, input, stop_func, depth, max_depth, codec_categories, exclude, result, found=(), stop=True,
867-
show=False, extended=False):
867+
show=False, scoring_heuristic=False, extended=False):
868868
""" Perform a breadth-first tree search using a ranking logic to select and prune the list of codecs. """
869869
if depth > 0 and stop_func(input):
870870
if not stop and show:
@@ -897,18 +897,18 @@ def expand(items, descr=None, transform=None):
897897
return r if transform is None else transform(*r)
898898
# parse valid encodings, expanding included/excluded codecs
899899
c, e = expand(codec_categories, "codec_categories", list_encodings), expand(exclude, "exclude")
900-
for new_input, encoding in __rank(prev_input, input, c, extended):
900+
for new_input, encoding in __rank(prev_input, input, c, scoring_heuristic, extended):
901901
if encoding in e:
902902
continue
903903
__guess(input, new_input, stop_func, depth+1, max_depth, codec_categories, exclude, result,
904904
found + (encoding, ), stop, show, extended)
905905

906906

907-
def __rank(prev_input, input, codecs, extended=False):
907+
def __rank(prev_input, input, codecs, heuristic=False, extended=False):
908908
""" Filter valid encodings and rank them by relevance. """
909909
ranking = {}
910910
for codec in codecs:
911-
for score, new_input, encoding in __score(prev_input, input, codec, extended):
911+
for score, new_input, encoding in __score(prev_input, input, codec, heuristic, extended):
912912
ranking[encoding] = (score, new_input)
913913
for encoding, result in sorted(ranking.items(), key=lambda x: -x[1][0]):
914914
yield result[1], encoding
@@ -923,7 +923,7 @@ def __init__(self, text, pad_char=None):
923923
self.entropy = entropy(text)
924924

925925

926-
def __score(prev_input, input, codec, extended=False):
926+
def __score(prev_input, input, codec, heuristic=False, extended=False):
927927
""" Score relevant encodings given an input. """
928928
obj, ci = None, lookup(codec) # NB: lookup(...) won't fail as the codec value comes from list_encodings(...)
929929
for encoding in ci.parameters.get('guess', [codec]):
@@ -939,60 +939,63 @@ def __score(prev_input, input, codec, extended=False):
939939
pad = ci.parameters.get('scoring', {}).get('padding_char')
940940
if obj is None:
941941
obj = _Text(input, pad)
942-
# from here, the goal (e.g. if the input is Base32) is to rank candidate encodings (e.g. multiple base codecs)
943-
# so that we can put the right one as early as possible and eventually exclude bad candidates
944-
s = .0
945-
# first, apply a bonus if the length of input text's charset is exactly the same as encoding's charset ;
946-
# on the contrary, if the length of input text's charset is strictly greater, give a penalty
947-
lcs = ci.parameters.get('scoring', {}).get('len_charset', 256)
948-
if isinstance(lcs, type(lambda: None)):
949-
lcs = int(lcs(encoding))
950-
if (pad and obj.padding and lcs + 1 == obj.lcharset) or lcs == obj.lcharset:
951-
s += .3
952-
elif (pad and obj.padding and lcs + 1 < obj.lcharset) or lcs < obj.lcharset:
953-
s -= .2 # this can occur for encodings with no_error set to True
954-
# then, take padding into account, giving a bonus if padding is to be encountered and effectively present, or a
955-
# penalty when it should not be encountered but it is present
956-
if pad and obj.padding:
957-
s += .2 # when padding is encountered while it is legitimate, it could be a good indication => good bonus
958-
elif not pad and obj.padding:
959-
s -= .1 # it could arise that a padding character is encountered while not being padding => small penalty
960-
# give a bonus when the rate of printable characters is greater or equal than expected and a penalty when lower
961-
# only for codecs that tolerate errors (otherwise, the printables rate can be biased)
962-
if not ci.parameters.get('no_error', False):
963-
pr = ci.parameters.get('scoring', {}).get('printables_rate', 0)
964-
if isinstance(pr, type(lambda: None)):
965-
pr = float(pr(obj.printables))
966-
if obj.printables - pr <= .05:
967-
s += .1
968-
# afterwards, if the input text has an entropy close to the expected one, give a bonus weighted on the number of
969-
# input characters to take bad entropies of shorter strings into account
970-
entr = ci.parameters.get('entropy', {})
971-
entr = entr.get(encoding, entr.get('default')) if isinstance(entr, dict) else entr
972-
if isinstance(entr, type(lambda: None)):
973-
try: # this case allows to consider the current encoding name from the current codec
974-
entr = entr(obj.entropy, encoding)
975-
except TypeError:
976-
entr = entr(obj.entropy)
977-
if entr is not None:
978-
# use a quadratic heuristic to compute a weight for the entropy delta, aligned on (100w, .1) and (200w, 1)
979-
d_entr = min(4e-05 * obj.len**2 - .003 * obj.len, 1) * abs(entr - obj.entropy)
980-
if d_entr <= .5:
981-
s += .5 - d_entr
982-
# finally, if relevant, apply a custom bonus (e.g. when a regex pattern is matched)
983-
bonus = ci.parameters.get('scoring', {}).get('bonus_func')
984-
if bonus is not None:
985-
if isinstance(bon, type(lambda: None)):
986-
bonus = bonus(obj, ci, encoding)
987-
if bonus:
988-
s += .2
942+
if heuristic:
943+
# from here, the goal (e.g. if the input is Base32) is to rank candidate encodings (e.g. multiple base
944+
# codecs) so that we can put the right one as early as possible and eventually exclude bad candidates
945+
s = .0
946+
# first, apply a bonus if the length of input text's charset is exactly the same as encoding's charset ;
947+
# on the contrary, if the length of input text's charset is strictly greater, give a penalty
948+
lcs = ci.parameters.get('scoring', {}).get('len_charset', 256)
949+
if isinstance(lcs, type(lambda: None)):
950+
lcs = int(lcs(encoding))
951+
if (pad and obj.padding and lcs + 1 == obj.lcharset) or lcs == obj.lcharset:
952+
s += .3
953+
elif (pad and obj.padding and lcs + 1 < obj.lcharset) or lcs < obj.lcharset:
954+
s -= .2 # this can occur for encodings with no_error set to True
955+
# then, take padding into account, giving a bonus if padding is to be encountered and effectively present,
956+
# or a penalty when it should not be encountered but it is present
957+
if pad and obj.padding:
958+
s += .2 # when padding is encountered while it is legitimate, it could be a good indication => bonus
959+
elif not pad and obj.padding:
960+
s -= .1 # it could arise a padding character is encountered while not being padding => small penalty
961+
# give a bonus when the rate of printable characters is greater or equal than expected and a penalty when
962+
# lower only for codecs that tolerate errors (otherwise, the printables rate can be biased)
963+
if not ci.parameters.get('no_error', False):
964+
pr = ci.parameters.get('scoring', {}).get('printables_rate', 0)
965+
if isinstance(pr, type(lambda: None)):
966+
pr = float(pr(obj.printables))
967+
if obj.printables - pr <= .05:
968+
s += .1
969+
# afterwards, if the input text has an entropy close to the expected one, give a bonus weighted on the
970+
# number of input characters to take bad entropies of shorter strings into account
971+
entr = ci.parameters.get('entropy', {})
972+
entr = entr.get(encoding, entr.get('default')) if isinstance(entr, dict) else entr
973+
if isinstance(entr, type(lambda: None)):
974+
try: # this case allows to consider the current encoding name from the current codec
975+
entr = entr(obj.entropy, encoding)
976+
except TypeError:
977+
entr = entr(obj.entropy)
978+
if entr is not None:
979+
# use a quadratic heuristic to compute a weight for the entropy delta, aligned on (100w,.1) and (200w,1)
980+
d_entr = min(4e-05 * obj.len**2 - .003 * obj.len, 1) * abs(entr - obj.entropy)
981+
if d_entr <= .5:
982+
s += .5 - d_entr
983+
# finally, if relevant, apply a custom bonus (e.g. when a regex pattern is matched)
984+
bonus = ci.parameters.get('scoring', {}).get('bonus_func')
985+
if bonus is not None:
986+
if isinstance(bon, type(lambda: None)):
987+
bonus = bonus(obj, ci, encoding)
988+
if bonus:
989+
s += .2
990+
else:
991+
s = 1.
989992
# exclude negative (and eventually null) scores as they are (hopefully) not relevant
990993
if extended and s >= .0 or not extended and s > .0:
991994
yield s, new_input, encoding
992995

993996

994997
def guess(input, stop_func=stopfunc.printables, max_depth=5, codec_categories=None, exclude=None, found=(), stop=True,
995-
show=False, extended=False):
998+
show=False, scoring_heuristic=False, extended=False):
996999
""" Try decoding without the knowledge of the encoding(s). """
9971000
if max_depth <= 0:
9981001
raise ValueError("Depth must be a non-null positive integer")
@@ -1004,7 +1007,8 @@ def guess(input, stop_func=stopfunc.printables, max_depth=5, codec_categories=No
10041007
if len(input) > 0:
10051008
result = []
10061009
for d in range(max_depth):
1007-
__guess("", input, stop_func, 0, d+1, codec_categories, exclude, result, tuple(found), stop, show, extended)
1010+
__guess("", input, stop_func, 0, d+1, codec_categories, exclude, result, tuple(found), stop, show,
1011+
scoring_heuristic, extended)
10081012
if stop and len(result) > 0:
10091013
return result
10101014
return result

‎codext/__init__.py‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -67,23 +67,29 @@ def main():
6767
sparsers = parser.add_subparsers(dest="command", help="command to be executed")
6868
parser.add_argument("-i", "--input-file", dest="infile", help="input file (if none, take stdin as input)")
6969
parser.add_argument("-o", "--output-file", dest="outfile", help="output file (if none, display result to stdout)")
70-
parser.add_argument("-s", "--strip-newlines", action="store_true", dest="strip", help="strip newlines from input")
70+
parser.add_argument("-s", "--strip-newlines", action="store_true", dest="strip",
71+
help="strip newlines from input (default: False)")
7172
encode = sparsers.add_parser("encode", help="encode input using the specified codecs")
7273
encode.add_argument("encoding", nargs="+", help="list of encodings to apply")
7374
encode.add_argument("-e", "--errors", default="strict", choices=["ignore", "leave", "replace", "strict"],
74-
help="error handling")
75+
help="error handling (default: strict)")
7576
decode = sparsers.add_parser("decode", help="decode input using the specified codecs")
7677
decode.add_argument("encoding", nargs="+", help="list of encodings to apply")
7778
decode.add_argument("-e", "--errors", default="strict", choices=["ignore", "leave", "replace", "strict"],
78-
help="error handling")
79+
help="error handling (default: strict)")
7980
guess = sparsers.add_parser("guess", help="try guessing the decoding codecs")
80-
guess.add_argument("encoding", nargs="*", help="list of known encodings to apply")
81-
guess.add_argument("-c", "--codec-categories", help="codec categories to be included in the search\n"
82-
"format: string|tuple|list(strings|tuples) ; see the documentation")
83-
guess.add_argument("-d", "--depth", default=3, type=int, help="maximum codec search depth")
84-
guess.add_argument("-e", "--exclude-codecs", help="codecs to be explicitely not used\n"
85-
"format: string|tuple|list(strings|tuples) ; see the documentation")
86-
guess.add_argument("-s", "--do-not-stop", action="store_true", help="do not stop if a valid output is found")
81+
guess.add_argument("encoding", nargs="*", help="list of known encodings to apply (default: none)")
82+
guess.add_argument("-c", "--codec-categories", help="codec categories to be included in the search ; "
83+
"format: string|tuple|list(strings|tuples)")
84+
guess.add_argument("-d", "--depth", default=3, type=int, help="maximum codec search depth (default: 3)")
85+
guess.add_argument("-e", "--exclude-codecs", help="codecs to be explicitely not used ; "
86+
"format: string|tuple|list(strings|tuples)")
87+
guess.add_argument("--extended", action="store_true",
88+
help="while using the scoring heuristic, also consider null scores (default: False)")
89+
guess.add_argument("--heuristic", action="store_true",
90+
help="use a scoring heuristic to accelerate guessing (default: False)")
91+
guess.add_argument("-s", "--do-not-stop", action="store_true",
92+
help="do not stop if a valid output is found (default: False)")
8793
search = sparsers.add_parser("search", help="search for codecs")
8894
search.add_argument("pattern", nargs="+", help="encoding pattern to search")
8995
args = parser.parse_args()
@@ -116,7 +122,8 @@ def main():
116122
print(ensure_str(c or "Could not decode :-("), end="")
117123
elif args.command == "guess":
118124
l = [o for o in codecs.guess(c, stopfunc.printables, args.depth, __literal_eval(args.codec_categories),
119-
__literal_eval(args.exclude_codecs), args.encoding, not args.do_not_stop, True)]
125+
__literal_eval(args.exclude_codecs), args.encoding, not args.do_not_stop, True,
126+
args.heuristic, args.extended)]
120127
for i, o in enumerate(l):
121128
out, e = o
122129
if len(e) > 0:

0 commit comments

Comments
 (0)