Skip to content

Commit 0d13231

Browse files
dhondtadhondta
committed
Improved codext tool
1 parent d09dd0b commit 0d13231

1 file changed

Lines changed: 65 additions & 50 deletions

File tree

‎codext/__init__.py‎

Lines changed: 65 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,23 @@
3333
pattern=r"^uu(?:[-_]encode|codec)?$", add_to_codecs=True, category="native")
3434

3535

36-
def __literal_eval(o):
37-
""" Non-failing ast.literal_eval alias function. """
38-
try:
39-
return literal_eval(str(o))
40-
except ValueError:
41-
return literal_eval("'" + str(o) + "'")
36+
def __format_list(items, include=True):
37+
if items is None:
38+
return
39+
d = {-1: list_encodings() if include else []}
40+
for n, i in enumerate(items):
41+
try:
42+
depth, i = i.split(":")
43+
depth = int(depth.strip().replace("~", "-"))
44+
if depth < 0:
45+
depth = -1
46+
except ValueError:
47+
if n == 0:
48+
d[-1] = []
49+
depth = -1
50+
d.setdefault(depth, [])
51+
d[depth].append(i.strip())
52+
return d
4253

4354

4455
def __print_tabular(lst, space=4):
@@ -70,6 +81,19 @@ def __print_tabular(lst, space=4):
7081

7182
def main():
7283
import argparse, os
84+
85+
class _CustomFormatter(argparse.RawTextHelpFormatter):
86+
def __init__(self, prog, **kwargs):
87+
kwargs['max_help_position'] = 32
88+
super(_CustomFormatter, self).__init__(prog, **kwargs)
89+
90+
def _format_action_invocation(self, action):
91+
if not action.option_strings:
92+
metavar, = self._metavar_formatter(action, action.dest)(1)
93+
return metavar
94+
else:
95+
return ", ".join(action.option_strings)
96+
7397
descr = "Codecs Extension (CodExt) {}\n\nAuthor : {} ({})\nCopyright: {}\nLicense : {}\nSource : {}\n" \
7498
"\nThis tool allows to encode/decode input strings/files with an extended set of codecs.\n\n" \
7599
.format(__version__, __author__, __email__, __copyright__, __license__, __source__)
@@ -87,78 +111,76 @@ def main():
87111
"echo -en \"test\" | codext encode base64 gzip | codext guess",
88112
"echo -en \"test\" | codext encode base64 gzip | codext guess gzip -c base",
89113
])
90-
parser = argparse.ArgumentParser(description=descr, epilog=examples, formatter_class=argparse.RawTextHelpFormatter)
91-
sparsers = parser.add_subparsers(dest="command", required=True, help="command to be executed")
114+
kw = {'formatter_class': _CustomFormatter}
115+
parser = argparse.ArgumentParser(description=descr, epilog=examples, **kw)
116+
kw2 = {'required': True} if PY3 else {}
117+
sparsers = parser.add_subparsers(dest="command", help="command to be executed", **kw2)
92118
parser.add_argument("-i", "--input-file", dest="infile", help="input file (if none, take stdin as input)")
93119
parser.add_argument("-o", "--output-file", dest="outfile", help="output file (if none, display result to stdout)")
94120
parser.add_argument("-s", "--strip-newlines", action="store_true", dest="strip",
95121
help="strip newlines from input (default: False)")
96-
encode = sparsers.add_parser("encode", help="encode input using the specified codecs")
122+
encode = sparsers.add_parser("encode", help="encode input using the specified codecs", **kw)
97123
encode.add_argument("encoding", nargs="+", help="list of encodings to apply")
98124
encode.add_argument("-e", "--errors", default="strict", choices=["ignore", "leave", "replace", "strict"],
99125
help="error handling (default: strict)")
100-
decode = sparsers.add_parser("decode", help="decode input using the specified codecs")
126+
decode = sparsers.add_parser("decode", help="decode input using the specified codecs", **kw)
101127
decode.add_argument("encoding", nargs="+", help="list of encodings to apply")
102128
decode.add_argument("-e", "--errors", default="strict", choices=["ignore", "leave", "replace", "strict"],
103129
help="error handling (default: strict)")
104-
guess = sparsers.add_parser("guess", help="try guessing the decoding codecs")
130+
guess = sparsers.add_parser("guess", help="try guessing the decoding codecs", **kw)
105131
guess.add_argument("encoding", nargs="*", help="list of known encodings to apply (default: none)")
106-
guess.add_argument("-c", "--codec-categories", nargs="*", help="codec categories to be included in the search ; "
107-
"format: string|tuple")
108-
guess.add_argument("-d", "--min-depth", default=0, type=int, help="minimum codec search depth before triggering "
109-
"results (default: 0)")
110-
guess.add_argument("-D", "--max-depth", default=5, type=int, help="maximum codec search depth (default: 5)")
111-
guess.add_argument("-e", "--exclude-codecs", nargs="*", help="codecs to be explicitely not used ; "
112-
"format: string|tuple")
132+
guess.add_argument("-e", "--exclude", nargs="*", action="extend", metavar="CAT|COD|ENC",
133+
help="categories, codecs and encodings to be explicitely not used ;\n "
134+
"format: [category|codec|encoding] OR depth:[category|codec|encoding]")
113135
guess.add_argument("-E", "--extended", action="store_true",
114136
help="while using the scoring heuristic, also consider null scores (default: False)")
115137
lng = "lang_%s" % LANG
116138
def_func = lng if getattr(stopfunc, lng, None) else "text"
117-
guess.add_argument("-f", "--stop-function", default=def_func, help="result checking function (default: %s) ; "
118-
"format: printables|text|flag|lang_[bigram]|[regex]\nNB: [regex] is case-sensitive ; add -i to "
119-
"force it as case-insensitive or add '(?i)' in front of the expression" % def_func)
120-
guess.add_argument("-i", "--case-insensitive", dest="icase", action="store_true",
121-
help="while using the regex stop function, set it as case-insensitive (default: False)")
139+
guess.add_argument("-f", "--stop-function", default=def_func, metavar="FUNC", help="result checking function "
140+
"(default: %s) ; format: printables|text|flag|lang_[bigram]|[regex]\nNB: [regex] is case-"
141+
"sensitive ; add -i to force it as case-insensitive or add '(?i)' in front of the expression"
142+
% def_func)
122143
guess.add_argument("-H", "--no-heuristic", action="store_true", help="DO NOT use the scoring heuristic ; slows down"
123144
" the search but may be more accurate (default: False)")
145+
guess.add_argument("-i", "--include", nargs="*", action="extend", metavar="CAT|COD|ENC",
146+
help="categories, codecs and encodings to be explicitely used ;\n "
147+
"format: [category|codec|encoding] OR depth:[category|codec|encoding]")
148+
guess.add_argument("-I", "--case-insensitive", dest="icase", action="store_true",
149+
help="while using the regex stop function, set it as case-insensitive (default: False)")
124150
if len(stopfunc.LANG_BACKENDS) > 0:
125151
_lb = stopfunc.LANG_BACKEND
126152
guess.add_argument("-l", "--lang-backend", default=_lb, choices=stopfunc.LANG_BACKENDS + ["none"],
127153
help="natural language detection backend (default: %s)" % _lb)
154+
guess.add_argument("-m", "--min-depth", default=0, type=int, metavar="INT",
155+
help="minimum codec search depth before triggering results (default: 0)")
156+
guess.add_argument("-M", "--max-depth", default=5, type=int, metavar="INT",
157+
help="maximum codec search depth (default: 5)")
128158
guess.add_argument("-s", "--do-not-stop", action="store_true",
129159
help="do not stop if a valid output is found (default: False)")
130160
guess.add_argument("-v", "--verbose", action="store_true",
131161
help="show guessing information and steps (default: False)")
132-
rank = sparsers.add_parser("rank", help="rank the most probable encodings based on the given input")
133-
rank.add_argument("-c", "--codec-categories", help="codec categories to be included in the search ; "
134-
"format: string|tuple|list(strings|tuples)")
135-
rank.add_argument("-e", "--exclude-codecs", help="codecs to be explicitely not used ; "
136-
"format: string|tuple|list(strings|tuples)")
162+
rank = sparsers.add_parser("rank", help="rank the most probable encodings based on the given input", **kw)
163+
rank.add_argument("-c", "--codec-categories", nargs="*", action="extend", metavar="CATEGORY",
164+
help="codec categories to be included in the search ; format: string|tuple|list(strings|tuples)")
165+
rank.add_argument("-e", "--exclude-codecs", nargs="*", action="extend", metavar="CODEC",
166+
help="codecs to be explicitely not used ; format: string|tuple|list(strings|tuples)")
137167
rank.add_argument("-E", "--extended", action="store_true",
138168
help="while using the scoring heuristic, also consider null scores (default: False)")
139169
rank.add_argument("-l", "--limit", type=int, default=10, help="limit the number of displayed results")
140170
search = sparsers.add_parser("search", help="search for codecs")
141171
search.add_argument("pattern", nargs="+", help="encoding pattern to search")
142172
listi = sparsers.add_parser("list", help="list items")
143-
lsparsers = listi.add_subparsers(dest="type", required=True, help="type of item to be listed")
173+
lsparsers = listi.add_subparsers(dest="type", help="type of item to be listed", **kw2)
144174
liste = lsparsers.add_parser("encodings", help="list encodings")
145-
liste.add_argument("category", nargs="*", help="selected categories")
175+
liste.add_argument("category", nargs="+", help="selected categories")
146176
listm = lsparsers.add_parser("macros", help="list macros")
147177
addm = sparsers.add_parser("add-macro", help="add a macro to the registry")
148178
addm.add_argument("name", help="macro's name")
149179
addm.add_argument("encoding", nargs="+", help="list of encodings to chain")
150180
remm = sparsers.add_parser("remove-macro", help="remove a macro from the registry")
151181
remm.add_argument("name", help="macro's name")
152182
args = parser.parse_args()
153-
try:
154-
args.codec_categories = _lst(map(__literal_eval, args.codec_categories))
155-
except (AttributeError, TypeError):
156-
pass
157-
try:
158-
args.exclude_codecs = _lst(map(__literal_eval, args.exclude_codecs))
159-
except (AttributeError, TypeError):
160-
pass
161-
#print(args.codec_categories, args.exclude_codecs)
183+
args.include, args.exclude = __format_list(args.include), __format_list(args.exclude, False)
162184
try:
163185
# if a search pattern is given, only handle it
164186
if args.command == "search":
@@ -211,17 +233,9 @@ def main():
211233
all(re.match(r"lang_[a-z]{2}$", x) is None for x in dir(stopfunc)):
212234
stopfunc._reload_lang(lb)
213235
r = codecs.guess(c,
214-
getattr(stopfunc, s, ["", "(?i)"][args.icase] + s),
215-
args.min_depth,
216-
args.max_depth,
217-
args.codec_categories,
218-
args.exclude_codecs,
219-
args.encoding,
220-
not args.do_not_stop,
221-
True, # show
222-
not args.no_heuristic,
223-
args.extended,
224-
args.verbose)
236+
getattr(stopfunc, s, ["", "(?i)"][args.icase] + s), args.min_depth, args.max_depth,
237+
args.include, args.exclude, args.encoding, not args.do_not_stop, True, # show
238+
not args.no_heuristic, args.extended, args.verbose)
225239
for i, o in enumerate(r.items()):
226240
e, out = o
227241
if len(e) > 0:
@@ -238,6 +252,7 @@ def main():
238252
s = "[+] %.5f: %s" % (i[0], e)
239253
print(s if len(s) <= 80 else s[:77] + "...")
240254
except Exception as e:
255+
raise e
241256
m = str(e)
242257
print("codext: " + m[0].lower() + m[1:])
243258

0 commit comments

Comments
 (0)