-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathProcessor.py
More file actions
599 lines (493 loc) · 19.2 KB
/
Copy pathProcessor.py
File metadata and controls
599 lines (493 loc) · 19.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
import logging
import re
from collections import defaultdict
# import bpdb # noqa: F401
log = logging.getLogger(__name__)
class Processor:
"""
processed records created by Parse class
from: https://docs.python.org/3.11/reference/lexical_analysis.html#formatted-string-literals
.. code-block::
f_string ::= (literal_char | "{{" | "}}" | replacement_field)*
replacement_field ::= "{" f_expression ["="] ["!" conversion] [":" format_spec] "}"
f_expression ::= (conditional_expression | "*" or_expr) \
("," conditional_expression | "," "*" or_expr)* [","] | yield_expression
conversion ::= "s" | "r" | "a"
format_spec ::= (literal_char | NULL | replacement_field)*
literal_char ::= <any code point except "{", "}" or NULL>
"""
def __init__(self, args=None, **kwargs):
# https://regex101.com/r/5cY7CW/1
self.pattern = r"(\{)([^}:]+)(?=(:[^}]+)?(\}))"
self.vars = []
def get_char_replacements(self, in_str):
"""
find something to replace double brackets and :: that isn't in the in_str
also see https://clang.llvm.org/doxygen/LiteralSupport_8cpp_source.html
"""
lbracket = self.get_unique(in_str, "⟪")
rbracket = self.get_unique(in_str, "⟫")
doublecolon = self.get_unique(in_str, "__DOUBLECOLON__")
return lbracket, rbracket, doublecolon
def get_unique(self, in_str, target):
while in_str.find(target) > 0:
target += target
return target
def fstring_elem_callback(self, match):
"""
ok now look for { in string and assume each one has a corresponding var
"""
var = match[2].strip()
if not var:
log.warning(f" no var found in fstring: {match}")
ends_with_equal = var.endswith("=")
if ends_with_equal:
var = var.rstrip("=")
self.vars.append(var)
if ends_with_equal:
repl = f"{var}={{"
else:
repl = "{"
return repl
def gen_fstring_changes(self, records):
changes = []
for rec in records:
"""
convert:
"this is a {foo_bar} test"
to:
fmt::format("this is a {} test", foo_bar)
------ f_str ------ -v_str-
"""
# in_str = repr(rec.value)[1:-1] # escape backslash
in_str = rec.spelling
(lbracket, rbracket, doublecolon) = self.get_char_replacements(in_str)
rbacket_rev = rbracket[::-1]
# replace protected patterns like :: {{ }} so regex doesn't have to real with them
in_str = in_str.replace("::", doublecolon)
in_str = in_str.replace("{{", lbracket)
# right-to-left replace
in_str = in_str[::-1].replace("}}", rbacket_rev)[::-1]
# implement a version of :
# https://docs.python.org/3/whatsnew/3.8.html#f-strings-support-for-self-documenting-expressions-and-debugging # noqa: E501
self.vars = []
f_str = re.sub(self.pattern, self.fstring_elem_callback, in_str)
f_str = f_str.replace(lbracket, "{{")
f_str = f_str.replace(rbracket, "}}")
f_str = f_str.replace(doublecolon, "::")
# are there any vars or const inside brackets?
if self.vars:
v_str = ", ".join(self.vars)
v_str = v_str.replace(doublecolon, "::")
replacement_str = f"fmt::format({f_str}, {v_str})"
else:
replacement_str = f_str
changes.append([rec, replacement_str])
return changes
def gen_class_changes_old(self, records):
"""
add a friend format statement to classes with private vars
OK: PUBLIC
NOT OK: INVALID PROTECTED PRIVATE NONE?
"""
changes = []
for rec in records:
ok_to_be_friends = rec.wants_to_be_friends and not rec.is_external
if ok_to_be_friends:
replacement_str = f" friend struct fmt::formatter<{rec.name}>;\n"
replacement_str += rec.last_tok.spelling
changes.append([rec.last_tok, replacement_str])
return changes
def remove_duplicate_class_instances(self, records):
"""
sometimes a class ends up in AST with template and non-template version
"""
rec_by_loc = defaultdict(list)
for rec in records:
if not rec.is_external:
loc = rec.last_tok.location
key = tuple([loc.line, loc.column])
rec_by_loc[key].append(rec)
#
# ok now remove duplicates, giving CLASS_TEMPLATE priority over CLASS_DECL
# ---> - --> -
# hack: T comes after D so...
for key, recs in rec_by_loc.items():
recs.sort(key=lambda rec: rec.class_kind)
recs[-1].needs_to_string = True
def gen_class_changes(self, records):
"""
inject to_string entry into every class/struct
"""
self.remove_duplicate_class_instances(records)
changes = []
for rec in records:
if not rec.is_external and rec.needs_to_string:
replacement_str = self.gen_to_string(rec)
replacement_str += rec.last_tok.spelling
changes.append([rec.last_tok, replacement_str])
"""
inject friend entry for all derived classes
"""
for rec in records:
if not rec.is_external:
for base in rec.bases:
replacement_str = self.gen_class_derived_friend_string(base, rec)
replacement_str += base.last_tok.spelling
changes.append([base.last_tok, replacement_str])
return changes
def gen_enum_changes(self, records):
"""
inject friend entry into every class/struct that have projected enum
"""
changes = []
for rec in records:
if not rec.is_external and rec.is_in_class:
replacement_str = self.gen_enum_friend_statement(rec)
replacement_str += rec.class_last_tok.spelling
changes.append([rec.class_last_tok, replacement_str])
return changes
def gen_class_derived_friend_string(self, base, rec):
"""
base classes need friend statement to access private vars, so
remove common prefix from base.name and rec.name
"""
derived_name = self.remove_common_namespace(base.name, rec.name)
return f"\n friend class {derived_name};\n"
def remove_common_namespace(self, source: str, target: str) -> str:
"""
if both derived and base class have exactly the same prefix, then leave it out
"""
source_list = source.split("::")
target_list = target.split("::")
tail = target_list.pop()
source_list.pop()
if target_list == source_list:
return tail
else:
return target
def gen_enum_friend_statement(self, rec):
"""
generate a way to stringify any function
"""
# skip if anon
if rec.is_anonymous:
return ""
out = self.gen_enum_header_comment(rec)
out += " friend "
out += self.gen_enum_switch_statement(rec)
return out
def gen_to_string(self, rec):
"""
generate a way to stringify any function
"""
vars = self.get_all_class_vars(rec)
log.debug(f"{rec.name} : {vars}")
# if len(vars) == 0:
# return f"{rec.name}"
decl_str, decl_expand, tvars = self.expand_template_decl(rec)
# produce the list of vars
# var.out : stores either 'foo =' or 'int foo = '
last_vartype = None
for var in vars:
# prefix = " " * var.indent # # noqa: F841
# vartype='T *' should still end up with description
vartype = var.vartype
if vartype != last_vartype:
var.out = f"{var.vartype} "
var.out += f"{var.name}={{}}"
last_vartype = var.vartype
vars_outlist = [var.out for var in vars]
# assemble the format string
fmt_string = f"{decl_expand}: "
fmt_string += ", ".join(vars_outlist)
# import bpdb; bpdb.set_trace()
# assemble the param string
# deal with pointers using fmt::ptr
# deal with special cases of derived variables in class templates using this->
# TODO: only use this-> for class templates
paramlist = tvars
for var in vars:
if var.indent > 0:
name = f"this->{var.name}"
else:
name = var.name
if var.is_pointer:
paramlist.append(f"fmt::ptr({name})")
else:
paramlist.append(name)
param_string = ""
if paramlist:
param_string = ", " + ", ".join(paramlist)
out = f""" // Generated to_string() for {rec.access_specifier} {rec.class_kind} {decl_str}
public:
auto to_string() const {{
const std::string fmt_string = "{fmt_string}";
return fstr::format(fmt_string{param_string});
}}
"""
return out
def gen_enum_format(self, records):
"""
given list of enum generate fmt: statements or format_as statements
"""
changes = ""
for rec in records:
log.debug(f" enum = {rec}")
changes += self.gen_one_enum(rec)
# for enum in namespaces add alias command to refer to top level
# version of format_as
changes += self.gen_enum_namespace_alias(records)
return changes
def gen_enum_namespace_alias(self, records):
nslist = set()
for rec in records:
if rec.is_in_class or rec.is_in_function or rec.namespace is None:
continue
nslist.add(rec.namespace)
out = "\n"
for ns in nslist:
out += f"namespace {ns} {{using ::format_as;}}\n"
return out
def gen_one_enum(self, rec):
""" """
# skip enum with no entries
if len(rec.values) == 0:
return ""
# skip if anon
if rec.is_anonymous:
return ""
# skip if already a friend statement
if rec.is_in_class or rec.is_in_function:
return ""
# comment out private enums..
out = ""
comment_out = (rec.access_specifier != "PUBLIC" and rec.is_external) or (rec.access_specifier == "PROTECTED")
if comment_out:
out += f"\n/******************* {rec.access_specifier} **\n"
out += self.gen_enum_header_comment(rec)
out += self.gen_enum_switch_statement(rec)
if comment_out:
out += f"\n******************** {rec.access_specifier} */\n"
return out
def gen_enum_header_comment(self, rec):
"""
comment for enum
"""
scoped_str = "scoped" if rec.is_scoped else ""
return (
f"// Generated formatter for {rec.access_specifier} enum {rec.name} of type {rec.enum_type} {scoped_str}\n"
)
def gen_enum_switch_statement(self, rec):
"""
create a format_as statement based on enum definition
.. code-block:: CPP
enum class color {red, green, blue};
auto format_as(const color c) {
fmt::string_view name = "unknown";
switch (c) {
case color::red: name = "red"; break;
case color::green: name = "green"; break;
case color::blue: name = "blue"; break;
}
return name;
};
"""
out = ""
decl = rec.name
out += f"""constexpr auto format_as(const {decl} obj) {{
fmt::string_view name = "<missing>";
switch (obj) {{
"""
out2 = ""
out2 += f"""
// Generated formatter for {rec.access_specifier} enum {decl} of type {rec.enum_type} scoped {rec.is_scoped}
template <>
struct fmt::formatter<{decl}>: formatter<string_view> {{
template <typename FormatContext>
auto format({decl} val, FormatContext& ctx) const {{
string_view name = "<unknown>";
switch (val) {{
"""
# if scoped and name of enum decl is A::B::my_enum then inherit the whole name
# if not scoped, leave out the my_enum part
if rec.is_scoped:
prefix = f"{decl}::"
else:
separator = "::"
prefix = separator.join(decl.rsplit(separator, 1)[:-1]) + separator
if prefix == separator:
prefix = ""
# if all index values are zero, its probably a problem
seen_index = set()
for elem in rec.values:
seen_index.add(elem.index)
is_valid_index = len(seen_index) > 1
seen_index = set()
for elem in rec.values:
is_duplicate = elem.index in seen_index
seen_index.add(elem.index)
width = max([len(x.name) for x in rec.values])
name_in_quotes = f'"{elem.name}"'
line = (
f"case {prefix}{elem.name:<{width}}: name = {name_in_quotes:<{width+2}}; break; // index={elem.index}"
)
if is_duplicate and is_valid_index:
out += f"// {line} <-- index is duplicate\n"
else:
out += f" {line}\n"
out += """ }
return name;
}
"""
out2 += """ }
return formatter<string_view>::format(name, ctx);
}
};"""
return out
def gen_class_format(self, records):
"""
look at simple class/struct and produce to_string statement
"""
changes = ""
for rec in records:
changes += self.gen_one_class(rec)
return changes
def expand_template_decl(self, rec):
"""
look thru decl for template arguments, eg:
LimitedInt<T, Min, Max>
for these template arguments, replace decl with:
LimitedInt<T{int}, Min{0}, Max{100}>
also varlist becomes:
typeid(T).name(), Min, Max
"""
decl = rec.name
# ok stop here if we're not a template
if rec.class_kind != "CLASS_TEMPLATE":
return decl, decl, []
# LimitedInt<T, Min, Max> -> "LimitedInt", ["T", "Min", "Max"]
# 3 types of template params: type, non_type and template
# non_type can have no name..
toutlist = []
tvarlist = []
prefix, bracket_vars = self.extract_template_list_items(rec.name)
out = f"{prefix}<"
# sometimes names in brackets don't exist as template names, eg
# template<class T, class V, bool = (sizeof(V) > sizeof(T))>
# here bool will end up as a non template var of name = ''
for bvar in bracket_vars:
tvar = next((tvar for tvar in rec.tvars if tvar.name == bvar), None)
if tvar is None:
continue
if tvar.template_type == "type":
toutlist.append(f"{bvar}:={{}}")
tvarlist.append(f"fstr::get_type_name<{bvar}>()")
elif tvar.template_type == "non_type":
if tvar.is_param_pack:
toutlist.append(f"{bvar}...")
else:
toutlist.append(f"{bvar}:={{}}")
tvarlist.append(bvar)
out += ", ".join(toutlist)
out += ">"
log.debug(f" template_decl_str = {out}")
return decl, out, tvarlist
def extract_template_list_items(self, text):
pattern = r"<(.*?)>"
matches = re.findall(pattern, text)
items = [item.strip() for item in matches[0].split(",")] if matches else []
match = re.search(r"(.*)<", text)
prefix = match.group(1) if match else ""
return prefix, items
def get_template_decl(self, rec):
"""
look thru definitions and produce list that goes inside template brackets
for template typename arguments, emit extra statement showing type of template param
.. code-block:: CPP
template <typename T, T Min, T Max>
"""
if rec.class_kind != "CLASS_TEMPLATE":
return "", []
tvarlist = []
ttypelist = []
for tvar in rec.tvars:
if tvar.name:
if tvar.is_template_type:
tvarlist.append(f"typename {tvar.name}")
ttypelist.append(tvar.name)
else:
tvarlist.append(f"{tvar.vartype} {tvar.name}")
template_decl_str = ", ".join(tvarlist)
log.debug(f" template_decl_str = {template_decl_str}")
return template_decl_str, ttypelist
def get_typeid_calls(self, rec):
""" """
if rec.class_kind != "CLASS_TEMPLATE":
return ""
tvarlist = []
ttypelist = []
for tvar in rec.tvars:
if tvar.is_template_type:
tvarlist.append(f"typename {tvar.name}")
ttypelist.append(tvar.name)
else:
tvarlist.append(f"{tvar.vartype} {tvar.name}")
template_decl_str = ", ".join(tvarlist)
print(f" template_decl_str={template_decl_str}, ttypelist={ttypelist}")
return template_decl_str, ttypelist
def get_all_class_vars(self, rec):
"""
deal with inherited classes having multiple echos of same var
"""
seen = set()
vars = []
# everthing is blocked for private classes
if rec.access_specifier != "PUBLIC" and rec.is_external:
return vars
# assume everything internal is going to be friended
for var in rec.vars:
if var.displayname not in seen:
if var.access_specifier == "PUBLIC" or not rec.is_external:
vars.append(var)
seen.add(var.displayname)
return vars
def gen_one_class(self, rec):
"""
follow example in fmt:: documentation
"""
vars = self.get_all_class_vars(rec)
log.debug(f"{rec.name} : {vars}")
# TODO: dump out empty class/struct ?
if len(vars) == 0:
return ""
template_decl_str, tvars = self.get_template_decl(rec)
decl = rec.name
out = f"""// Generated formatter for {rec.access_specifier} {rec.class_kind} {decl}
template <{template_decl_str}>
struct fmt::formatter<{decl}> {{
constexpr auto parse(format_parse_context& ctx) {{
return ctx.begin();
}}
template <typename FormatContext>
auto format(const {decl}& obj, FormatContext& ctx) {{
return format_to(ctx.out(),
R"({rec.class_kind} {decl}:
"""
for tvar in tvars:
out += f" type({tvar}): {{}} \n"
for var in vars:
prefix = " " * var.indent
out += f" {prefix} {var.access_specifier} {var.vartype} {var.name}: {{}} \n"
out += ')"'
tvarlist = [f"typeid({tvar}).name() " for tvar in tvars]
if tvarlist:
out += ", " + ", ".join(tvarlist)
varlist = [f"obj.{var.name}" for var in vars]
if varlist:
out += ", " + ", ".join(varlist)
out += """);
}
};
"""
return out