Repository navigation
cppcheckdata.py: compile match() patterns once
#8999
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
e94d0cd
baf9f30
b18f149
ec8567f
c59f7f1
955b562
943920f
0b14ed6
b14b932
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1624,37 +1624,6 @@ def simpleMatch(token, pattern): | |
| '<*>': lambda tok: tok.link if tok.str == '<' and tok.link else None, | ||
| } | ||
|
|
||
| def match_atom(token, p): | ||
| if not token: | ||
| return None | ||
| if not p: | ||
| return None | ||
| if token.str == p: | ||
| return token | ||
| if p in ['!', '|', '||', '%', '!=', '*']: | ||
| return None | ||
| if p in patterns: | ||
| return patterns[p](token) | ||
| if '|' in p: | ||
| for x in p.split('|'): | ||
| t = match_atom(token, x) | ||
| if t: | ||
| return t | ||
| elif p.startswith('!!'): | ||
| t = match_atom(token, p[2:]) | ||
| if not t: | ||
| return token | ||
| elif p.startswith('**'): | ||
| a = p[2:] | ||
| t = token | ||
| while t: | ||
| if match_atom(t, a): | ||
| return t | ||
| if t.link and t.str in ['(', '[', '<', '{']: | ||
| t = t.link | ||
| t = t.next | ||
| return None | ||
|
|
||
| class MatchResult: | ||
| def __init__(self, matches, bindings=None, keys=None): | ||
| self.__dict__.update(bindings or {}) | ||
|
|
@@ -1678,22 +1647,137 @@ def bind_split(s): | |
| return (p[0], p[2]) | ||
| return (s, None) | ||
|
|
||
| class CompiledPattern: | ||
| """ | ||
| A match() pattern compiled once so it can be matched against many tokens. | ||
|
|
||
| Each word of the pattern is compiled into a matcher: a function that takes | ||
| a token (never None) and returns the last token the word matched, or None. | ||
|
|
||
| Attributes: | ||
| pattern The pattern string | ||
| atoms List of (matcher, binding name) for each word in the pattern | ||
| failed The MatchResult returned when the pattern does not match | ||
| """ | ||
|
|
||
| _literal_atoms = frozenset(['!', '|', '||', '|=', '%', '!=', '*']) | ||
| _link_open = frozenset(['(', '[', '<', '{']) | ||
|
|
||
| def __init__(self, pattern): | ||
| self.pattern = pattern | ||
| words = [bind_split(word) for word in pattern.split()] | ||
| self.atoms = [(self.compile_atom(p), b) for p, b in words] | ||
| keys = [b for p, b in words] + ['end'] if words else [] | ||
| self.failed = MatchResult(False, keys=keys) | ||
|
|
||
| def __repr__(self): | ||
| return "CompiledPattern(pattern={})".format(repr(self.pattern)) | ||
|
|
||
| def match(self, token): | ||
| if not self.atoms: | ||
| # empty pattern (or only whitespace) never matches | ||
| return self.failed | ||
| end = None | ||
| bindings = {} | ||
| for m, b in self.atoms: | ||
| if token is None: | ||
| return self.failed | ||
| t = m(token) | ||
| if b: | ||
| bindings[b] = token | ||
| if t is None: | ||
| return self.failed | ||
| end = t | ||
| token = t.next | ||
| bindings['end'] = end | ||
| return MatchResult(True, bindings=bindings) | ||
|
|
||
| @classmethod | ||
| def compile_atom(cls, p): | ||
| """Compile one pattern word into a matcher.""" | ||
| if not p: | ||
| return cls._match_none | ||
| if p in cls._literal_atoms: | ||
| return cls._match_literal(p) | ||
| if p in patterns: | ||
| return patterns[p] | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. AI review This copies the entry of the module-level
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. User arent meant to modify |
||
| prefix, word = p[:2], p[2:] | ||
| # a '!!' or '**' prefix applies to the whole word only when the word | ||
| # is a literal: '!!|=' is "not |=", but '!!a|b' is "not a, or b" | ||
| if '|' in p and not (prefix in ('!!', '**') and word in cls._literal_atoms): | ||
| return cls._match_alternatives([cls.compile_atom(x) for x in p.split('|')]) | ||
| if prefix == '!!': | ||
| return cls._match_negation(cls.compile_atom(word)) | ||
| if prefix == '**': | ||
| return cls._match_forward(cls.compile_atom(word)) | ||
| return cls._match_literal(p) | ||
|
|
||
| @staticmethod | ||
| def _match_none(tok): | ||
| return None | ||
|
|
||
| @staticmethod | ||
| def _match_literal(p): | ||
| def m(tok): | ||
| return tok if tok.str == p else None | ||
| return m | ||
|
|
||
| @staticmethod | ||
| def _match_alternatives(alternatives): | ||
| def m(tok): | ||
| for a in alternatives: | ||
| t = a(tok) | ||
| if t: | ||
| return t | ||
| return None | ||
| return m | ||
|
|
||
| @staticmethod | ||
| def _match_negation(inner): | ||
| def m(tok): | ||
| return None if inner(tok) else tok | ||
| return m | ||
|
|
||
| @classmethod | ||
| def _match_forward(cls, inner): | ||
| link_open = cls._link_open | ||
| def m(tok): | ||
| t = tok | ||
| while t: | ||
| if inner(t): | ||
| return t | ||
| if t.link and t.str in link_open: | ||
| t = t.link | ||
| t = t.next | ||
| return None | ||
| return m | ||
|
|
||
| @functools.lru_cache(maxsize=1024) | ||
| def compile_pattern(pattern): | ||
| """ | ||
| Return the CompiledPattern for a pattern string. | ||
|
|
||
| The most recently used patterns are kept, so compiling the same pattern | ||
| string again is cheap. | ||
| """ | ||
| return CompiledPattern(pattern) | ||
|
|
||
| def match(token, pattern): | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. AI review The old if not pattern:
return MatchResult(False)That guard also keeps falsy patterns out of |
||
| """ | ||
| Match the tokens starting at token against pattern. | ||
|
|
||
| Returns a MatchResult, which is true when the pattern matches. | ||
|
|
||
| Use constant pattern strings. A pattern is compiled the first time it is | ||
| used and only a limited number of compiled patterns are kept, so a pattern | ||
| string built at run time, such as name + ' (', is compiled again for every | ||
| new string. Compare the varying part separately instead: | ||
|
|
||
| match(tok, '%name% (') and tok.str == name | ||
| """ | ||
| if not pattern: | ||
| return MatchResult(False) | ||
| end = None | ||
| bindings = {} | ||
| words = [bind_split(word) for word in pattern.split()] | ||
| for p, b in words: | ||
| t = match_atom(token, p) | ||
| if b: | ||
| bindings[b] = token | ||
| if not t: | ||
| return MatchResult(False, keys=[xx for pp, xx in words]+['end']) | ||
| end = t | ||
| token = t.next | ||
| bindings['end'] = end | ||
| return MatchResult(True, bindings=bindings) | ||
| return compile_pattern(pattern).match(token) | ||
|
|
||
| def get_function_call_name_args(token): | ||
| """Get function name and arguments for function call | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -139,6 +139,7 @@ class Shape { | |
| int arr[3]; | ||
| arr[0] = a; | ||
| a += 1; | ||
| a |= b; | ||
| if (a > b) { | ||
| return a; | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -523,6 +523,7 @@ def test_literal_sequence(self, match_cfg): | |
| def test_empty_pattern_and_no_token(self, match_cfg): | ||
| calc_def = find_token(match_cfg, 'calc') | ||
| assert not cppcheckdata.match(calc_def, '') | ||
| assert not cppcheckdata.match(calc_def, None) | ||
| assert not cppcheckdata.match(None, 'calc') | ||
|
|
||
| def test_end(self, match_cfg): | ||
|
|
@@ -572,6 +573,23 @@ def test_literal_percent(self, match_cfg): | |
| assert cppcheckdata.match(mod, '%') | ||
| assert cppcheckdata.match(mod.previous, '%var% % %var% ;') | ||
|
|
||
| def test_literal_or_assign(self, match_cfg): | ||
| or_assign = find_token(match_cfg, '|=') | ||
| assert cppcheckdata.match(or_assign, '|=') | ||
| assert cppcheckdata.match(or_assign.previous, '%var% |= %var% ;') | ||
| assert cppcheckdata.match(or_assign, '%assign%') | ||
| assert not cppcheckdata.match(find_token(match_cfg, '|'), '|=') | ||
| # '|=' is a literal, not an alternation of '' and '=' | ||
| # (unlike the C++ Token::Match, which also matches a '=' token here) | ||
| assert not cppcheckdata.match(find_token(match_cfg, '='), '|=') | ||
|
|
||
| def test_whitespace_only_pattern(self, match_cfg): | ||
| calc_def = find_token(match_cfg, 'calc') | ||
| res = cppcheckdata.match(calc_def, ' ') | ||
| assert not res | ||
| with pytest.raises(AttributeError): | ||
| res.end # pylint: disable=W0104 | ||
|
|
||
| def test_literal_parentheses(self, match_cfg): | ||
| calc_call = find_token(match_cfg, 'calc', skip=1) | ||
| assert cppcheckdata.match(calc_call, 'calc ( %var% , %var% ) ;') | ||
|
|
@@ -696,6 +714,37 @@ def test_negation_keyword(self, match_cfg): | |
| assert cppcheckdata.match(calc_call, '%name% !!%op%') | ||
| assert not cppcheckdata.match(calc_call, '%name% !!(') | ||
|
|
||
| def test_negation_literal_or_assign(self, match_cfg): | ||
| # '!!|=' is "not |=", not an alternation of '!!' and '=' | ||
| or_assign = find_token(match_cfg, '|=') | ||
| assert not cppcheckdata.match(or_assign.previous, '%var% !!|=') | ||
| bit_or = find_token(match_cfg, '|') | ||
| assert cppcheckdata.match(bit_or.previous, '%var% !!|=') | ||
| assert cppcheckdata.match(find_token(match_cfg, '=').previous, '%var% !!|=') | ||
|
|
||
| def test_negation_alternatives(self, match_cfg): | ||
| # '!!a|b' is "not a, or b", as in the C++ Token::Match | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. AI review The C++ |
||
| struct_tok = find_token(match_cfg, 'struct') | ||
| assert not cppcheckdata.match(struct_tok, 'struct !!Point|Foo') | ||
| assert cppcheckdata.match(struct_tok, 'struct !!Foo|Point') | ||
| assert cppcheckdata.match(struct_tok, 'struct !!Foo|Bar') | ||
|
|
||
| def test_find_forward_alternatives(self, match_cfg): | ||
| # '**a|b' is "a somewhere ahead, or b here" | ||
| calc_def = find_token(match_cfg, 'calc') | ||
| res = cppcheckdata.match(calc_def, 'calc **{|nosuchtoken') | ||
| assert res | ||
| assert res.end.str == '{' | ||
| res = cppcheckdata.match(calc_def, 'calc **nosuchtoken|(') | ||
| assert res | ||
| assert res.end.str == '(' | ||
| assert not cppcheckdata.match(calc_def, 'calc **nosuchtoken|{') | ||
| # but a literal word is searched for as a whole | ||
| arr_decl = find_token(match_cfg, 'arr') | ||
| res = cppcheckdata.match(arr_decl, 'arr **|=') | ||
| assert res | ||
| assert res.end.str == '|=' | ||
|
|
||
| @pytest.mark.xfail(strict=False, | ||
| reason='in the C++ Token::Match a negation also matches when there is no token, ' | ||
| 'in match() it does not') | ||
|
|
@@ -761,6 +810,23 @@ def test_find_forward(self, match_cfg): | |
| assert res.end.str == '{' | ||
| assert not cppcheckdata.match(calc_def, 'calc **nosuchtoken') | ||
|
|
||
| # ---- compiled pattern cache ---- | ||
|
|
||
| def test_compile_pattern_reused(self): | ||
| compiled = cppcheckdata.compile_pattern('calc ( %var%') | ||
| assert compiled.pattern == 'calc ( %var%' | ||
| assert cppcheckdata.compile_pattern('calc ( %var%') is compiled | ||
|
|
||
| def test_compiled_patterns_limited(self, match_cfg): | ||
| calc_def = find_token(match_cfg, 'calc') | ||
| maxsize = cppcheckdata.compile_pattern.cache_info().maxsize | ||
| assert maxsize is not None | ||
| for i in range(maxsize + 100): | ||
| assert not cppcheckdata.match(calc_def, 'nosuchtoken{} ('.format(i)) | ||
| assert cppcheckdata.compile_pattern.cache_info().currsize == maxsize | ||
| # a pattern that was dropped is compiled again and still matches | ||
| assert cppcheckdata.match(calc_def, 'calc ( int') | ||
|
|
||
|
|
||
| class TestSuppressions: | ||
| def make_suppression(self, **kwargs): | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
AI review
match()returns this oneMatchResultfrom every failed call with this pattern, andMatchResultis mutable. If an addon assigns to a failed result (r = match(tok, 'a@x'); r.x = ...), every later failed match with the same pattern returns that value: I reproduced.x == 'stale'on a later, unrelated failed match. Before this PR each failed call got a new object. A simple fix is to store onlykeysand buildMatchResult(False, keys=self._failed_keys)on failure, which is still much cheaper than before. Another option is to makeMatchResultreject attribute assignment.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I dont think this is an issue. The
MatchResultshouldn't be modified.