Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion Lib/re/_optimizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -505,5 +505,10 @@ def _walk(seq):

def optimize(pattern):
"""Rewrite a parsed pattern in place and return it."""
_walk(pattern.data)
# The parser records whether the tree holds a construct that the
# peepholes below can rewrite. For an ordinary pattern the walk would
# only add a full pass over the tree, so skip it. A tree that was not
# produced by the parser has an unknown marker and must be optimized.
if getattr(pattern.state, 'fusable', None) is not False:
_walk(pattern.data)
return pattern
35 changes: 29 additions & 6 deletions Lib/re/_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,11 @@ def __init__(self):
self.groupwidths = [None] # group 0
self.lookbehindgroups = None
self.grouprefpos = {}
# Whether the parsed tree contains a construct that the optimizer can
# rewrite. None means the tree was not produced by parse() (e.g. it
# was assembled by hand), so the optimizer must not skip it. parse()
# sets False and sets True as such constructs are parsed.
self.fusable = None
@property
def groups(self):
return len(self.groupwidths)
Expand Down Expand Up @@ -506,21 +511,29 @@ def _parse_sub(source, state, verbose, nested):

# check if the branch can be replaced by a character set
set = []
complement = False
for item in items:
if len(item) != 1:
break
op, av = item[0]
if op in _SETITEMCODES:
set.append((op, av))
elif op is IN and av[0][0] is not NEGATE:
set.extend(av)
elif op is IN:
if av[0][0] is NEGATE:
# Not foldable here, but the optimizer can still fuse this
# branch if every alternative is a one-character matcher.
complement = True
else:
set.extend(av)
else:
break
else:
# we can store this as a character set instead of a
# branch (the compiler may optimize this even more)
subpattern.append((IN, _uniq(set)))
return subpattern
if not complement:
# we can store this as a character set instead of a
# branch (the compiler may optimize this even more)
subpattern.append((IN, _uniq(set)))
return subpattern
state.fusable = True

subpattern.append((BRANCH, (None, items)))
return subpattern
Expand Down Expand Up @@ -555,6 +568,9 @@ def _union(left, right, state):
right_items = _flat_items(right)
if left_items is not None and right_items is not None:
return [_charset_node(_uniq(left_items + right_items))]
# The optimizer may still be able to fold the two alternatives into one
# charset, e.g. when one of them is a complemented class.
state.fusable = True
return [(BRANCH, (None, [SubPattern(state, left),
SubPattern(state, right)]))]

Expand All @@ -564,6 +580,8 @@ def _intersect(left, right, state):

def _difference(left, right, state):
# A -- B: A, then require the character not to match B (lookbehind).
# The optimizer can fuse the lookbehind back into a single charset.
state.fusable = True
return left + [(ASSERT_NOT, (-1, SubPattern(state, right)))]

# Map a set-operator token to the function combining the accumulated result
Expand Down Expand Up @@ -937,6 +955,10 @@ def _parse(source, state, verbose, nested, first=False):
if char == "=":
subpatternappend((ASSERT, (dir, p)))
elif p:
if dir < 0:
# The optimizer may fuse this negative lookbehind
# with the preceding character matcher.
state.fusable = True
subpatternappend((ASSERT_NOT, (dir, p)))
else:
subpatternappend((FAILURE, ()))
Expand Down Expand Up @@ -1124,6 +1146,7 @@ def parse(str, flags=0, state=None):
state = State()
state.flags = flags
state.str = str
state.fusable = False

p = _parse_sub(source, state, flags & SRE_FLAG_VERBOSE, 0)
p.state.flags = fix_flags(str, p.state.flags)
Expand Down
16 changes: 16 additions & 0 deletions Lib/test/test_re.py
Original file line number Diff line number Diff line change
Expand Up @@ -2198,6 +2198,22 @@ def test_locale_ignorecase_negated_set(self):
self.assertIsNone(re.fullmatch(rb'\w(?<!b)', b'B', IL))
self.assertTrue(re.fullmatch(rb'\w(?<!b)', b'A', IL))

def test_optimizer_walk_only_for_fusable_patterns(self):
# The parser marks the patterns in which _optimizer's peepholes can
# rewrite something; the compiler skips the extra tree walk otherwise.
parse = re._parser.parse
self.assertIsNone(re._parser.State().fusable)
for pattern in ('abc', 'a|bc', 'a|b', '[^a]', '[^a]b|c',
'(?<=a)b', '(?!a)b', '[^a]||b', '[a-f&&[^bc]]',
r'[\w||[a-z]]', '(a|[^b]c)'):
with self.subTest(pattern=pattern):
self.assertIs(parse(pattern).state.fusable, False)
for pattern in ('a|[^bc]', '(a|[^bc])c*', r'\w(?<!b)', '(?<!a)b',
'[a-c--b]', r'[\w&&[a-c--b]]', '[a||[^bc]]',
'(a(?<!b))c'):
with self.subTest(pattern=pattern):
self.assertIs(parse(pattern).state.fusable, True)

def test_scoped_flags(self):
self.assertTrue(re.match(r'(?i:a)b', 'Ab'))
self.assertIsNone(re.match(r'(?i:a)b', 'aB'))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix a :mod:`re` compilation performance regression (:gh:`161125`) by
skipping the set-operation fusion pass for patterns that contain no
fusable constructs.
Loading