Skip to content

gh-153568: Use FIRST sets to speed up parser choices #156209

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pablogsal wants to merge 2 commits into
base: main
Choose a base branch
from
Open

gh-153568: Use FIRST sets to speed up parser choices #156209

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
456eb6f
gh-153568: Use FIRST sets to speed up parser choices
pablogsal Aug 21, 2026
6409262
gh-153568: Make FIRST-set dispatch conservative
pablogsal Aug 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 145 additions & 0 deletions Lib/test/test_peg_generator/test_c_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,100 @@ def test_prefix_respects_cut(self) -> None:
)
""")

def test_first_set_compound_negative_lookahead(self) -> None:
self.run_test("""
start: choice NEWLINE ENDMARKER
choice: !('a' 'b') ('a' | 'c') | 'd'
""", """
self.check_input_strings_for_grammar(
valid_cases=['a', 'c', 'd'], invalid_cases=['a b'],
)
""")

def test_first_set_mutual_recursion(self) -> None:
self.run_test("""
start: a NEWLINE ENDMARKER
a: b 'x' | 'a'
b: a 'y' | 'b'
""", """
self.check_input_strings_for_grammar(
valid_cases=['a', 'b x', 'a y x', 'b x y x'],
invalid_cases=['a y', 'b'],
)
""")

def test_first_set_preserves_leading_cut(self) -> None:
self.run_test("""
start: choice NEWLINE ENDMARKER
choice: ~ NAME | NUMBER
""", """
self.check_input_strings_for_grammar(
valid_cases=['name'], invalid_cases=['42'],
)
""")

def test_first_set_preserves_forced_parse(self) -> None:
self.run_test("""
start: choice NEWLINE ENDMARKER
choice: &&'+' | NUMBER
""", """
self.check_input_strings_for_grammar(
valid_cases=['+'], invalid_cases=['42'],
)
""")

def test_first_set_nullable_prefix_action(self) -> None:
self.run_test("""
start: choice NEWLINE ENDMARKER
choice: guard NAME | NUMBER
guard: &NUMBER { RAISE_SYNTAX_ERROR("guard reached") }
""", """
with self.assertRaisesRegex(SyntaxError, 'guard reached'):
parse.parse_string('42', mode=0)
""")

def check_first_set_lookahead_forced_parse(self, predicate: str) -> None:
self.run_test(f"""
start: choice NEWLINE ENDMARKER
choice: {predicate}(NAME &&'+') STRING | NAME
""", """
with self.assertRaises(SyntaxError):
parse.parse_string('name', mode=0)
""")

def test_first_set_positive_lookahead_forced_parse(self) -> None:
self.check_first_set_lookahead_forced_parse('&')

def test_first_set_negative_lookahead_forced_parse(self) -> None:
self.check_first_set_lookahead_forced_parse('!')

def check_first_set_lookahead_action(self, predicate: str) -> None:
self.run_test(f"""
start: choice NEWLINE ENDMARKER
choice: {predicate}guard NAME | NUMBER
guard: NUMBER {{ RAISE_SYNTAX_ERROR("guard reached") }}
""", """
with self.assertRaisesRegex(SyntaxError, 'guard reached'):
parse.parse_string('42', mode=0)
""")

def test_first_set_positive_lookahead_action(self) -> None:
self.check_first_set_lookahead_action('&')

def test_first_set_negative_lookahead_action(self) -> None:
self.check_first_set_lookahead_action('!')

def test_first_set_named_soft_keyword(self) -> None:
self.run_test('''
start: choice NEWLINE ENDMARKER
choice: SOFT_KEYWORD ':' | NUMBER
spelling: "soft"
''', """
self.check_input_strings_for_grammar(
valid_cases=['soft :', '42'], invalid_cases=['other :'],
)
""")

def test_c_parser(self) -> None:
grammar_source = """
start[mod_ty]: a[asdl_stmt_seq*]=stmt* $ { _PyAST_Module(a, NULL, p->arena) }
Expand Down Expand Up @@ -533,6 +627,57 @@ def test_soft_keywords_lookahead(self) -> None:
"""
self.run_test(grammar_source, test_source)

def test_first_set_dispatch(self) -> None:
grammar = parse_string(
"""
start: expr NEWLINE
expr: NAME | NUMBER | '(' NAME ')'
""",
GrammarParser,
)
parser_source = generate_c_parser_source(grammar)
self.assertIn("switch (_current_token_type)", parser_source)

overlapping_grammar = parse_string(
"start: expr NEWLINE $\nexpr: NAME '+' NAME | NAME\n",
GrammarParser,
)
parser_source = generate_c_parser_source(overlapping_grammar)
self.assertIn("_first_set_mask", parser_source)
self.run_test(
"start: expr NEWLINE $\nexpr: NAME '+' NAME | NAME\n",
'self.check_input_strings_for_grammar(["a\\n", "a + b\\n"])',
)

nullable_grammar = parse_string(
"start: expr NEWLINE\nexpr: ['+'] | NUMBER\n",
GrammarParser,
)
parser_source = generate_c_parser_source(nullable_grammar)
self.assertNotIn("switch (_current_token_type)", parser_source)

def test_first_set_dispatch_soft_keywords(self) -> None:
grammar_source = 'start: ("foo" | "bar") NEWLINE $\n'
grammar = parse_string(grammar_source, GrammarParser)
parser_source = generate_c_parser_source(grammar)
self.assertIn("_first_set_mask", parser_source)
self.run_test(
grammar_source,
'self.check_input_strings_for_grammar(["foo\\n", "bar\\n"], ["baz\\n"])',
)

def test_first_set_dispatch_invalid_rules(self) -> None:
grammar_source = """
start: value NEWLINE $
value: NAME '+' NAME | invalid_value
invalid_value: NAME { RAISE_SYNTAX_ERROR("expected an addition") }
"""
test_source = """
with self.assertRaisesRegex(SyntaxError, "expected an addition"):
parse.parse_string("name\\n", mode=0)
"""
self.run_test(grammar_source, test_source)

def test_forced(self) -> None:
grammar_source = """
start: NAME &&':' | NAME
Expand Down
56 changes: 47 additions & 9 deletions Lib/test/test_peg_generator/test_first_sets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import unittest

from test import test_tools
from typing import Dict, Set

test_tools.skip_if_missing("peg_generator")
with test_tools.imports_under_tool("peg_generator"):
Expand All @@ -12,7 +11,7 @@


class TestFirstSets(unittest.TestCase):
def calculate_first_sets(self, grammar_source: str) -> Dict[str, Set[str]]:
def calculate_first_sets(self, grammar_source: str) -> dict[str, set[str | None]]:
grammar: Grammar = parse_string(grammar_source, GrammarParser)
return FirstSetCalculator(grammar.rules).calculate()

Expand Down Expand Up @@ -164,8 +163,8 @@ def test_positive_lookahead(self) -> None:
self.assertEqual(
self.calculate_first_sets(grammar),
{
"expr": {"'a'"},
"start": {"'a'"},
"expr": {None, "'a'", "'b'", "'c'"},
"start": {None, "'a'", "'b'", "'c'"},
"opt": {"'b'", "'c'", "'a'"},
},
)
Expand All @@ -180,8 +179,8 @@ def test_negative_lookahead(self) -> None:
self.calculate_first_sets(grammar),
{
"opt": {"'b'", "'a'", "'c'"},
"expr": {"'b'", "'c'"},
"start": {"'b'", "'c'"},
"expr": {None, "'a'", "'b'", "'c'"},
"start": {None, "'a'", "'b'", "'c'"},
},
)

Expand Down Expand Up @@ -226,21 +225,20 @@ def test_mutual_left_recursion(self) -> None:
self.calculate_first_sets(grammar),
{
"foo": {"'D'", "'B'"},
"bar": {"'D'"},
"bar": {"'B'", "'D'"},
"start": {"'D'", "'B'"},
},
)

def test_nasty_left_recursion(self) -> None:
# TODO: Validate this
grammar = """
start: target '='
target: maybe '+' | NAME
maybe: maybe '-' | target
"""
self.assertEqual(
self.calculate_first_sets(grammar),
{"maybe": set(), "target": {"NAME"}, "start": {"NAME"}},
{"maybe": {"NAME"}, "target": {"NAME"}, "start": {"NAME"}},
)

def test_nullable_rule(self) -> None:
Expand Down Expand Up @@ -284,3 +282,43 @@ def test_multiple_nullable_rules(self) -> None:
"another": {"'/'"},
},
)

def test_compound_negative_lookahead(self) -> None:
sets = self.calculate_first_sets("""
start: choice NEWLINE ENDMARKER
choice: !('a' 'b') ('a' | 'c') | 'd'
""")
self.assertEqual(sets['choice'], {None, "'a'", "'c'", "'d'"})

def test_nullable_recursive_rules(self) -> None:
sets = self.calculate_first_sets("""
start: a NUMBER ENDMARKER
a: b | NAME
b: a | ['+']
""")
self.assertEqual(sets['a'], {'', 'NAME', "'+'"})
self.assertEqual(sets['b'], {'', 'NAME', "'+'"})
self.assertEqual(sets['start'], {'NUMBER', 'NAME', "'+'"})

def test_control_flow_before_first_token(self) -> None:
sets = self.calculate_first_sets("""
start: NAME ENDMARKER
cut: ~ NAME
forced: &&'+'
guarded: &forced NUMBER
action: [NAME] { _PyPegen_dummy_name(p) }
""")
self.assertEqual(sets['cut'], {None, 'NAME'})
self.assertEqual(sets['forced'], {None, "'+'"})
self.assertEqual(sets['guarded'], {None, 'NUMBER'})
self.assertEqual(sets['action'], {None, '', 'NAME'})

def test_nullable_repeat_and_gather(self) -> None:
sets = self.calculate_first_sets("""
start: NAME ENDMARKER
optional: [NAME]
repeat: optional+ NUMBER
gather: ','.optional+ NUMBER
""")
self.assertEqual(sets['repeat'], {'NAME', 'NUMBER'})
self.assertEqual(sets['gather'], {'NAME', "','", 'NUMBER'})
Loading
Loading