Skip to content

Commit 1f961fc

Browse files
williamwen42Lucaskabela
authored andcommitted
[dynamo] fix error_on_graph_break bug where non-empty checkpoint results in unwanted graph break resumption (#166586)
Fixes #166589 Pull Request resolved: #166586 Approved by: https://github.com/Lucaskabela ghstack dependencies: #166476, #166477 (cherry picked from commit 267d019)
1 parent cbe1a35 commit 1f961fc

File tree

2 files changed

+30
-0
lines changed

2 files changed

+30
-0
lines changed

test/dynamo/test_decorators.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2016,6 +2016,23 @@ def f():
20162016

20172017
self.assertEqual(f(), 1)
20182018

2019+
def test_error_on_graph_break_nonempty_checkpoint(self):
2020+
cnts = torch._dynamo.testing.CompileCounter()
2021+
2022+
@torch.compile(backend=cnts)
2023+
def fn(x):
2024+
x = x + 1
2025+
x = x + 1
2026+
x = x + 1
2027+
with torch._dynamo.error_on_graph_break(True):
2028+
torch._dynamo.graph_break()
2029+
return x + 1
2030+
2031+
with self.assertRaises(Unsupported):
2032+
fn(torch.ones(3))
2033+
2034+
self.assertEqual(cnts.frame_count, 0)
2035+
20192036
def test_nested_compile_fullgraph(self):
20202037
# Test that fullgraph=True cannot be toggled back by fullgraph=False
20212038
inp = torch.ones(3)

torch/_dynamo/symbolic_convert.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1355,6 +1355,19 @@ def step(self) -> bool:
13551355
except (ReturnValueOp, YieldValueOp):
13561356
return False
13571357
except Unsupported:
1358+
# More restrictive condition than should_compile_partial_graph:
1359+
# if this condition is true, then we SHOULD NOT attempt to find
1360+
# a previous checkpoint to resume from and try to resume - we should
1361+
# immediately error out.
1362+
# The condition is more restrictive because, it may be possible to resume significantly earlier
1363+
# in the code (the most recent speculation point). This happens, for example, in the case
1364+
# of a graph break in a try block.
1365+
if (
1366+
self.one_graph
1367+
or self.error_on_graph_break
1368+
or self.is_tracing_resume_prologue
1369+
):
1370+
raise
13581371
if self.current_speculation is None:
13591372
log.debug("empty checkpoint")
13601373
raise

0 commit comments

Comments
 (0)