Skip to content

Commit fe2d7d7

Browse files
andystaplesCopilot
andauthored
Honor non-retriable activity and sub-orchestration failures (#318)
* Honor non-retriable activity and sub-orchestration failures Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Make regression test orchestrator return explicit Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Use explicit UTC time in retry regression test Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent 5c842a3 commit fe2d7d7

5 files changed

Lines changed: 108 additions & 3 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,11 @@ A single callback failure is still raised directly.
3434
3535
FIXED
3636

37+
- Activity and sub-orchestration retry policies now honor
38+
`failureDetails.isNonRetriable`, surfacing the failure immediately without
39+
scheduling a retry timer or another attempt. Existing histories that already
40+
retried such failures can replay differently; drain affected instances on the
41+
previous SDK before upgrading.
3742
- Grouped failures now retain nested exception messages, tracebacks, and notes
3843
in failure details, including `ExceptionGroup` failures from task callbacks.
3944
Traceback rendering is limited to 15 children per group and 10 group levels,

‎azure-functions-durable/CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@ can change during replay; see the [core upgrade warning](../CHANGELOG.md#unrelea
2929

3030
FIXED
3131

32+
- With the corresponding core SDK update, activity and sub-orchestration retry
33+
policies, including compatibility `RetryOptions`, honor
34+
`failureDetails.isNonRetriable` and surface the failure immediately without retrying.
35+
Histories that already retried such failures can replay differently; drain
36+
affected instances on the previous SDK before upgrading.
3237
- With the corresponding core SDK update, grouped failures retain nested
3338
exception messages, tracebacks, and notes in failure details. Tracebacks are
3439
limited to 15 children per group and 10 group levels with truncation markers;

‎durabletask-azuremanaged/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ can change during replay; see the [core upgrade warning](../CHANGELOG.md#unrelea
2020

2121
FIXED
2222

23+
- With the corresponding core SDK update, activity and sub-orchestration retry
24+
policies honor `failureDetails.isNonRetriable` and surface the failure immediately
25+
without retrying. Histories that already retried such failures can replay
26+
differently; drain affected instances on the previous SDK before upgrading.
2327
- With the corresponding core SDK update, grouped failures retain nested
2428
exception messages, tracebacks, and notes in failure details. Tracebacks are
2529
limited to 15 children per group and 10 group levels with truncation markers;

‎durabletask/worker.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2872,7 +2872,10 @@ def _cancel_timer() -> None:
28722872
activity_task_obj = cast(object, activity_task)
28732873
if isinstance(activity_task_obj, task.RetryableTask):
28742874
retryable_activity_task = cast(task.RetryableTask[Any], activity_task_obj)
2875-
next_delay = retryable_activity_task.compute_next_delay()
2875+
next_delay = (
2876+
None if event.taskFailed.failureDetails.isNonRetriable
2877+
else retryable_activity_task.compute_next_delay()
2878+
)
28762879
if next_delay is None:
28772880
retryable_activity_task.fail(
28782881
f"{ctx.instance_id}: Activity task #{task_id} failed: {event.taskFailed.failureDetails.errorMessage}",
@@ -2983,7 +2986,10 @@ def _cancel_timer() -> None:
29832986
sub_orch_task_obj = cast(object, sub_orch_task)
29842987
if isinstance(sub_orch_task_obj, task.RetryableTask):
29852988
retryable_sub_orch_task = cast(task.RetryableTask[Any], sub_orch_task_obj)
2986-
next_delay = retryable_sub_orch_task.compute_next_delay()
2989+
next_delay = (
2990+
None if failedEvent.failureDetails.isNonRetriable
2991+
else retryable_sub_orch_task.compute_next_delay()
2992+
)
29872993
if next_delay is None:
29882994
retryable_sub_orch_task.fail(
29892995
f"Sub-orchestration task #{task_id} failed: {failedEvent.failureDetails.errorMessage}",

‎tests/durabletask/test_orchestration_executor.py‎

Lines changed: 86 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
import json
55
import logging
6-
from datetime import datetime, timedelta
6+
from datetime import UTC, datetime, timedelta
77
from typing import Any
88

99
import pytest
@@ -938,6 +938,91 @@ def get_exception_properties(self, exception: Exception):
938938
assert failure.innerFailure.properties["code"].string_value == "VALUE"
939939

940940

941+
@pytest.mark.parametrize("is_sub_orch", [False, True], ids=["activity", "sub-orchestration"])
942+
@pytest.mark.parametrize("is_non_retriable", [None, False, True], ids=["unset", "false", "true"])
943+
@pytest.mark.parametrize("is_replaying", [False, True], ids=["new-failure", "replayed-failure"])
944+
@pytest.mark.parametrize("catch_failure", [False, True], ids=["unhandled", "caught"])
945+
def test_retry_policy_respects_non_retriable_failure(
946+
is_sub_orch: bool,
947+
is_non_retriable: bool | None,
948+
is_replaying: bool,
949+
catch_failure: bool,
950+
):
951+
"""Non-retriable failures bypass retries and retain their failure details."""
952+
953+
def orchestrator(ctx: task.OrchestrationContext, _):
954+
retry_policy = task.RetryPolicy(
955+
first_retry_interval=timedelta(seconds=1),
956+
max_number_of_attempts=3,
957+
)
958+
try:
959+
if is_sub_orch:
960+
yield ctx.call_sub_orchestrator("child", retry_policy=retry_policy)
961+
else:
962+
yield ctx.call_activity("child", retry_policy=retry_policy)
963+
except task.TaskFailedError as error:
964+
if not catch_failure:
965+
raise
966+
assert error.details.error_type == "builtins.ValueError"
967+
assert error.details.message == "Permanent failure"
968+
return "compensated"
969+
return None
970+
971+
registry = worker._Registry()
972+
name = registry.add_orchestrator(orchestrator)
973+
current_timestamp = datetime.now(UTC).replace(tzinfo=None)
974+
old_events = [
975+
helpers.new_orchestrator_started_event(current_timestamp),
976+
helpers.new_execution_started_event(name, TEST_INSTANCE_ID, encoded_input=None),
977+
]
978+
if is_sub_orch:
979+
old_events.append(helpers.new_sub_orchestration_created_event(1, "child", "child-instance"))
980+
failed_event = helpers.new_sub_orchestration_failed_event(1, ValueError("Permanent failure"))
981+
failure_details = failed_event.subOrchestrationInstanceFailed.failureDetails
982+
else:
983+
old_events.append(helpers.new_task_scheduled_event(1, "child"))
984+
failed_event = helpers.new_task_failed_event(1, ValueError("Permanent failure"))
985+
failure_details = failed_event.taskFailed.failureDetails
986+
if is_non_retriable is not None:
987+
failure_details.isNonRetriable = is_non_retriable
988+
989+
new_events = [helpers.new_orchestrator_started_event(current_timestamp)]
990+
if is_replaying:
991+
old_events.append(failed_event)
992+
else:
993+
new_events.append(failed_event)
994+
executor = worker._OrchestrationExecutor(registry, TEST_LOGGER, JsonDataConverter())
995+
result = executor.execute(TEST_INSTANCE_ID, old_events, new_events)
996+
997+
if is_non_retriable:
998+
completion = get_and_validate_complete_orchestration_action_list(1, result.actions)
999+
if catch_failure:
1000+
assert completion.orchestrationStatus == pb.ORCHESTRATION_STATUS_COMPLETED
1001+
assert completion.result.value == json.dumps("compensated")
1002+
else:
1003+
assert completion.orchestrationStatus == pb.ORCHESTRATION_STATUS_FAILED
1004+
assert completion.failureDetails.innerFailure == failure_details
1005+
else:
1006+
assert len(result.actions) == 1
1007+
timer = result.actions[0]
1008+
assert timer.HasField("createTimer")
1009+
expected_fire_at = current_timestamp + timedelta(seconds=1)
1010+
assert timer.createTimer.fireAt.ToDatetime() == expected_fire_at
1011+
1012+
old_events += new_events
1013+
executor = worker._OrchestrationExecutor(registry, TEST_LOGGER, JsonDataConverter())
1014+
result = executor.execute(TEST_INSTANCE_ID, old_events, [
1015+
helpers.new_orchestrator_started_event(expected_fire_at),
1016+
helpers.new_timer_fired_event(timer.id, expected_fire_at),
1017+
])
1018+
retries = [
1019+
action for action in result.actions
1020+
if action.HasField("createSubOrchestration" if is_sub_orch else "scheduleTask")
1021+
]
1022+
assert len(retries) == 1
1023+
assert retries[0].id == 1
1024+
1025+
9411026
def test_activity_retry_policies():
9421027
"""Tests the retry policy logic for activity tasks"""
9431028

0 commit comments

Comments
 (0)