|
3 | 3 |
|
4 | 4 | import json |
5 | 5 | import logging |
6 | | -from datetime import datetime, timedelta |
| 6 | +from datetime import UTC, datetime, timedelta |
7 | 7 | from typing import Any |
8 | 8 |
|
9 | 9 | import pytest |
@@ -938,6 +938,91 @@ def get_exception_properties(self, exception: Exception): |
938 | 938 | assert failure.innerFailure.properties["code"].string_value == "VALUE" |
939 | 939 |
|
940 | 940 |
|
| 941 | +@pytest.mark.parametrize("is_sub_orch", [False, True], ids=["activity", "sub-orchestration"]) |
| 942 | +@pytest.mark.parametrize("is_non_retriable", [None, False, True], ids=["unset", "false", "true"]) |
| 943 | +@pytest.mark.parametrize("is_replaying", [False, True], ids=["new-failure", "replayed-failure"]) |
| 944 | +@pytest.mark.parametrize("catch_failure", [False, True], ids=["unhandled", "caught"]) |
| 945 | +def test_retry_policy_respects_non_retriable_failure( |
| 946 | + is_sub_orch: bool, |
| 947 | + is_non_retriable: bool | None, |
| 948 | + is_replaying: bool, |
| 949 | + catch_failure: bool, |
| 950 | +): |
| 951 | + """Non-retriable failures bypass retries and retain their failure details.""" |
| 952 | + |
| 953 | + def orchestrator(ctx: task.OrchestrationContext, _): |
| 954 | + retry_policy = task.RetryPolicy( |
| 955 | + first_retry_interval=timedelta(seconds=1), |
| 956 | + max_number_of_attempts=3, |
| 957 | + ) |
| 958 | + try: |
| 959 | + if is_sub_orch: |
| 960 | + yield ctx.call_sub_orchestrator("child", retry_policy=retry_policy) |
| 961 | + else: |
| 962 | + yield ctx.call_activity("child", retry_policy=retry_policy) |
| 963 | + except task.TaskFailedError as error: |
| 964 | + if not catch_failure: |
| 965 | + raise |
| 966 | + assert error.details.error_type == "builtins.ValueError" |
| 967 | + assert error.details.message == "Permanent failure" |
| 968 | + return "compensated" |
| 969 | + return None |
| 970 | + |
| 971 | + registry = worker._Registry() |
| 972 | + name = registry.add_orchestrator(orchestrator) |
| 973 | + current_timestamp = datetime.now(UTC).replace(tzinfo=None) |
| 974 | + old_events = [ |
| 975 | + helpers.new_orchestrator_started_event(current_timestamp), |
| 976 | + helpers.new_execution_started_event(name, TEST_INSTANCE_ID, encoded_input=None), |
| 977 | + ] |
| 978 | + if is_sub_orch: |
| 979 | + old_events.append(helpers.new_sub_orchestration_created_event(1, "child", "child-instance")) |
| 980 | + failed_event = helpers.new_sub_orchestration_failed_event(1, ValueError("Permanent failure")) |
| 981 | + failure_details = failed_event.subOrchestrationInstanceFailed.failureDetails |
| 982 | + else: |
| 983 | + old_events.append(helpers.new_task_scheduled_event(1, "child")) |
| 984 | + failed_event = helpers.new_task_failed_event(1, ValueError("Permanent failure")) |
| 985 | + failure_details = failed_event.taskFailed.failureDetails |
| 986 | + if is_non_retriable is not None: |
| 987 | + failure_details.isNonRetriable = is_non_retriable |
| 988 | + |
| 989 | + new_events = [helpers.new_orchestrator_started_event(current_timestamp)] |
| 990 | + if is_replaying: |
| 991 | + old_events.append(failed_event) |
| 992 | + else: |
| 993 | + new_events.append(failed_event) |
| 994 | + executor = worker._OrchestrationExecutor(registry, TEST_LOGGER, JsonDataConverter()) |
| 995 | + result = executor.execute(TEST_INSTANCE_ID, old_events, new_events) |
| 996 | + |
| 997 | + if is_non_retriable: |
| 998 | + completion = get_and_validate_complete_orchestration_action_list(1, result.actions) |
| 999 | + if catch_failure: |
| 1000 | + assert completion.orchestrationStatus == pb.ORCHESTRATION_STATUS_COMPLETED |
| 1001 | + assert completion.result.value == json.dumps("compensated") |
| 1002 | + else: |
| 1003 | + assert completion.orchestrationStatus == pb.ORCHESTRATION_STATUS_FAILED |
| 1004 | + assert completion.failureDetails.innerFailure == failure_details |
| 1005 | + else: |
| 1006 | + assert len(result.actions) == 1 |
| 1007 | + timer = result.actions[0] |
| 1008 | + assert timer.HasField("createTimer") |
| 1009 | + expected_fire_at = current_timestamp + timedelta(seconds=1) |
| 1010 | + assert timer.createTimer.fireAt.ToDatetime() == expected_fire_at |
| 1011 | + |
| 1012 | + old_events += new_events |
| 1013 | + executor = worker._OrchestrationExecutor(registry, TEST_LOGGER, JsonDataConverter()) |
| 1014 | + result = executor.execute(TEST_INSTANCE_ID, old_events, [ |
| 1015 | + helpers.new_orchestrator_started_event(expected_fire_at), |
| 1016 | + helpers.new_timer_fired_event(timer.id, expected_fire_at), |
| 1017 | + ]) |
| 1018 | + retries = [ |
| 1019 | + action for action in result.actions |
| 1020 | + if action.HasField("createSubOrchestration" if is_sub_orch else "scheduleTask") |
| 1021 | + ] |
| 1022 | + assert len(retries) == 1 |
| 1023 | + assert retries[0].id == 1 |
| 1024 | + |
| 1025 | + |
941 | 1026 | def test_activity_retry_policies(): |
942 | 1027 | """Tests the retry policy logic for activity tasks""" |
943 | 1028 |
|
|
0 commit comments