Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions taskiq/abc/middleware.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from collections.abc import Coroutine
from collections.abc import Set as AbstractSet
from types import CoroutineType
from typing import TYPE_CHECKING, Any, Union

Expand Down Expand Up @@ -140,3 +141,16 @@ def on_error(
:param result: returned value.
:param exception: found exception.
"""

@classmethod
def task_only_labels(cls) -> AbstractSet[str]:
"""
Returns labels that should not be sent over the wire.

Implementations can add labels here when their values are not suitable
for serialization. The labels remain on the registered task but are
omitted from outgoing messages.

:return: Names of the task-only labels.
"""
return frozenset()
11 changes: 6 additions & 5 deletions taskiq/kicker.py
Original file line number Diff line number Diff line change
Expand Up @@ -323,12 +323,13 @@ def _prepare_message(
for kwarg_name, kwarg_val in kwargs.items():
formatted_kwargs[kwarg_name] = self._prepare_arg(kwarg_val)

task_only_labels = {
label
for middleware in self.broker.middlewares
for label in middleware.task_only_labels()
}
for label, label_val in self.labels.items():
# `types_of_exceptions` is only ever read back from the
# locally registered task's labels (see retry middlewares),
# never from the wire. It holds exception classes, which
# can't be faithfully serialized, so don't ship it at all.
if label == "types_of_exceptions":
if label in task_only_labels:
continue
labels[label], labels_types[label] = prepare_label(label_val)

Expand Down
11 changes: 11 additions & 0 deletions taskiq/middlewares/simple_retry_middleware.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from collections.abc import Iterable
from collections.abc import Set as AbstractSet
from logging import getLogger
from typing import Any

Expand Down Expand Up @@ -114,3 +115,13 @@ async def on_error(
"Task '%s' invocation failed. Maximum retries count is reached.",
message.task_name,
)

@classmethod
def task_only_labels(cls) -> AbstractSet[str]:
"""
Adds ``types_of_exceptions`` to the set of task-only labels.

Per-task ``types_of_exceptions`` set via the task decorator is read
from the registered task object and cannot be properly serialized.
"""
return super().task_only_labels() | {"types_of_exceptions"}
11 changes: 11 additions & 0 deletions taskiq/middlewares/smart_retry_middleware.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import datetime
import random
from collections.abc import Iterable
from collections.abc import Set as AbstractSet
from logging import getLogger
from typing import Any

Expand Down Expand Up @@ -213,3 +214,13 @@ async def on_error(
"Task '%s' invocation failed. Maximum retries count is reached.",
message.task_name,
)

@classmethod
def task_only_labels(cls) -> AbstractSet[str]:
"""
Adds ``types_of_exceptions`` to the set of task-only labels.

Per-task ``types_of_exceptions`` set via the task decorator is read
from the registered task object and cannot be properly serialized.
"""
return super().task_only_labels() | {"types_of_exceptions"}
50 changes: 40 additions & 10 deletions tests/test_kicker.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,30 @@
from collections.abc import Set as AbstractSet
from typing import Any

from taskiq import InMemoryBroker
from taskiq import InMemoryBroker, SimpleRetryMiddleware
from taskiq.abc.middleware import TaskiqMiddleware
from taskiq.kicker import AsyncKicker


async def test_types_of_exceptions_not_serialized() -> None:
"""`types_of_exceptions` label should never be sent over the wire."""
broker = InMemoryBroker()
class _TaskOnlyLabelMiddleware(TaskiqMiddleware):
"""Middleware that declares a label as task-only (never goes on wire)."""

@broker.task(types_of_exceptions=(ValueError, TypeError))
@classmethod
def task_only_labels(cls) -> AbstractSet[str]:
return super().task_only_labels() | {"label_not_on_wire"}


async def test_task_only_labels_not_serialized() -> None:
"""Labels doesn't go on wire when corresponding middlewares are registered."""
broker = InMemoryBroker().with_middlewares(
SimpleRetryMiddleware(),
_TaskOnlyLabelMiddleware(),
)

@broker.task(
types_of_exceptions=(ValueError, TypeError),
label_not_on_wire="test_label_value",
)
async def run_task() -> None:
pass

Expand All @@ -17,26 +33,40 @@ async def run_task() -> None:

assert "types_of_exceptions" not in message.labels
assert "types_of_exceptions" not in (message.labels_types or {})
assert "label_not_on_wire" not in message.labels
assert "label_not_on_wire" not in (message.labels_types or {})


async def test_types_of_exceptions_still_local_on_task() -> None:
"""The registered task still keeps the real exception types locally."""
broker = InMemoryBroker()
async def test_task_only_labels_still_local_on_task() -> None:
"""The registered task still keeps the real label values locally."""
broker = InMemoryBroker().with_middlewares(
SimpleRetryMiddleware(),
_TaskOnlyLabelMiddleware(),
)

@broker.task(types_of_exceptions=(ValueError, TypeError))
@broker.task(
types_of_exceptions=(ValueError, TypeError),
label_not_on_wire="test_label_value",
)
async def run_task() -> None:
pass

task = broker.find_task(run_task.task_name)
assert task is not None
assert task.labels["types_of_exceptions"] == (ValueError, TypeError)
assert task.labels["label_not_on_wire"] == "test_label_value"


async def test_other_labels_still_serialized() -> None:
"""Unrelated labels are unaffected by the fix."""
broker = InMemoryBroker().with_middlewares(
SimpleRetryMiddleware(),
_TaskOnlyLabelMiddleware(),
)

kicker: AsyncKicker[Any, Any] = AsyncKicker(
task_name="some_task",
broker=InMemoryBroker(),
broker=broker,
labels={"retries": 3, "queue": "high_priority"},
)
message = kicker._prepare_message()
Expand Down
Loading