Repository navigation
Expand file tree
/
Copy pathartemis_transport.py
More file actions
312 lines (274 loc) · 14 KB
/
Copy pathartemis_transport.py
File metadata and controls
312 lines (274 loc) · 14 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
"""Apache ActiveMQ Artemis transport over AMQP 1.0. Requires the ``artemis`` extra:
pip install "babelqueue[artemis]"
Artemis speaks AMQP 1.0 (not the 0-9-1 of RabbitMQ), so this transport uses the
``python-qpid-proton`` blocking client rather than ``pika``. Producing sends the canonical
envelope as the message body and projects the contract envelope fields onto the AMQP
properties a JMS peer reads: ``correlation-id`` = trace_id (JMSCorrelationID), ``creation-time``
= meta.created_at (JMSTimestamp), the ``x-opt-jms-type`` message annotation = URN (JMSType, the
AMQP-JMS mapping), plus the ``bq_`` application properties (``bq_schema_version`` /
``bq_source_lang`` / ``bq_attempts`` / ``bq_app_id``; underscores, since JMS property names must
be valid Java identifiers) — so a Java (JMS) or .NET/Node/... peer
routes and correlates without parsing the body. The URN in the body's ``job`` field stays
authoritative.
Consuming reserves one message at a time (``receive`` -> process -> ``accept``); the
authoritative attempt count is the envelope's ``attempts`` (the body), reconciled against the
broker's native AMQP ``delivery-count`` as ``attempts = max(body, delivery_count)`` — no -1,
because the AMQP header counter is 0-based (0 on first delivery), counting prior failed
deliveries. (The Java JMS binding reads ``JMSXDeliveryCount`` which is 1-based and subtracts 1,
arriving at the same 0-based ``attempts``.) A message left un-accepted is redelivered
(at-least-once).
Out-of-band transport headers (a W3C ``traceparent`` for cross-hop span linkage, ADR-0028) ride as
extra AMQP 1.0 application-properties (string-valued, hyphen-free names) beside the ``bq_*`` projection:
:meth:`ArtemisTransport.publish_with_headers` writes them (the projection wins a collision) and
:meth:`~ArtemisTransport.pop` surfaces every non-projection application-property on
:attr:`~babelqueue.transport.ReceivedMessage.headers`; the ``bq_*`` projection keys are not leaked. The
body is untouched (GR-1).
This implements §7 of the broker-bindings contract. The envelope is unchanged
(``schema_version`` stays 1); Apache ActiveMQ Artemis is purely additive.
URL form: ``artemis://host:5672`` (or ``artemis+ssl://`` for TLS) — translated to the
``amqp://`` / ``amqps://`` proton speaks. For credentials or a custom connection, build the
transport directly and pass it via ``BabelQueue(transport=...)`` or
``ArtemisTransport(connection=...)``.
"""
from __future__ import annotations
import logging
import re
from typing import Any, Dict, Optional
from urllib.parse import quote
from .codec import ASCII_WHITESPACE, SCHEMA_VERSION, EnvelopeCodec, parse_envelope
from .headers import out_of_band_headers
from .transport import ReceivedMessage, Transport, truncate_declared_version
_logger = logging.getLogger("babelqueue.artemis")
#: A blank ``bq_schema_version``: empty or only ASCII whitespace (space, \t, \n, \v, \f, \r) — the
#: same set in every SDK (GR-5). Deliberately not ``str.strip()``, which also strips U+001C-U+001F,
#: U+0085, U+00A0, U+2028, U+3000 ...
_BLANK_ASCII_PATTERN = re.compile("[" + re.escape(ASCII_WHITESPACE) + "]*")
#: Prefix of the reserved ``route_urn`` a version-gated message is delivered under. It is not a
#: ``urn:babel:`` URN, so no handler can legitimately own it; the runtime dead-letters it
#: directly (never retried, never decoded; the DLQ record is the incoming raw body). The
#: rejected ``bq_schema_version`` is named in the logged error.
UNSUPPORTED_SCHEMA_ROUTE_PREFIX = "babelqueue:unsupported-schema-version:"
JMS_TYPE_ANNOTATION = "x-opt-jms-type"
APP_ID = "babelqueue"
class ArtemisTransport(Transport):
def __init__(
self,
url: str = "artemis://localhost:5672",
*,
connection: Any = None,
credit: int = 1,
receive_timeout_millis: int = 1000,
**connect_options: Any,
) -> None:
self._url = url or "artemis://localhost:5672"
self._credit = credit
self._receive_timeout_millis = receive_timeout_millis
self._connect_options = connect_options
self._senders: Dict[str, Any] = {}
self._receivers: Dict[str, Any] = {}
if connection is not None:
self._connection = connection
return
self._connection = self._build_connection() # pragma: no cover - needs Artemis / network
def _build_connection(self) -> Any: # pragma: no cover - needs Artemis / network
try:
from proton.utils import BlockingConnection
except ImportError as exc:
raise ImportError(
"ArtemisTransport requires the 'python-qpid-proton' package. Install with "
'pip install "babelqueue[artemis]".'
) from exc
return BlockingConnection(self._to_amqp_url(self._url), **self._connect_options)
# -- helpers ------------------------------------------------------------
@staticmethod
def _to_amqp_url(url: str) -> str:
"""``artemis://`` -> ``amqp://`` and ``artemis+ssl://`` -> ``amqps://`` (proton's schemes)."""
if url.startswith("artemis+ssl://"):
return "amqps://" + url[len("artemis+ssl://"):]
if url.startswith("artemis://"):
return "amqp://" + url[len("artemis://"):]
return url
def _sender(self, queue: str) -> Any:
sender = self._senders.get(queue)
if sender is None:
sender = self._connection.create_sender(queue)
self._senders[queue] = sender
return sender
def _receiver(self, queue: str) -> Any:
receiver = self._receivers.get(queue)
if receiver is None:
receiver = self._connection.create_receiver(queue, credit=self._credit)
self._receivers[queue] = receiver
return receiver
@staticmethod
def _projection(body: str) -> Dict[str, str]:
"""AMQP application properties (string->string) — a redundant, routable view of the
body: bq_schema_version/bq_source_lang/bq_attempts/bq_app_id. §7.2 (the URN is carried
by the x-opt-jms-type annotation, trace_id by correlation-id). The names use UNDERSCORES,
not hyphens: a JMS property name must be a valid Java identifier, and every Artemis SDK
uses the same JMS-legal form for cross-protocol parity."""
env = parse_envelope(body)
if not env:
return {}
meta = env.get("meta") or {}
props: Dict[str, str] = {}
if meta.get("schema_version") is not None:
props["bq_schema_version"] = str(meta["schema_version"])
if meta.get("lang"):
props["bq_source_lang"] = str(meta["lang"])
props["bq_attempts"] = str(int(env.get("attempts", 0) or 0))
props["bq_app_id"] = APP_ID
return props
@staticmethod
def _jms_type(body: str) -> str:
env = parse_envelope(body)
return str(env["job"]) if env and env.get("job") else ""
@staticmethod
def _correlation_id(body: str) -> str:
env = parse_envelope(body)
return str(env["trace_id"]) if env and env.get("trace_id") else ""
@staticmethod
def _creation_seconds(body: str) -> Optional[float]:
"""proton's creation_time is float seconds; the contract's created_at is epoch ms."""
env = parse_envelope(body)
meta = (env or {}).get("meta") or {}
created_at = meta.get("created_at")
if created_at is None:
return None
try:
return int(created_at) / 1000.0
except (TypeError, ValueError): # pragma: no cover - defensive
return None
def _message(self, body: str, headers: Optional[Dict[str, str]] = None) -> Any:
"""Build the proton Message projecting the §7 JMS-readable metadata. ``headers`` are
out-of-band application-properties (ADR-0028) merged *under* the ``bq_*`` projection."""
from proton import Message, symbol
message: Any = Message(body=body)
props = out_of_band_headers(headers)
props.update(self._projection(body))
if props:
message.properties = props
correlation_id = self._correlation_id(body)
if correlation_id:
message.correlation_id = correlation_id
creation = self._creation_seconds(body)
if creation is not None:
message.creation_time = creation
jms_type = self._jms_type(body)
if jms_type:
message.annotations = {symbol(JMS_TYPE_ANNOTATION): jms_type}
return message
@staticmethod
def _reconcile(body: str, delivery_count: Any) -> str:
"""Set attempts to max(current, delivery_count). The AMQP delivery-count header is
0-based (0 on first delivery), so it maps directly to attempts with no -1. The runtime
retries by republishing with attempts+1 in the body (delivery-count back to 0), so a
republished message must not have its higher body count lowered."""
try:
dc = int(delivery_count)
except (ValueError, TypeError):
return body
if dc <= 0:
return body
env = EnvelopeCodec.decode_lenient(body)
if not env or dc <= int(env.get("attempts", 0) or 0):
return body
env["attempts"] = dc
return EnvelopeCodec.encode(env)
@staticmethod
def _delivery_count(message: Any) -> int:
value = getattr(message, "delivery_count", 0)
try:
return int(value)
except (TypeError, ValueError): # pragma: no cover - defensive
return 0
@staticmethod
def _unsupported_schema_version(properties: Any) -> Optional[str]:
"""The §7.7 step-3 gate on the ``bq_schema_version`` application property (underscores —
ADR-0017): the offending raw value, or ``None``.
A missing property, a null value or a blank value (the empty string or only ASCII
whitespace — the same set in every SDK, GR-5, not ``str.strip()``) is not gated: the usual
decode path runs. A ``bytes`` value is decoded as UTF-8. A present value is accepted only
when it is exactly the core's supported ``SCHEMA_VERSION``: the string ``"1"`` or the
integer ``1`` (a ``bool`` is not an integer here, though it is one in Python); ``"2"``,
``"x"``, ``"01"``, ``" 1"``, ``0``, ``2``, ``1.5``, ``True`` and any other type are
rejected."""
if not isinstance(properties, dict):
return None
raw = properties.get("bq_schema_version")
if raw is None:
return None
if isinstance(raw, (bytes, bytearray)):
raw = bytes(raw).decode("utf-8", "replace")
if isinstance(raw, str):
if _BLANK_ASCII_PATTERN.fullmatch(raw) is not None or raw == str(SCHEMA_VERSION):
return None
return raw
if isinstance(raw, int) and not isinstance(raw, bool) and raw == SCHEMA_VERSION:
return None
return str(raw)
@staticmethod
def _payload(message: Any) -> str:
body = getattr(message, "body", None)
if isinstance(body, bytes):
# Lossless: see ``PulsarTransport._payload`` (ADR-0035 poison record, no crash).
return body.decode("utf-8", "surrogateescape")
return str(body) if body is not None else ""
# -- Transport ----------------------------------------------------------
def publish(self, queue: str, body: str) -> None:
self._sender(queue).send(self._message(body))
def publish_with_headers(self, queue: str, body: str, headers: Dict[str, str]) -> None:
""":meth:`publish` with out-of-band ``headers`` (a W3C ``traceparent``, the replay-bypass
marker ...) carried as extra AMQP 1.0 application-properties beside the ``bq_*`` projection
(:class:`~babelqueue.transport.HeaderPublisher`, ADR-0028, broker-bindings 7.2). The
projection wins a key collision and the body is byte-identical to :meth:`publish`."""
self._sender(queue).send(self._message(body, headers))
def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
wait = self._receive_timeout_millis / 1000.0
if timeout and timeout > 0:
wait = timeout
receiver = self._receiver(queue)
try:
message = receiver.receive(timeout=wait)
except Exception as exc: # noqa: BLE001 - proton raises proton.Timeout on no message
if type(exc).__name__ == "Timeout":
return None
raise
if message is None: # pragma: no cover - defensive (real client raises instead)
return None
# §7.7 step 3: gate on the ``bq_schema_version`` property BEFORE any body decode (the
# attempts reconcile below decodes the body). A rejected message keeps its raw body and the
# reserved route; the runtime then dead-letters it directly (no retry).
unsupported = truncate_declared_version(
self._unsupported_schema_version(getattr(message, "properties", None))
)
if unsupported is not None:
_logger.warning(
"Artemis message on %r carries unsupported bq_schema_version %r (supported: %s); "
"not decoding it.",
queue,
unsupported,
SCHEMA_VERSION,
)
return ReceivedMessage(
body=self._payload(message),
queue=queue,
handle=receiver,
route_urn=UNSUPPORTED_SCHEMA_ROUTE_PREFIX + quote(unsupported, safe=""),
)
body = self._reconcile(self._payload(message), self._delivery_count(message))
return ReceivedMessage(
body=body,
queue=queue,
handle=receiver,
headers=out_of_band_headers(getattr(message, "properties", None)),
)
def ack(self, message: ReceivedMessage) -> None:
if message.handle is None:
return
message.handle.accept()
def close(self) -> None: # pragma: no cover - resource cleanup
try:
self._connection.close()
except Exception: # noqa: BLE001 - best-effort cleanup
pass