Skip to content

fix(tracing): Flush pending span buckets in segment finish order #8261

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
sentry-junior wants to merge 1 commit into
base: master
Choose a base branch
from
+42 −3
Draft

fix(tracing): Flush pending span buckets in segment finish order #8261

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions sentry_sdk/_span_batcher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,9 @@ def __init__(

self._last_full_flush: float = time.monotonic() # drives time-based flushes
self._flush_event = threading.Event()
self._pending_flush: set[str] = set() # buckets to be flushed
# Buckets to be flushed, in the order they became ready. A dict, not a
# set, so that traces are sent in the order their segments finished.
self._pending_flush: dict[str, None] = {}

self._flusher: "Optional[threading.Thread]" = None
self._flusher_pid: "Optional[int]" = None
Expand All @@ -83,7 +85,7 @@ def _reset_thread_state(self) -> None:

self._last_full_flush = time.monotonic()
self._flush_event = threading.Event()
self._pending_flush = set()
self._pending_flush = {}

self._flusher = None
self._flusher_pid = None
Expand Down Expand Up @@ -143,7 +145,7 @@ def add(self, span: "SpanJSON") -> None:
or self._running_size[span["trace_id"]]
>= self.MAX_BYTES_BEFORE_FLUSH
):
self._pending_flush.add(span["trace_id"])
self._pending_flush[span["trace_id"]] = None
notify = True
else:
notify = False
Expand Down
37 changes: 37 additions & 0 deletions tests/tracing/test_span_batcher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import os
import sys
import threading
import time
from unittest import mock

Expand Down Expand Up @@ -479,6 +480,42 @@ def test_trace_bucket_flushes_when_segment_ends(
assert items[1].payload["name"] == "segment span"


def test_pending_traces_flush_in_segment_finish_order(
sentry_init, capture_envelopes, monkeypatch
):
"""Traces whose segments finish before the flusher wakes are sent in finish order."""
monkeypatch.setattr(SpanBatcher, "FLUSH_WAIT_TIME", 100000)

# Hold the flusher thread back until every segment below has finished, so
# all traces are pending when it runs.
flusher_may_run = threading.Event()
original_flush_loop = SpanBatcher._flush_loop

def delayed_flush_loop(self):
flusher_may_run.wait()
original_flush_loop(self)

monkeypatch.setattr(SpanBatcher, "_flush_loop", delayed_flush_loop)

sentry_init(traces_sample_rate=1.0, trace_lifecycle="stream")
envelopes = capture_envelopes()

finished_trace_ids = []
for i in range(10):
sentry_sdk.traces.new_trace()
with sentry_sdk.traces.start_span(name=f"segment {i}") as segment:
with sentry_sdk.traces.start_span(name="child"):
pass
finished_trace_ids.append(segment.trace_id)

flusher_may_run.set()
time.sleep(0.1)

assert [
envelope.headers["trace"]["trace_id"] for envelope in envelopes
] == finished_trace_ids


@pytest.mark.skipif(
sys.platform == "win32"
or not hasattr(os, "fork")
Expand Down
Loading