Skip to content

Commit ffe7737

Browse files
ZylphrexZylphrex
authored
feat(profiling): Better gevent support (getsentry#1822)
We're missing frames from gevent threads. Using `gevent.threadpool.ThreadPool` seems to fix that. The monkey patching gevent does is causing the sampler thread to run in a greenlet on the same thread as the all other greenlets. So when it is taking a sample, the sampler is current greenlet thus no useful stacks can be seen.
1 parent 1445c73 commit ffe7737

2 files changed

Lines changed: 173 additions & 67 deletions

File tree

‎sentry_sdk/profiler.py‎

Lines changed: 127 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,15 @@
104104
},
105105
)
106106

107+
try:
108+
from gevent.monkey import is_module_patched # type: ignore
109+
except ImportError:
110+
111+
def is_module_patched(*args, **kwargs):
112+
# type: (*Any, **Any) -> bool
113+
# unable to import from gevent means no modules have been patched
114+
return False
115+
107116

108117
_scheduler = None # type: Optional[Scheduler]
109118

@@ -128,11 +137,31 @@ def setup_profiler(options):
128137

129138
frequency = 101
130139

131-
profiler_mode = options["_experiments"].get("profiler_mode", SleepScheduler.mode)
132-
if profiler_mode == SleepScheduler.mode:
133-
_scheduler = SleepScheduler(frequency=frequency)
140+
if is_module_patched("threading") or is_module_patched("_thread"):
141+
# If gevent has patched the threading modules then we cannot rely on
142+
# them to spawn a native thread for sampling.
143+
# Instead we default to the GeventScheduler which is capable of
144+
# spawning native threads within gevent.
145+
default_profiler_mode = GeventScheduler.mode
146+
else:
147+
default_profiler_mode = ThreadScheduler.mode
148+
149+
profiler_mode = options["_experiments"].get("profiler_mode", default_profiler_mode)
150+
151+
if (
152+
profiler_mode == ThreadScheduler.mode
153+
# for legacy reasons, we'll keep supporting sleep mode for this scheduler
154+
or profiler_mode == "sleep"
155+
):
156+
_scheduler = ThreadScheduler(frequency=frequency)
157+
elif profiler_mode == GeventScheduler.mode:
158+
try:
159+
_scheduler = GeventScheduler(frequency=frequency)
160+
except ImportError:
161+
raise ValueError("Profiler mode: {} is not available".format(profiler_mode))
134162
else:
135163
raise ValueError("Unknown profiler mode: {}".format(profiler_mode))
164+
136165
_scheduler.setup()
137166

138167
atexit.register(teardown_profiler)
@@ -445,6 +474,11 @@ def __init__(self, frequency):
445474
# type: (int) -> None
446475
self.interval = 1.0 / frequency
447476

477+
self.sampler = self.make_sampler()
478+
479+
self.new_profiles = deque() # type: Deque[Profile]
480+
self.active_profiles = set() # type: Set[Profile]
481+
448482
def __enter__(self):
449483
# type: () -> Scheduler
450484
self.setup()
@@ -462,50 +496,6 @@ def teardown(self):
462496
# type: () -> None
463497
raise NotImplementedError
464498

465-
def start_profiling(self, profile):
466-
# type: (Profile) -> None
467-
raise NotImplementedError
468-
469-
def stop_profiling(self, profile):
470-
# type: (Profile) -> None
471-
raise NotImplementedError
472-
473-
474-
class ThreadScheduler(Scheduler):
475-
"""
476-
This abstract scheduler is based on running a daemon thread that will call
477-
the sampler at a regular interval.
478-
"""
479-
480-
mode = "thread"
481-
name = None # type: Optional[str]
482-
483-
def __init__(self, frequency):
484-
# type: (int) -> None
485-
super(ThreadScheduler, self).__init__(frequency=frequency)
486-
487-
self.sampler = self.make_sampler()
488-
489-
# used to signal to the thread that it should stop
490-
self.event = threading.Event()
491-
492-
# make sure the thread is a daemon here otherwise this
493-
# can keep the application running after other threads
494-
# have exited
495-
self.thread = threading.Thread(name=self.name, target=self.run, daemon=True)
496-
497-
self.new_profiles = deque() # type: Deque[Profile]
498-
self.active_profiles = set() # type: Set[Profile]
499-
500-
def setup(self):
501-
# type: () -> None
502-
self.thread.start()
503-
504-
def teardown(self):
505-
# type: () -> None
506-
self.event.set()
507-
self.thread.join()
508-
509499
def start_profiling(self, profile):
510500
# type: (Profile) -> None
511501
profile.active = True
@@ -515,10 +505,6 @@ def stop_profiling(self, profile):
515505
# type: (Profile) -> None
516506
profile.active = False
517507

518-
def run(self):
519-
# type: () -> None
520-
raise NotImplementedError
521-
522508
def make_sampler(self):
523509
# type: () -> Callable[..., None]
524510
cwd = os.getcwd()
@@ -600,14 +586,99 @@ def _sample_stack(*args, **kwargs):
600586
return _sample_stack
601587

602588

603-
class SleepScheduler(ThreadScheduler):
589+
class ThreadScheduler(Scheduler):
604590
"""
605-
This scheduler uses time.sleep to wait the required interval before calling
606-
the sampling function.
591+
This scheduler is based on running a daemon thread that will call
592+
the sampler at a regular interval.
607593
"""
608594

609-
mode = "sleep"
610-
name = "sentry.profiler.SleepScheduler"
595+
mode = "thread"
596+
name = "sentry.profiler.ThreadScheduler"
597+
598+
def __init__(self, frequency):
599+
# type: (int) -> None
600+
super(ThreadScheduler, self).__init__(frequency=frequency)
601+
602+
# used to signal to the thread that it should stop
603+
self.event = threading.Event()
604+
605+
# make sure the thread is a daemon here otherwise this
606+
# can keep the application running after other threads
607+
# have exited
608+
self.thread = threading.Thread(name=self.name, target=self.run, daemon=True)
609+
610+
def setup(self):
611+
# type: () -> None
612+
self.thread.start()
613+
614+
def teardown(self):
615+
# type: () -> None
616+
self.event.set()
617+
self.thread.join()
618+
619+
def run(self):
620+
# type: () -> None
621+
last = time.perf_counter()
622+
623+
while True:
624+
if self.event.is_set():
625+
break
626+
627+
self.sampler()
628+
629+
# some time may have elapsed since the last time
630+
# we sampled, so we need to account for that and
631+
# not sleep for too long
632+
elapsed = time.perf_counter() - last
633+
if elapsed < self.interval:
634+
time.sleep(self.interval - elapsed)
635+
636+
# after sleeping, make sure to take the current
637+
# timestamp so we can use it next iteration
638+
last = time.perf_counter()
639+
640+
641+
class GeventScheduler(Scheduler):
642+
"""
643+
This scheduler is based on the thread scheduler but adapted to work with
644+
gevent. When using gevent, it may monkey patch the threading modules
645+
(`threading` and `_thread`). This results in the use of greenlets instead
646+
of native threads.
647+
648+
This is an issue because the sampler CANNOT run in a greenlet because
649+
1. Other greenlets doing sync work will prevent the sampler from running
650+
2. The greenlet runs in the same thread as other greenlets so when taking
651+
a sample, other greenlets will have been evicted from the thread. This
652+
results in a sample containing only the sampler's code.
653+
"""
654+
655+
mode = "gevent"
656+
name = "sentry.profiler.GeventScheduler"
657+
658+
def __init__(self, frequency):
659+
# type: (int) -> None
660+
661+
# This can throw an ImportError that must be caught if `gevent` is
662+
# not installed.
663+
from gevent.threadpool import ThreadPool # type: ignore
664+
665+
super(GeventScheduler, self).__init__(frequency=frequency)
666+
667+
# used to signal to the thread that it should stop
668+
self.event = threading.Event()
669+
670+
# Using gevent's ThreadPool allows us to bypass greenlets and spawn
671+
# native threads.
672+
self.pool = ThreadPool(1)
673+
674+
def setup(self):
675+
# type: () -> None
676+
self.pool.spawn(self.run)
677+
678+
def teardown(self):
679+
# type: () -> None
680+
self.event.set()
681+
self.pool.join()
611682

612683
def run(self):
613684
# type: () -> None

‎tests/test_profiler.py‎

Lines changed: 46 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,32 +6,56 @@
66
import pytest
77

88
from sentry_sdk.profiler import (
9+
GeventScheduler,
910
Profile,
10-
SleepScheduler,
11+
ThreadScheduler,
1112
extract_frame,
1213
extract_stack,
1314
get_frame_name,
1415
setup_profiler,
1516
)
1617
from sentry_sdk.tracing import Transaction
1718

19+
try:
20+
import gevent
21+
except ImportError:
22+
gevent = None
23+
1824

1925
minimum_python_33 = pytest.mark.skipif(
2026
sys.version_info < (3, 3), reason="Profiling is only supported in Python >= 3.3"
2127
)
2228

29+
requires_gevent = pytest.mark.skipif(gevent is None, reason="gevent not enabled")
30+
2331

2432
def process_test_sample(sample):
2533
return [(tid, (stack, stack)) for tid, stack in sample]
2634

2735

28-
@minimum_python_33
29-
def test_profiler_invalid_mode(teardown_profiling):
36+
@pytest.mark.parametrize(
37+
"mode",
38+
[
39+
pytest.param("foo"),
40+
pytest.param(
41+
"gevent",
42+
marks=pytest.mark.skipif(gevent is not None, reason="gevent not enabled"),
43+
),
44+
],
45+
)
46+
def test_profiler_invalid_mode(mode, teardown_profiling):
3047
with pytest.raises(ValueError):
31-
setup_profiler({"_experiments": {"profiler_mode": "magic"}})
48+
setup_profiler({"_experiments": {"profiler_mode": mode}})
3249

3350

34-
@pytest.mark.parametrize("mode", ["sleep"])
51+
@pytest.mark.parametrize(
52+
"mode",
53+
[
54+
pytest.param("thread"),
55+
pytest.param("sleep"),
56+
pytest.param("gevent", marks=requires_gevent),
57+
],
58+
)
3559
def test_profiler_valid_mode(mode, teardown_profiling):
3660
# should not raise any exceptions
3761
setup_profiler({"_experiments": {"profiler_mode": mode}})
@@ -56,7 +80,6 @@ def inherited_instance_method(self):
5680

5781
def inherited_instance_method_wrapped(self):
5882
def wrapped():
59-
self
6083
return inspect.currentframe()
6184

6285
return wrapped
@@ -68,7 +91,6 @@ def inherited_class_method(cls):
6891
@classmethod
6992
def inherited_class_method_wrapped(cls):
7093
def wrapped():
71-
cls
7294
return inspect.currentframe()
7395

7496
return wrapped
@@ -84,7 +106,6 @@ def instance_method(self):
84106

85107
def instance_method_wrapped(self):
86108
def wrapped():
87-
self
88109
return inspect.currentframe()
89110

90111
return wrapped
@@ -96,7 +117,6 @@ def class_method(cls):
96117
@classmethod
97118
def class_method_wrapped(cls):
98119
def wrapped():
99-
cls
100120
return inspect.currentframe()
101121

102122
return wrapped
@@ -258,7 +278,19 @@ def get_scheduler_threads(scheduler):
258278
@minimum_python_33
259279
@pytest.mark.parametrize(
260280
("scheduler_class",),
261-
[pytest.param(SleepScheduler, id="sleep scheduler")],
281+
[
282+
pytest.param(ThreadScheduler, id="thread scheduler"),
283+
pytest.param(
284+
GeventScheduler,
285+
marks=[
286+
requires_gevent,
287+
pytest.mark.skip(
288+
reason="cannot find this thread via threading.enumerate()"
289+
),
290+
],
291+
id="gevent scheduler",
292+
),
293+
],
262294
)
263295
def test_thread_scheduler_single_background_thread(scheduler_class):
264296
scheduler = scheduler_class(frequency=1000)
@@ -576,7 +608,10 @@ def test_thread_scheduler_single_background_thread(scheduler_class):
576608
)
577609
@pytest.mark.parametrize(
578610
("scheduler_class",),
579-
[pytest.param(SleepScheduler, id="sleep scheduler")],
611+
[
612+
pytest.param(ThreadScheduler, id="thread scheduler"),
613+
pytest.param(GeventScheduler, marks=requires_gevent, id="gevent scheduler"),
614+
],
580615
)
581616
def test_profile_processing(
582617
DictionaryContaining, # noqa: N803

0 commit comments

Comments
 (0)