Skip to content

Commit e61d23b

Browse files
Serialize stop-the-world transactions across interpreters (#9094)
* Serialize stop-the-world transactions across interpreters Coordinate individual stop requests with process-wide collection, retain snapshot owners through rollback, and release transferred ownership after restart. Assisted-by: OpenAI Codex: model identifier unavailable * Reject reentrant stop-the-world transactions Use non-reused logical requester identities to fail before waiting on the same thread, while preserving cross-thread restart and fork ownership. Assisted-by: OpenAI Codex: model identifier unavailable --------- Co-authored-by: dot <dot@youknowone.org>
1 parent e157f64 commit e61d23b

2 files changed

Lines changed: 129 additions & 74 deletions

File tree

‎crates/vm/src/gc_state.rs‎

Lines changed: 28 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -207,22 +207,24 @@ type GcMap<K, V> = std::collections::HashMap<K, V, GcBuildHasher>;
207207
/// incref. Reading a refcount that another interpreter is changing is what
208208
/// makes an object look unreachable when it is not, so every live interpreter
209209
/// is stopped, not just the collecting one. Stopping in `runtime` id order
210-
/// keeps exclusion acquisition ordered; the `collecting` mutex additionally
211-
/// serializes collections process-wide, so no second collector can take these
212-
/// exclusions in another order.
210+
/// is deterministic. One process-wide STW transaction covers the entire group,
211+
/// preventing a single-interpreter requester from stopping a collector while
212+
/// that collector waits for a different interpreter.
213213
#[cfg(feature = "threading")]
214214
struct CollectStopTheWorld {
215-
/// Stopped interpreter states, in stop order. Held as strong references so
216-
/// an interpreter cannot be dropped between stop and restart, and kept past
217-
/// the restart so that releasing the last one — which frees that
218-
/// interpreter's objects, and so removes them from these lists — happens
219-
/// after the collection has let go of the generation locks.
220-
stopped: Vec<crate::common::rc::PyRc<crate::vm::PyGlobalState>>,
215+
/// Entire interpreter snapshot, retained through restart even on unwind.
216+
/// Dropping the last state can release Python objects and run finalizers,
217+
/// so both stopped and not-yet-stopped states must outlive the transaction
218+
/// and the collection's generation locks.
219+
states: Vec<crate::common::rc::PyRc<crate::vm::PyGlobalState>>,
220+
stopped_count: usize,
221221
/// Keeps interpreters from registering between the snapshot below and the
222-
/// restart. One registered in that window would be missing from `stopped`,
222+
/// restart. One registered in that window would be missing from `states`,
223223
/// so its bootstrap would keep running — and mutating the shared generation
224224
/// lists — while this collection reads them.
225225
admission: Option<parking_lot::MutexGuard<'static, ()>>,
226+
/// Acquired before admission and retained until every state is restarted.
227+
transaction: Option<crate::vm::StopTheWorldTransaction>,
226228
restarted: bool,
227229
}
228230

@@ -237,8 +239,10 @@ impl CollectStopTheWorld {
237239
// keep the historical no-barrier fallback.
238240
if !crate::vm::thread::current_vm_is_set() {
239241
return Self {
240-
stopped: Vec::new(),
242+
states: Vec::new(),
243+
stopped_count: 0,
241244
admission: None,
245+
transaction: None,
242246
restarted: true,
243247
};
244248
}
@@ -247,14 +251,20 @@ impl CollectStopTheWorld {
247251
// `stop_the_world` unwinds, dropping this guard restarts the
248252
// interpreters already stopped, instead of leaving their threads parked
249253
// and their exclusion held forever.
254+
let transaction = crate::vm::StopTheWorldTransaction::acquire();
250255
let mut guard = Self {
251-
stopped: Vec::new(),
256+
states: Vec::new(),
257+
stopped_count: 0,
252258
admission: Some(crate::vm::runtime::lock_admission_for_stop()),
259+
transaction: Some(transaction),
253260
restarted: false,
254261
};
255-
for state in crate::vm::runtime::live_interpreter_states() {
256-
state.stop_the_world.stop_the_world(&state);
257-
guard.stopped.push(state);
262+
guard.states = crate::vm::runtime::live_interpreter_states();
263+
for state in &guard.states {
264+
state
265+
.stop_the_world
266+
.stop_with_transaction(state, guard.transaction.as_ref().unwrap());
267+
guard.stopped_count += 1;
258268
}
259269
guard
260270
}
@@ -267,17 +277,18 @@ impl CollectStopTheWorld {
267277
self.restarted = true;
268278
// Reverse of the stop order. The references stay until this guard is
269279
// dropped; see the field comment.
270-
for state in self.stopped.iter().rev() {
280+
for state in self.states[..self.stopped_count].iter().rev() {
271281
state.stop_the_world.start_the_world(state);
272282
}
273283
// Nothing is parked any more, so registration may resume.
274284
self.admission = None;
285+
self.transaction = None;
275286
}
276287

277288
/// Whether this collection actually stopped the world.
278289
#[cfg(all(unix, debug_assertions))]
279290
fn is_stopped(&self) -> bool {
280-
!self.stopped.is_empty()
291+
self.stopped_count != 0
281292
}
282293
}
283294

‎crates/vm/src/vm/mod.rs‎

Lines changed: 101 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,68 @@ impl Default for ExceptionStack {
189189
}
190190
}
191191

192+
/// Serializes complete stop/restart transactions across interpreters.
193+
/// A collector holds one permit while stopping every interpreter; other callers
194+
/// transfer their permit to the stopped state until the matching restart.
195+
#[cfg(feature = "threading")]
196+
pub(crate) struct StopTheWorldTransaction {
197+
_private: (),
198+
}
199+
200+
#[cfg(feature = "threading")]
201+
static STW_TRANSACTION: AtomicU64 = AtomicU64::new(0);
202+
203+
#[cfg(feature = "threading")]
204+
static NEXT_STW_REQUESTER: AtomicU64 = AtomicU64::new(1);
205+
206+
#[cfg(feature = "threading")]
207+
thread_local! {
208+
// OS thread IDs can be reused after a requester exits, even while another
209+
// thread still owes its matching restart. Logical IDs are never reused.
210+
static STW_REQUESTER: u64 = NEXT_STW_REQUESTER
211+
.try_update(Ordering::Relaxed, Ordering::Relaxed, |id| id.checked_add(1))
212+
.expect("stop-the-world requester IDs exhausted");
213+
}
214+
215+
#[cfg(feature = "threading")]
216+
impl StopTheWorldTransaction {
217+
pub(crate) fn acquire() -> Self {
218+
let requester = STW_REQUESTER.with(|id| *id);
219+
loop {
220+
match STW_TRANSACTION.compare_exchange(
221+
0,
222+
requester,
223+
Ordering::AcqRel,
224+
Ordering::Relaxed,
225+
) {
226+
Ok(_) => return Self { _private: () },
227+
Err(owner) => assert_ne!(
228+
owner, requester,
229+
"stop-the-world transactions cannot be nested on the same thread"
230+
),
231+
}
232+
let wait = || {
233+
while STW_TRANSACTION.load(Ordering::Acquire) != 0 {
234+
std::thread::yield_now();
235+
}
236+
};
237+
// Wait detached from the CURRENT interpreter, not the interpreter
238+
// we intend to stop. Reattach without owning the permit, then retry:
239+
// a suspended or shutting-down waiter must not strand ownership.
240+
if thread::try_with_current_vm(|vm| vm.allow_threads(wait)).is_none() {
241+
wait();
242+
}
243+
}
244+
}
245+
}
246+
247+
#[cfg(feature = "threading")]
248+
impl Drop for StopTheWorldTransaction {
249+
fn drop(&mut self) {
250+
STW_TRANSACTION.store(0, Ordering::Release);
251+
}
252+
}
253+
192254
/// Stop-the-world state for fork safety. Before `fork()`, the requester
193255
/// stops all other Python threads so they are not holding internal locks.
194256
#[cfg(feature = "threading")]
@@ -199,10 +261,9 @@ pub struct StopTheWorldState {
199261
world_stopped: AtomicBool,
200262
/// Ident of the thread that requested the stop (like `stw->requester`)
201263
requester: AtomicU64,
202-
/// Single exclusion held for the whole stop→start span. Fork and GC are
203-
/// both stop-the-world requesters driving this shared state; only one may
204-
/// hold it at a time. Acquired before any stop bookkeeping (see
205-
/// `acquire_exclusion`) and released by `start_the_world`/`reset_after_fork`.
264+
/// Ownership transferred by a successful public stop. GC retains its
265+
/// transaction externally and leaves this false. Only a matching restart
266+
/// or the forking owner's child reset may release this ownership.
206267
exclusion: AtomicBool,
207268
/// Signaled by suspending threads when their state transitions to SUSPENDED
208269
notify_mutex: std::sync::Mutex<()>,
@@ -401,50 +462,11 @@ impl StopTheWorldState {
401462
forced_parks != 0 && self.thread_countdown.load(Ordering::Acquire) == 0
402463
}
403464

404-
/// Acquire the single stop-the-world exclusion in a park-friendly way.
405-
///
406-
/// Fork and GC both request stop-the-world through the same shared state;
407-
/// without this exclusion their `requester`/`requested`/countdown words
408-
/// could be clobbered by an interleaving requester, so the completion
409-
/// check could never converge and a requester would wait on itself forever.
410-
///
411-
/// The acquire must be park-friendly. While another requester's stop is in
412-
/// progress it sets this thread's stop bit and waits for it to suspend;
413-
/// blocking on a plain lock here would keep this thread from ever reaching
414-
/// that safepoint, so the active requester would wait for this thread while
415-
/// this thread waits for the lock — a deadlock swap. Instead we poll and
416-
/// honor the suspend request between tries. Suspending here is safe as long
417-
/// as any lock a spinning requester still holds is never acquired
418-
/// attached-blocking by another thread. The fork requester holds IMP_LOCK,
419-
/// but its acquisition detaches (`allow_threads`), so no attached thread
420-
/// blocks on it; the GC requester holds only the `collecting` mutex, which
421-
/// is only ever `try_lock`'d. The active requester therefore force-parks
422-
/// this thread, finishes its whole stop→start span, releases the exclusion,
423-
/// and only then does this thread resume and acquire it.
424-
fn acquire_exclusion(&self, state: &PyGlobalState) {
425-
if self
426-
.exclusion
427-
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed)
428-
.is_ok()
429-
{
430-
return;
431-
}
432-
loop {
433-
crate::vm::thread::suspend_if_needed(state);
434-
std::thread::yield_now();
435-
if self
436-
.exclusion
437-
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed)
438-
.is_ok()
439-
{
440-
return;
441-
}
442-
}
443-
}
444-
445-
/// Release the stop-the-world exclusion taken by `acquire_exclusion`.
465+
/// Release only a transaction owned by this state, after every wake/clear.
446466
fn release_exclusion(&self) {
447-
self.exclusion.store(false, Ordering::Release);
467+
if self.exclusion.swap(false, Ordering::AcqRel) {
468+
drop(StopTheWorldTransaction { _private: () });
469+
}
448470
}
449471

450472
/// Stop all non-requester threads (`stop_the_world`).
@@ -454,11 +476,27 @@ impl StopTheWorldState {
454476
/// 3. Waits (polling with 1 ms condvar timeout) for attached threads
455477
/// to self-suspend in `check_signals`.
456478
///
457-
/// Takes the shared exclusion first so at most one requester (fork or GC)
458-
/// drives the stop→start span at a time; it is released by
459-
/// `start_the_world`/`reset_after_fork`.
479+
/// Public calls own one process-wide transaction until the matching start.
480+
/// The caller must finish stop before invoking start, including when those
481+
/// calls occur on different threads; concurrent duplicate starts are invalid.
482+
/// Transactions are non-reentrant: a nested stop on the requesting thread
483+
/// panics before changing any stop state or releasing the outer transaction.
460484
pub fn stop_the_world(&self, state: &PyGlobalState) {
461-
self.acquire_exclusion(state);
485+
let transaction = StopTheWorldTransaction::acquire();
486+
self.stop_with_transaction(state, &transaction);
487+
self.exclusion.store(true, Ordering::Release);
488+
core::mem::forget(transaction);
489+
}
490+
491+
/// Stop one state within a transaction retained by the caller (notably GC).
492+
pub(crate) fn stop_with_transaction(
493+
&self,
494+
state: &PyGlobalState,
495+
_transaction: &StopTheWorldTransaction,
496+
) {
497+
// A partial stop must clear requests and wake parked slots before its
498+
// caller releases the transaction or rolls back previously stopped VMs.
499+
let rollback = scopeguard::guard((), |_| self.resume_threads(state, false));
462500
let start = std::time::Instant::now();
463501
let requester_ident = crate::stdlib::_thread::get_ident();
464502
self.requester.store(requester_ident, Ordering::Relaxed);
@@ -478,6 +516,7 @@ impl StopTheWorldState {
478516
stw_trace(format_args!(
479517
"stop end requester={requester_ident} wait_ns=0 polls=0"
480518
));
519+
scopeguard::ScopeGuard::into_inner(rollback);
481520
return;
482521
}
483522

@@ -526,10 +565,17 @@ impl StopTheWorldState {
526565
stw_trace(format_args!(
527566
"stop end requester={requester_ident} wait_ns={wait_ns} polls={polls}"
528567
));
568+
scopeguard::ScopeGuard::into_inner(rollback);
529569
}
530570

531571
/// Resume all suspended threads (`start_the_world`).
532572
pub fn start_the_world(&self, state: &PyGlobalState) {
573+
self.resume_threads(state, true);
574+
self.release_exclusion();
575+
}
576+
577+
/// Rollback also permits threads that never reached SUSPENDED.
578+
fn resume_threads(&self, state: &PyGlobalState, fully_stopped: bool) {
533579
use thread::ThreadState;
534580
let requester = self.requester.load(Ordering::Relaxed);
535581
stw_trace(format_args!("start begin requester={requester}"));
@@ -560,7 +606,7 @@ impl StopTheWorldState {
560606
continue;
561607
}
562608
debug_assert!(
563-
state == ThreadState::Suspended as i32,
609+
!fully_stopped || state == ThreadState::Suspended as i32,
564610
"non-requester thread not suspended at start-the-world: id={id} state={state}"
565611
);
566612
if state == ThreadState::Suspended as i32 {
@@ -579,9 +625,6 @@ impl StopTheWorldState {
579625
crate::signal::clear_stop_bit();
580626
#[cfg(debug_assertions)]
581627
self.debug_assert_all_non_requester_detached(state);
582-
// Release the exclusion last, ending the stop→start span so the next
583-
// requester (fork or GC) can proceed.
584-
self.release_exclusion();
585628
stw_trace(format_args!("start end requester={requester}"));
586629
}
587630

@@ -595,8 +638,9 @@ impl StopTheWorldState {
595638
// Only one thread survives fork; any stop-the-world bit inherited
596639
// from the parent is stale.
597640
crate::signal::clear_stop_bit();
598-
// The surviving child thread inherited the exclusion taken by the
599-
// pre-fork `stop_the_world`; release it (no start_the_world runs here).
641+
// before_fork owns the sole transaction. Release it exactly once via
642+
// that state's ownership flag; resetting other interpreters must not
643+
// release a later transaction. No unconditional global reset is needed.
600644
self.release_exclusion();
601645
stw_trace(format_args!("reset-after-fork"));
602646
}

0 commit comments

Comments
 (0)