@@ -189,6 +189,68 @@ impl Default for ExceptionStack {
189189 }
190190}
191191
192+ /// Serializes complete stop/restart transactions across interpreters.
193+ /// A collector holds one permit while stopping every interpreter; other callers
194+ /// transfer their permit to the stopped state until the matching restart.
195+ #[ cfg( feature = "threading" ) ]
196+ pub ( crate ) struct StopTheWorldTransaction {
197+ _private : ( ) ,
198+ }
199+
200+ #[ cfg( feature = "threading" ) ]
201+ static STW_TRANSACTION : AtomicU64 = AtomicU64 :: new ( 0 ) ;
202+
203+ #[ cfg( feature = "threading" ) ]
204+ static NEXT_STW_REQUESTER : AtomicU64 = AtomicU64 :: new ( 1 ) ;
205+
206+ #[ cfg( feature = "threading" ) ]
207+ thread_local ! {
208+ // OS thread IDs can be reused after a requester exits, even while another
209+ // thread still owes its matching restart. Logical IDs are never reused.
210+ static STW_REQUESTER : u64 = NEXT_STW_REQUESTER
211+ . try_update( Ordering :: Relaxed , Ordering :: Relaxed , |id| id. checked_add( 1 ) )
212+ . expect( "stop-the-world requester IDs exhausted" ) ;
213+ }
214+
215+ #[ cfg( feature = "threading" ) ]
216+ impl StopTheWorldTransaction {
217+ pub ( crate ) fn acquire ( ) -> Self {
218+ let requester = STW_REQUESTER . with ( |id| * id) ;
219+ loop {
220+ match STW_TRANSACTION . compare_exchange (
221+ 0 ,
222+ requester,
223+ Ordering :: AcqRel ,
224+ Ordering :: Relaxed ,
225+ ) {
226+ Ok ( _) => return Self { _private : ( ) } ,
227+ Err ( owner) => assert_ne ! (
228+ owner, requester,
229+ "stop-the-world transactions cannot be nested on the same thread"
230+ ) ,
231+ }
232+ let wait = || {
233+ while STW_TRANSACTION . load ( Ordering :: Acquire ) != 0 {
234+ std:: thread:: yield_now ( ) ;
235+ }
236+ } ;
237+ // Wait detached from the CURRENT interpreter, not the interpreter
238+ // we intend to stop. Reattach without owning the permit, then retry:
239+ // a suspended or shutting-down waiter must not strand ownership.
240+ if thread:: try_with_current_vm ( |vm| vm. allow_threads ( wait) ) . is_none ( ) {
241+ wait ( ) ;
242+ }
243+ }
244+ }
245+ }
246+
247+ #[ cfg( feature = "threading" ) ]
248+ impl Drop for StopTheWorldTransaction {
249+ fn drop ( & mut self ) {
250+ STW_TRANSACTION . store ( 0 , Ordering :: Release ) ;
251+ }
252+ }
253+
192254/// Stop-the-world state for fork safety. Before `fork()`, the requester
193255/// stops all other Python threads so they are not holding internal locks.
194256#[ cfg( feature = "threading" ) ]
@@ -199,10 +261,9 @@ pub struct StopTheWorldState {
199261 world_stopped : AtomicBool ,
200262 /// Ident of the thread that requested the stop (like `stw->requester`)
201263 requester : AtomicU64 ,
202- /// Single exclusion held for the whole stop→start span. Fork and GC are
203- /// both stop-the-world requesters driving this shared state; only one may
204- /// hold it at a time. Acquired before any stop bookkeeping (see
205- /// `acquire_exclusion`) and released by `start_the_world`/`reset_after_fork`.
264+ /// Ownership transferred by a successful public stop. GC retains its
265+ /// transaction externally and leaves this false. Only a matching restart
266+ /// or the forking owner's child reset may release this ownership.
206267 exclusion : AtomicBool ,
207268 /// Signaled by suspending threads when their state transitions to SUSPENDED
208269 notify_mutex : std:: sync:: Mutex < ( ) > ,
@@ -401,50 +462,11 @@ impl StopTheWorldState {
401462 forced_parks != 0 && self . thread_countdown . load ( Ordering :: Acquire ) == 0
402463 }
403464
404- /// Acquire the single stop-the-world exclusion in a park-friendly way.
405- ///
406- /// Fork and GC both request stop-the-world through the same shared state;
407- /// without this exclusion their `requester`/`requested`/countdown words
408- /// could be clobbered by an interleaving requester, so the completion
409- /// check could never converge and a requester would wait on itself forever.
410- ///
411- /// The acquire must be park-friendly. While another requester's stop is in
412- /// progress it sets this thread's stop bit and waits for it to suspend;
413- /// blocking on a plain lock here would keep this thread from ever reaching
414- /// that safepoint, so the active requester would wait for this thread while
415- /// this thread waits for the lock — a deadlock swap. Instead we poll and
416- /// honor the suspend request between tries. Suspending here is safe as long
417- /// as any lock a spinning requester still holds is never acquired
418- /// attached-blocking by another thread. The fork requester holds IMP_LOCK,
419- /// but its acquisition detaches (`allow_threads`), so no attached thread
420- /// blocks on it; the GC requester holds only the `collecting` mutex, which
421- /// is only ever `try_lock`'d. The active requester therefore force-parks
422- /// this thread, finishes its whole stop→start span, releases the exclusion,
423- /// and only then does this thread resume and acquire it.
424- fn acquire_exclusion ( & self , state : & PyGlobalState ) {
425- if self
426- . exclusion
427- . compare_exchange ( false , true , Ordering :: AcqRel , Ordering :: Relaxed )
428- . is_ok ( )
429- {
430- return ;
431- }
432- loop {
433- crate :: vm:: thread:: suspend_if_needed ( state) ;
434- std:: thread:: yield_now ( ) ;
435- if self
436- . exclusion
437- . compare_exchange ( false , true , Ordering :: AcqRel , Ordering :: Relaxed )
438- . is_ok ( )
439- {
440- return ;
441- }
442- }
443- }
444-
445- /// Release the stop-the-world exclusion taken by `acquire_exclusion`.
465+ /// Release only a transaction owned by this state, after every wake/clear.
446466 fn release_exclusion ( & self ) {
447- self . exclusion . store ( false , Ordering :: Release ) ;
467+ if self . exclusion . swap ( false , Ordering :: AcqRel ) {
468+ drop ( StopTheWorldTransaction { _private : ( ) } ) ;
469+ }
448470 }
449471
450472 /// Stop all non-requester threads (`stop_the_world`).
@@ -454,11 +476,27 @@ impl StopTheWorldState {
454476 /// 3. Waits (polling with 1 ms condvar timeout) for attached threads
455477 /// to self-suspend in `check_signals`.
456478 ///
457- /// Takes the shared exclusion first so at most one requester (fork or GC)
458- /// drives the stop→start span at a time; it is released by
459- /// `start_the_world`/`reset_after_fork`.
479+ /// Public calls own one process-wide transaction until the matching start.
480+ /// The caller must finish stop before invoking start, including when those
481+ /// calls occur on different threads; concurrent duplicate starts are invalid.
482+ /// Transactions are non-reentrant: a nested stop on the requesting thread
483+ /// panics before changing any stop state or releasing the outer transaction.
460484 pub fn stop_the_world ( & self , state : & PyGlobalState ) {
461- self . acquire_exclusion ( state) ;
485+ let transaction = StopTheWorldTransaction :: acquire ( ) ;
486+ self . stop_with_transaction ( state, & transaction) ;
487+ self . exclusion . store ( true , Ordering :: Release ) ;
488+ core:: mem:: forget ( transaction) ;
489+ }
490+
491+ /// Stop one state within a transaction retained by the caller (notably GC).
492+ pub ( crate ) fn stop_with_transaction (
493+ & self ,
494+ state : & PyGlobalState ,
495+ _transaction : & StopTheWorldTransaction ,
496+ ) {
497+ // A partial stop must clear requests and wake parked slots before its
498+ // caller releases the transaction or rolls back previously stopped VMs.
499+ let rollback = scopeguard:: guard ( ( ) , |_| self . resume_threads ( state, false ) ) ;
462500 let start = std:: time:: Instant :: now ( ) ;
463501 let requester_ident = crate :: stdlib:: _thread:: get_ident ( ) ;
464502 self . requester . store ( requester_ident, Ordering :: Relaxed ) ;
@@ -478,6 +516,7 @@ impl StopTheWorldState {
478516 stw_trace ( format_args ! (
479517 "stop end requester={requester_ident} wait_ns=0 polls=0"
480518 ) ) ;
519+ scopeguard:: ScopeGuard :: into_inner ( rollback) ;
481520 return ;
482521 }
483522
@@ -526,10 +565,17 @@ impl StopTheWorldState {
526565 stw_trace ( format_args ! (
527566 "stop end requester={requester_ident} wait_ns={wait_ns} polls={polls}"
528567 ) ) ;
568+ scopeguard:: ScopeGuard :: into_inner ( rollback) ;
529569 }
530570
531571 /// Resume all suspended threads (`start_the_world`).
532572 pub fn start_the_world ( & self , state : & PyGlobalState ) {
573+ self . resume_threads ( state, true ) ;
574+ self . release_exclusion ( ) ;
575+ }
576+
577+ /// Rollback also permits threads that never reached SUSPENDED.
578+ fn resume_threads ( & self , state : & PyGlobalState , fully_stopped : bool ) {
533579 use thread:: ThreadState ;
534580 let requester = self . requester . load ( Ordering :: Relaxed ) ;
535581 stw_trace ( format_args ! ( "start begin requester={requester}" ) ) ;
@@ -560,7 +606,7 @@ impl StopTheWorldState {
560606 continue ;
561607 }
562608 debug_assert ! (
563- state == ThreadState :: Suspended as i32 ,
609+ !fully_stopped || state == ThreadState :: Suspended as i32 ,
564610 "non-requester thread not suspended at start-the-world: id={id} state={state}"
565611 ) ;
566612 if state == ThreadState :: Suspended as i32 {
@@ -579,9 +625,6 @@ impl StopTheWorldState {
579625 crate :: signal:: clear_stop_bit ( ) ;
580626 #[ cfg( debug_assertions) ]
581627 self . debug_assert_all_non_requester_detached ( state) ;
582- // Release the exclusion last, ending the stop→start span so the next
583- // requester (fork or GC) can proceed.
584- self . release_exclusion ( ) ;
585628 stw_trace ( format_args ! ( "start end requester={requester}" ) ) ;
586629 }
587630
@@ -595,8 +638,9 @@ impl StopTheWorldState {
595638 // Only one thread survives fork; any stop-the-world bit inherited
596639 // from the parent is stale.
597640 crate :: signal:: clear_stop_bit ( ) ;
598- // The surviving child thread inherited the exclusion taken by the
599- // pre-fork `stop_the_world`; release it (no start_the_world runs here).
641+ // before_fork owns the sole transaction. Release it exactly once via
642+ // that state's ownership flag; resetting other interpreters must not
643+ // release a later transaction. No unconditional global reset is needed.
600644 self . release_exclusion ( ) ;
601645 stw_trace ( format_args ! ( "reset-after-fork" ) ) ;
602646 }
0 commit comments