@@ -28,6 +28,7 @@ use std::collections::HashMap;
2828use std::thread_local;
2929
3030pub(crate) mod critical_section;
31+ pub(crate) mod lock;
3132
3233/// Thread states for stop-the-world support (`_Py_THREAD_*`).
3334///
@@ -64,7 +65,6 @@ impl ThreadState {
6465/// The exception field uses atomic operations for lock-free cross-thread reads.
6566#[cfg(feature = "threading")]
6667pub struct ThreadSlot {
67- pub(crate) interpreter_id: i64,
6868 /// Top of the owning thread's Python call stack, published for
6969 /// cross-thread readers (`sys._current_frames`, cross-thread `f_back`).
7070 /// The rest of the stack is reachable via each frame's `previous` pointer.
@@ -99,6 +99,9 @@ pub struct ThreadSlot {
9999 pub thread: std::thread::Thread,
100100 /// QSBR state for deferred memory reclamation.
101101 pub(crate) qsbr: Arc<crate::object::qsbr::QsbrSlot>,
102+ /// `tstate->critical_section`: tagged pointer to the top-most critical
103+ /// section. Only the owning thread reads or writes it.
104+ pub(crate) critical_section: AtomicUsize,
102105}
103106
104107#[cfg(feature = "threading")]
@@ -159,6 +162,12 @@ thread_local! {
159162 static INTERP_THREAD_SLOTS: RefCell<HashMap<i64, CurrentFrameSlot>> =
160163 RefCell::new(HashMap::new());
161164
165+ /// Critical sections of the slots `purge_other_interpreter_slots_after_fork`
166+ /// dropped, carried into the slot the next enter of that interpreter builds.
167+ #[cfg(feature = "threading")]
168+ static FORKED_CRITICAL_SECTIONS: RefCell<HashMap<i64, usize>> =
169+ RefCell::new(HashMap::new());
170+
162171 /// Current thread's slot for the currently entered interpreter.
163172 #[cfg(feature = "threading")]
164173 static CURRENT_THREAD_SLOT: RefCell<Option<CurrentFrameSlot>> = const { RefCell::new(None) };
@@ -360,7 +369,7 @@ pub fn restore_current_thread(state: SavedThreadState) {
360369 // subinterpreter slot (DETACHED); attaching that would leave the parent
361370 // slot detached and later confuse outermost detach.
362371 init_thread_slot_if_needed(vm);
363- attach_thread(&vm.state );
372+ attach_thread(vm );
364373 VM_STACK.with(|vms| *vms.borrow_mut() = vm_stack);
365374}
366375
@@ -384,7 +393,7 @@ pub fn attach_current_thread(
384393
385394 init_thread_slot_if_needed(vm);
386395
387- attach_thread(&vm.state );
396+ attach_thread(vm );
388397
389398 VM_STACK.with(|vms| {
390399 debug_assert!(vms.borrow().is_empty());
@@ -439,7 +448,6 @@ fn ensure_thread_slot(vm: &VirtualMachine) -> CurrentFrameSlot {
439448 let thread_id = crate::stdlib::_thread::get_ident();
440449 let mut registry = vm.state.thread_frames.lock();
441450 let new_slot = Arc::new(ThreadSlot {
442- interpreter_id: interp_id,
443451 #[cfg(unix)]
444452 top_frame: AtomicPtr::new(core::ptr::null_mut()),
445453 top_iframe: AtomicUsize::new(0),
@@ -472,6 +480,11 @@ fn ensure_thread_slot(vm: &VirtualMachine) -> CurrentFrameSlot {
472480 stop_requested: core::sync::atomic::AtomicBool::new(false),
473481 thread: std::thread::current(),
474482 qsbr: crate::object::qsbr::QSBR.register(),
483+ critical_section: AtomicUsize::new(
484+ FORKED_CRITICAL_SECTIONS
485+ .with(|forked| forked.borrow_mut().remove(&interp_id))
486+ .unwrap_or(0),
487+ ),
475488 });
476489 registry.insert(thread_id, new_slot.clone());
477490 drop(registry);
@@ -545,7 +558,7 @@ fn begin_interpreter_section(vm: &VirtualMachine) -> bool {
545558 detach_thread();
546559 }
547560 set_current_thread_slot(target);
548- attach_thread(&vm.state );
561+ attach_thread(vm );
549562 true
550563}
551564
@@ -564,7 +577,7 @@ fn end_interpreter_section(switched: bool) {
564577 // SAFETY: entries on VM_STACK are valid for their enter/set_current_vm scope.
565578 let vm = unsafe { vm_ptr.as_ref() };
566579 set_current_thread_slot(ensure_thread_slot(vm));
567- attach_thread(&vm.state );
580+ attach_thread(vm );
568581 }
569582}
570583
@@ -595,9 +608,9 @@ fn hang_thread() -> ! {
595608/// `Thread.is_alive()` remains true for a daemon forced off during finalize.
596609#[cfg(feature = "threading")]
597610pub fn hang_current_thread(state: &PyGlobalState) -> ! {
598- critical_section::suspend(state.interpreter_id);
599611 CURRENT_THREAD_SLOT.with(|slot| {
600612 if let Some(s) = slot.borrow().as_ref() {
613+ critical_section::suspend_all(s);
601614 let prev = s
602615 .state
603616 .swap(ThreadState::ShuttingDown as i32, Ordering::AcqRel);
@@ -638,7 +651,7 @@ pub fn set_other_threads_shutting_down(state: &PyGlobalState) {
638651}
639652
640653#[cfg(feature = "threading")]
641- fn attach_thread(state : &PyGlobalState ) {
654+ fn attach_thread(vm : &VirtualMachine ) {
642655 CURRENT_THREAD_SLOT.with(|slot| {
643656 if let Some(s) = slot.borrow().as_ref() {
644657 super::stw_trace(format_args!("attach begin"));
@@ -654,30 +667,32 @@ fn attach_thread(state: &PyGlobalState) {
654667 super::stw_trace(format_args!("attach DETACHED->ATTACHED"));
655668 break;
656669 }
657- Err(thread_state ) => match ThreadState::from_i32(thread_state ) {
670+ Err(state ) => match ThreadState::from_i32(state ) {
658671 Some(ThreadState::Suspended) => {
659672 // Parked by stop-the-world — wait until released to DETACHED
660673 super::stw_trace(format_args!("attach wait-suspended"));
661674 let wait_yields = wait_while_suspended(s);
662- state.stop_the_world.add_attach_wait_yields(wait_yields);
675+ vm. state.stop_the_world.add_attach_wait_yields(wait_yields);
663676 // Retry CAS
664677 }
665678 Some(ThreadState::ShuttingDown) => {
666679 super::stw_trace(format_args!("attach hang shutting-down"));
667680 hang_thread();
668681 }
669682 _ => {
670- debug_assert!(
671- false,
672- "unexpected thread state in attach: {thread_state}"
673- );
683+ debug_assert!(false, "unexpected thread state in attach: {state}");
674684 break;
675685 }
676686 },
677687 }
678688 }
679689 }
680690 });
691+ if let Some(s) = current_thread_slot() {
692+ // Resume previous critical section. This acquires the lock(s) from the
693+ // top-most critical section.
694+ critical_section::resume(&s);
695+ }
681696 // A stop-the-world may have been requested while this thread was detached.
682697 // Honoring it here (rather than only at the next bytecode safepoint) keeps
683698 // a thread doing rapid allow_threads calls from re-attaching and running
@@ -686,18 +701,15 @@ fn attach_thread(state: &PyGlobalState) {
686701 // it. Safe against a concurrent start_the_world: suspend_if_needed decides
687702 // whether to park under the registry lock, so it never parks after the
688703 // request has been withdrawn.
689- suspend_if_needed(state);
690- critical_section::resume(state);
704+ suspend_if_needed(&vm.state);
691705}
692706
693707/// Transition ATTACHED → DETACHED (like `_PyThreadState_Detach`).
694708#[cfg(feature = "threading")]
695709fn detach_thread() {
696- if let Some(slot) = current_thread_slot() {
697- critical_section::suspend(slot.interpreter_id);
698- }
699710 CURRENT_THREAD_SLOT.with(|slot| {
700711 if let Some(s) = slot.borrow().as_ref() {
712+ critical_section::suspend_all(s);
701713 match s.state.compare_exchange(
702714 ThreadState::Attached as i32,
703715 ThreadState::Detached as i32,
@@ -727,11 +739,6 @@ fn detach_thread() {
727739/// `Py_BEGIN_ALLOW_THREADS` / `Py_END_ALLOW_THREADS` equivalent.
728740#[cfg(feature = "threading")]
729741pub fn allow_threads<R>(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R {
730- allow_threads_for_state(&vm.state, f)
731- }
732-
733- #[cfg(feature = "threading")]
734- fn allow_threads_for_state<R>(state: &PyGlobalState, f: impl FnOnce() -> R) -> R {
735742 // Preserve save/restore semantics:
736743 // only detach if this call observed ATTACHED at entry, and always restore
737744 // on unwind.
@@ -745,7 +752,7 @@ fn allow_threads_for_state<R>(state: &PyGlobalState, f: impl FnOnce() -> R) -> R
745752 }
746753
747754 detach_thread();
748- let reattach_guard = scopeguard::guard(state , attach_thread);
755+ let reattach_guard = scopeguard::guard(vm , attach_thread);
749756 let result = f();
750757 drop(reattach_guard);
751758 result
@@ -781,7 +788,7 @@ pub fn attach_for_callback<R>(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R {
781788 return f();
782789 }
783790
784- attach_thread(&vm.state );
791+ attach_thread(vm );
785792 // Detach again even if `f` unwinds, so the `allow_threads` this is nested
786793 // inside still finds the state it left behind.
787794 let redetach_guard = scopeguard::guard((), |()| detach_thread());
@@ -851,25 +858,11 @@ pub fn suspend_if_needed(state: &PyGlobalState) {
851858#[cfg(feature = "threading")]
852859#[cold]
853860fn do_suspend(state: &PyGlobalState) {
854- if let Some(slot) = current_thread_slot() {
855- assert_eq!(slot.interpreter_id, state.interpreter_id);
856- }
857- critical_section::suspend(state.interpreter_id);
858- // Restore even when the request expired before the registry was locked.
859- // The guard runs after CURRENT_THREAD_SLOT's borrow has ended.
860- let _resume = scopeguard::guard(state, |state| {
861- if current_thread_slot().is_some_and(|slot| {
862- slot.interpreter_id == state.interpreter_id
863- && slot.state.load(Ordering::Acquire) == ThreadState::Attached as i32
864- }) {
865- critical_section::resume(state);
866- }
867- });
868861 let stw = &state.stop_the_world;
869- CURRENT_THREAD_SLOT.with(|slot| {
862+ let reattached = CURRENT_THREAD_SLOT.with(|slot| {
870863 let borrowed = slot.borrow();
871864 let Some(s) = borrowed.as_ref() else {
872- return;
865+ return false ;
873866 };
874867
875868 // Decide whether to park while holding the thread registry. Both edges
@@ -883,6 +876,8 @@ fn do_suspend(state: &PyGlobalState) {
883876 let park = {
884877 let _registry = state.thread_frames.lock();
885878 if stw.requested.load(Ordering::Acquire) {
879+ // Release critical sections before publishing SUSPENDED.
880+ critical_section::suspend_all(s);
886881 Some(s.state.compare_exchange(
887882 ThreadState::Attached as i32,
888883 ThreadState::Suspended as i32,
@@ -899,7 +894,7 @@ fn do_suspend(state: &PyGlobalState) {
899894 match park {
900895 None => {
901896 super::stw_trace(format_args!("suspend skip not-requested"));
902- return;
897+ return false ;
903898 }
904899 Some(Ok(_)) => {
905900 // Consumed this thread's stop request bit.
@@ -909,13 +904,13 @@ fn do_suspend(state: &PyGlobalState) {
909904 Some(ThreadState::Detached) => {
910905 // Leaving VM; caller will re-check on next entry.
911906 super::stw_trace(format_args!("suspend skip DETACHED"));
912- return;
907+ return false ;
913908 }
914909 Some(ThreadState::Suspended) => {
915910 // Already parked by another path.
916911 s.stop_requested.store(false, Ordering::Release);
917912 super::stw_trace(format_args!("suspend skip already-suspended"));
918- return;
913+ return false ;
919914 }
920915 Some(ThreadState::ShuttingDown) => {
921916 s.stop_requested.store(false, Ordering::Release);
@@ -924,7 +919,7 @@ fn do_suspend(state: &PyGlobalState) {
924919 }
925920 _ => {
926921 debug_assert!(false, "unexpected thread state in suspend: {state}");
927- return;
922+ return false ;
928923 }
929924 },
930925 }
@@ -969,7 +964,11 @@ fn do_suspend(state: &PyGlobalState) {
969964 }
970965 s.stop_requested.store(false, Ordering::Release);
971966 super::stw_trace(format_args!("suspend resume -> ATTACHED"));
967+ true
972968 });
969+ if reattached && let Some(s) = current_thread_slot() {
970+ critical_section::resume(&s);
971+ }
973972}
974973
975974#[cfg(feature = "threading")]
@@ -1157,14 +1156,19 @@ pub fn get_all_current_exceptions(vm: &VirtualMachine) -> Vec<(u64, Option<PyBas
11571156/// Called at thread exit (or when leaving an interpreter permanently).
11581157#[cfg(feature = "threading")]
11591158pub fn cleanup_current_thread_frames(vm: &VirtualMachine) {
1160- critical_section::assert_empty(vm.state.interpreter_id);
11611159 let thread_id = crate::stdlib::_thread::get_ident();
11621160 let interp_id = vm.state.interpreter_id;
11631161
11641162 // Prefer the slot registered for this interpreter; fall back to CURRENT.
11651163 let slot_for_interp = INTERP_THREAD_SLOTS.with(|slots| slots.borrow_mut().remove(&interp_id));
11661164 let current_slot = CURRENT_THREAD_SLOT.with(|slot| slot.borrow().as_ref().cloned());
11671165 let slot_to_clean = slot_for_interp.or(current_slot);
1166+ debug_assert!(
1167+ slot_to_clean
1168+ .as_ref()
1169+ .is_none_or(|slot| slot.critical_section.load(Ordering::Relaxed) == 0),
1170+ "thread cleanup cannot abandon a critical section"
1171+ );
11681172
11691173 // A dying thread should not remain logically ATTACHED while its
11701174 // thread-state slot is being removed.
@@ -1261,8 +1265,10 @@ pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) {
12611265 }
12621266 };
12631267 let top_iframe_ptr = get_current_frame() as usize;
1268+ // The surviving thread keeps its critical sections.
1269+ let critical_section =
1270+ current_thread_slot().map_or(0, |slot| slot.critical_section.load(Ordering::Relaxed));
12641271 let new_slot = Arc::new(ThreadSlot {
1265- interpreter_id: vm.state.interpreter_id,
12661272 // The surviving child thread keeps executing its current frame chain.
12671273 // Only publish heavy frames for signal safety.
12681274 #[cfg(unix)]
@@ -1277,6 +1283,7 @@ pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) {
12771283 stop_requested: core::sync::atomic::AtomicBool::new(false),
12781284 thread: std::thread::current(),
12791285 qsbr: crate::object::qsbr::QSBR.register(),
1286+ critical_section: AtomicUsize::new(critical_section),
12801287 });
12811288 FRAME_SLOT_CACHE.with(|cache| {
12821289 #[cfg(unix)]
@@ -1309,7 +1316,14 @@ pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) {
13091316#[cfg(feature = "threading")]
13101317pub fn purge_other_interpreter_slots_after_fork(keep_id: i64) {
13111318 INTERP_THREAD_SLOTS.with(|slots| {
1312- slots.borrow_mut().retain(|&id, _| id == keep_id);
1319+ slots.borrow_mut().retain(|&id, slot| {
1320+ let critical_section = slot.critical_section.load(Ordering::Relaxed);
1321+ if id != keep_id && critical_section != 0 {
1322+ FORKED_CRITICAL_SECTIONS
1323+ .with(|forked| forked.borrow_mut().insert(id, critical_section));
1324+ }
1325+ id == keep_id
1326+ });
13131327 });
13141328}
13151329
0 commit comments