Skip to content

Commit 33c5206

Browse files
committed
Replace operation sections with PyMutex critical sections
Add PyMutex with the lock.c protocol: LOCKED/HAS_PARKED bits, a bounded spin, direct handoff to waiters parked longer than 1 ms, and detaching while parked. Rebuild critical sections on PyMutex with the stack kept as a tagged pointer in ThreadSlot: single and two-mutex sections, recursive top-most skipping, locking skipped while the world is stopped, SuspendAll on detach/suspend/shutdown and Resume of the top-most section on attach. Fork carries the stacks into the replacement slots. Assisted-by: Claude Code:claude-opus-5-5
1 parent aaf295e commit 33c5206

5 files changed

Lines changed: 862 additions & 345 deletions

File tree

‎.cspell.dict/rust-more.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@ trai
8383
ulonglong
8484
unic
8585
unistd
86+
unparked
87+
unparking
8688
unraw
8789
unsync
8890
wasip1

‎crates/vm/src/vm/stop_the_world.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ impl Drop for ForkStopTheWorld {
166166
fn drop(&mut self) {
167167
if self.child_repair_incomplete {
168168
// Partial child repair cannot support Rust unwind recovery or
169-
// Python destruction. Like CPython's fatal child-setup path, fail
169+
// Python destruction. Like a fatal child-setup failure, abort
170170
// before destroying the retained interpreter/thread roots.
171171
rustpython_host_env::os::abort();
172172
}

‎crates/vm/src/vm/thread.rs‎

Lines changed: 63 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ use std::collections::HashMap;
2828
use std::thread_local;
2929

3030
pub(crate) mod critical_section;
31+
pub(crate) mod lock;
3132

3233
/// Thread states for stop-the-world support (`_Py_THREAD_*`).
3334
///
@@ -64,7 +65,6 @@ impl ThreadState {
6465
/// The exception field uses atomic operations for lock-free cross-thread reads.
6566
#[cfg(feature = "threading")]
6667
pub struct ThreadSlot {
67-
pub(crate) interpreter_id: i64,
6868
/// Top of the owning thread's Python call stack, published for
6969
/// cross-thread readers (`sys._current_frames`, cross-thread `f_back`).
7070
/// The rest of the stack is reachable via each frame's `previous` pointer.
@@ -99,6 +99,9 @@ pub struct ThreadSlot {
9999
pub thread: std::thread::Thread,
100100
/// QSBR state for deferred memory reclamation.
101101
pub(crate) qsbr: Arc<crate::object::qsbr::QsbrSlot>,
102+
/// `tstate->critical_section`: tagged pointer to the top-most critical
103+
/// section. Only the owning thread reads or writes it.
104+
pub(crate) critical_section: AtomicUsize,
102105
}
103106

104107
#[cfg(feature = "threading")]
@@ -159,6 +162,12 @@ thread_local! {
159162
static INTERP_THREAD_SLOTS: RefCell<HashMap<i64, CurrentFrameSlot>> =
160163
RefCell::new(HashMap::new());
161164

165+
/// Critical sections of the slots `purge_other_interpreter_slots_after_fork`
166+
/// dropped, carried into the slot the next enter of that interpreter builds.
167+
#[cfg(feature = "threading")]
168+
static FORKED_CRITICAL_SECTIONS: RefCell<HashMap<i64, usize>> =
169+
RefCell::new(HashMap::new());
170+
162171
/// Current thread's slot for the currently entered interpreter.
163172
#[cfg(feature = "threading")]
164173
static CURRENT_THREAD_SLOT: RefCell<Option<CurrentFrameSlot>> = const { RefCell::new(None) };
@@ -360,7 +369,7 @@ pub fn restore_current_thread(state: SavedThreadState) {
360369
// subinterpreter slot (DETACHED); attaching that would leave the parent
361370
// slot detached and later confuse outermost detach.
362371
init_thread_slot_if_needed(vm);
363-
attach_thread(&vm.state);
372+
attach_thread(vm);
364373
VM_STACK.with(|vms| *vms.borrow_mut() = vm_stack);
365374
}
366375

@@ -384,7 +393,7 @@ pub fn attach_current_thread(
384393

385394
init_thread_slot_if_needed(vm);
386395

387-
attach_thread(&vm.state);
396+
attach_thread(vm);
388397

389398
VM_STACK.with(|vms| {
390399
debug_assert!(vms.borrow().is_empty());
@@ -439,7 +448,6 @@ fn ensure_thread_slot(vm: &VirtualMachine) -> CurrentFrameSlot {
439448
let thread_id = crate::stdlib::_thread::get_ident();
440449
let mut registry = vm.state.thread_frames.lock();
441450
let new_slot = Arc::new(ThreadSlot {
442-
interpreter_id: interp_id,
443451
#[cfg(unix)]
444452
top_frame: AtomicPtr::new(core::ptr::null_mut()),
445453
top_iframe: AtomicUsize::new(0),
@@ -472,6 +480,11 @@ fn ensure_thread_slot(vm: &VirtualMachine) -> CurrentFrameSlot {
472480
stop_requested: core::sync::atomic::AtomicBool::new(false),
473481
thread: std::thread::current(),
474482
qsbr: crate::object::qsbr::QSBR.register(),
483+
critical_section: AtomicUsize::new(
484+
FORKED_CRITICAL_SECTIONS
485+
.with(|forked| forked.borrow_mut().remove(&interp_id))
486+
.unwrap_or(0),
487+
),
475488
});
476489
registry.insert(thread_id, new_slot.clone());
477490
drop(registry);
@@ -545,7 +558,7 @@ fn begin_interpreter_section(vm: &VirtualMachine) -> bool {
545558
detach_thread();
546559
}
547560
set_current_thread_slot(target);
548-
attach_thread(&vm.state);
561+
attach_thread(vm);
549562
true
550563
}
551564

@@ -564,7 +577,7 @@ fn end_interpreter_section(switched: bool) {
564577
// SAFETY: entries on VM_STACK are valid for their enter/set_current_vm scope.
565578
let vm = unsafe { vm_ptr.as_ref() };
566579
set_current_thread_slot(ensure_thread_slot(vm));
567-
attach_thread(&vm.state);
580+
attach_thread(vm);
568581
}
569582
}
570583

@@ -595,9 +608,9 @@ fn hang_thread() -> ! {
595608
/// `Thread.is_alive()` remains true for a daemon forced off during finalize.
596609
#[cfg(feature = "threading")]
597610
pub fn hang_current_thread(state: &PyGlobalState) -> ! {
598-
critical_section::suspend(state.interpreter_id);
599611
CURRENT_THREAD_SLOT.with(|slot| {
600612
if let Some(s) = slot.borrow().as_ref() {
613+
critical_section::suspend_all(s);
601614
let prev = s
602615
.state
603616
.swap(ThreadState::ShuttingDown as i32, Ordering::AcqRel);
@@ -638,7 +651,7 @@ pub fn set_other_threads_shutting_down(state: &PyGlobalState) {
638651
}
639652

640653
#[cfg(feature = "threading")]
641-
fn attach_thread(state: &PyGlobalState) {
654+
fn attach_thread(vm: &VirtualMachine) {
642655
CURRENT_THREAD_SLOT.with(|slot| {
643656
if let Some(s) = slot.borrow().as_ref() {
644657
super::stw_trace(format_args!("attach begin"));
@@ -654,30 +667,32 @@ fn attach_thread(state: &PyGlobalState) {
654667
super::stw_trace(format_args!("attach DETACHED->ATTACHED"));
655668
break;
656669
}
657-
Err(thread_state) => match ThreadState::from_i32(thread_state) {
670+
Err(state) => match ThreadState::from_i32(state) {
658671
Some(ThreadState::Suspended) => {
659672
// Parked by stop-the-world — wait until released to DETACHED
660673
super::stw_trace(format_args!("attach wait-suspended"));
661674
let wait_yields = wait_while_suspended(s);
662-
state.stop_the_world.add_attach_wait_yields(wait_yields);
675+
vm.state.stop_the_world.add_attach_wait_yields(wait_yields);
663676
// Retry CAS
664677
}
665678
Some(ThreadState::ShuttingDown) => {
666679
super::stw_trace(format_args!("attach hang shutting-down"));
667680
hang_thread();
668681
}
669682
_ => {
670-
debug_assert!(
671-
false,
672-
"unexpected thread state in attach: {thread_state}"
673-
);
683+
debug_assert!(false, "unexpected thread state in attach: {state}");
674684
break;
675685
}
676686
},
677687
}
678688
}
679689
}
680690
});
691+
if let Some(s) = current_thread_slot() {
692+
// Resume previous critical section. This acquires the lock(s) from the
693+
// top-most critical section.
694+
critical_section::resume(&s);
695+
}
681696
// A stop-the-world may have been requested while this thread was detached.
682697
// Honoring it here (rather than only at the next bytecode safepoint) keeps
683698
// a thread doing rapid allow_threads calls from re-attaching and running
@@ -686,18 +701,15 @@ fn attach_thread(state: &PyGlobalState) {
686701
// it. Safe against a concurrent start_the_world: suspend_if_needed decides
687702
// whether to park under the registry lock, so it never parks after the
688703
// request has been withdrawn.
689-
suspend_if_needed(state);
690-
critical_section::resume(state);
704+
suspend_if_needed(&vm.state);
691705
}
692706

693707
/// Transition ATTACHED → DETACHED (like `_PyThreadState_Detach`).
694708
#[cfg(feature = "threading")]
695709
fn detach_thread() {
696-
if let Some(slot) = current_thread_slot() {
697-
critical_section::suspend(slot.interpreter_id);
698-
}
699710
CURRENT_THREAD_SLOT.with(|slot| {
700711
if let Some(s) = slot.borrow().as_ref() {
712+
critical_section::suspend_all(s);
701713
match s.state.compare_exchange(
702714
ThreadState::Attached as i32,
703715
ThreadState::Detached as i32,
@@ -727,11 +739,6 @@ fn detach_thread() {
727739
/// `Py_BEGIN_ALLOW_THREADS` / `Py_END_ALLOW_THREADS` equivalent.
728740
#[cfg(feature = "threading")]
729741
pub fn allow_threads<R>(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R {
730-
allow_threads_for_state(&vm.state, f)
731-
}
732-
733-
#[cfg(feature = "threading")]
734-
fn allow_threads_for_state<R>(state: &PyGlobalState, f: impl FnOnce() -> R) -> R {
735742
// Preserve save/restore semantics:
736743
// only detach if this call observed ATTACHED at entry, and always restore
737744
// on unwind.
@@ -745,7 +752,7 @@ fn allow_threads_for_state<R>(state: &PyGlobalState, f: impl FnOnce() -> R) -> R
745752
}
746753

747754
detach_thread();
748-
let reattach_guard = scopeguard::guard(state, attach_thread);
755+
let reattach_guard = scopeguard::guard(vm, attach_thread);
749756
let result = f();
750757
drop(reattach_guard);
751758
result
@@ -781,7 +788,7 @@ pub fn attach_for_callback<R>(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R {
781788
return f();
782789
}
783790

784-
attach_thread(&vm.state);
791+
attach_thread(vm);
785792
// Detach again even if `f` unwinds, so the `allow_threads` this is nested
786793
// inside still finds the state it left behind.
787794
let redetach_guard = scopeguard::guard((), |()| detach_thread());
@@ -851,25 +858,11 @@ pub fn suspend_if_needed(state: &PyGlobalState) {
851858
#[cfg(feature = "threading")]
852859
#[cold]
853860
fn do_suspend(state: &PyGlobalState) {
854-
if let Some(slot) = current_thread_slot() {
855-
assert_eq!(slot.interpreter_id, state.interpreter_id);
856-
}
857-
critical_section::suspend(state.interpreter_id);
858-
// Restore even when the request expired before the registry was locked.
859-
// The guard runs after CURRENT_THREAD_SLOT's borrow has ended.
860-
let _resume = scopeguard::guard(state, |state| {
861-
if current_thread_slot().is_some_and(|slot| {
862-
slot.interpreter_id == state.interpreter_id
863-
&& slot.state.load(Ordering::Acquire) == ThreadState::Attached as i32
864-
}) {
865-
critical_section::resume(state);
866-
}
867-
});
868861
let stw = &state.stop_the_world;
869-
CURRENT_THREAD_SLOT.with(|slot| {
862+
let reattached = CURRENT_THREAD_SLOT.with(|slot| {
870863
let borrowed = slot.borrow();
871864
let Some(s) = borrowed.as_ref() else {
872-
return;
865+
return false;
873866
};
874867

875868
// Decide whether to park while holding the thread registry. Both edges
@@ -883,6 +876,8 @@ fn do_suspend(state: &PyGlobalState) {
883876
let park = {
884877
let _registry = state.thread_frames.lock();
885878
if stw.requested.load(Ordering::Acquire) {
879+
// Release critical sections before publishing SUSPENDED.
880+
critical_section::suspend_all(s);
886881
Some(s.state.compare_exchange(
887882
ThreadState::Attached as i32,
888883
ThreadState::Suspended as i32,
@@ -899,7 +894,7 @@ fn do_suspend(state: &PyGlobalState) {
899894
match park {
900895
None => {
901896
super::stw_trace(format_args!("suspend skip not-requested"));
902-
return;
897+
return false;
903898
}
904899
Some(Ok(_)) => {
905900
// Consumed this thread's stop request bit.
@@ -909,13 +904,13 @@ fn do_suspend(state: &PyGlobalState) {
909904
Some(ThreadState::Detached) => {
910905
// Leaving VM; caller will re-check on next entry.
911906
super::stw_trace(format_args!("suspend skip DETACHED"));
912-
return;
907+
return false;
913908
}
914909
Some(ThreadState::Suspended) => {
915910
// Already parked by another path.
916911
s.stop_requested.store(false, Ordering::Release);
917912
super::stw_trace(format_args!("suspend skip already-suspended"));
918-
return;
913+
return false;
919914
}
920915
Some(ThreadState::ShuttingDown) => {
921916
s.stop_requested.store(false, Ordering::Release);
@@ -924,7 +919,7 @@ fn do_suspend(state: &PyGlobalState) {
924919
}
925920
_ => {
926921
debug_assert!(false, "unexpected thread state in suspend: {state}");
927-
return;
922+
return false;
928923
}
929924
},
930925
}
@@ -969,7 +964,11 @@ fn do_suspend(state: &PyGlobalState) {
969964
}
970965
s.stop_requested.store(false, Ordering::Release);
971966
super::stw_trace(format_args!("suspend resume -> ATTACHED"));
967+
true
972968
});
969+
if reattached && let Some(s) = current_thread_slot() {
970+
critical_section::resume(&s);
971+
}
973972
}
974973

975974
#[cfg(feature = "threading")]
@@ -1157,14 +1156,19 @@ pub fn get_all_current_exceptions(vm: &VirtualMachine) -> Vec<(u64, Option<PyBas
11571156
/// Called at thread exit (or when leaving an interpreter permanently).
11581157
#[cfg(feature = "threading")]
11591158
pub fn cleanup_current_thread_frames(vm: &VirtualMachine) {
1160-
critical_section::assert_empty(vm.state.interpreter_id);
11611159
let thread_id = crate::stdlib::_thread::get_ident();
11621160
let interp_id = vm.state.interpreter_id;
11631161

11641162
// Prefer the slot registered for this interpreter; fall back to CURRENT.
11651163
let slot_for_interp = INTERP_THREAD_SLOTS.with(|slots| slots.borrow_mut().remove(&interp_id));
11661164
let current_slot = CURRENT_THREAD_SLOT.with(|slot| slot.borrow().as_ref().cloned());
11671165
let slot_to_clean = slot_for_interp.or(current_slot);
1166+
debug_assert!(
1167+
slot_to_clean
1168+
.as_ref()
1169+
.is_none_or(|slot| slot.critical_section.load(Ordering::Relaxed) == 0),
1170+
"thread cleanup cannot abandon a critical section"
1171+
);
11681172

11691173
// A dying thread should not remain logically ATTACHED while its
11701174
// thread-state slot is being removed.
@@ -1261,8 +1265,10 @@ pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) {
12611265
}
12621266
};
12631267
let top_iframe_ptr = get_current_frame() as usize;
1268+
// The surviving thread keeps its critical sections.
1269+
let critical_section =
1270+
current_thread_slot().map_or(0, |slot| slot.critical_section.load(Ordering::Relaxed));
12641271
let new_slot = Arc::new(ThreadSlot {
1265-
interpreter_id: vm.state.interpreter_id,
12661272
// The surviving child thread keeps executing its current frame chain.
12671273
// Only publish heavy frames for signal safety.
12681274
#[cfg(unix)]
@@ -1277,6 +1283,7 @@ pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) {
12771283
stop_requested: core::sync::atomic::AtomicBool::new(false),
12781284
thread: std::thread::current(),
12791285
qsbr: crate::object::qsbr::QSBR.register(),
1286+
critical_section: AtomicUsize::new(critical_section),
12801287
});
12811288
FRAME_SLOT_CACHE.with(|cache| {
12821289
#[cfg(unix)]
@@ -1309,7 +1316,14 @@ pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) {
13091316
#[cfg(feature = "threading")]
13101317
pub fn purge_other_interpreter_slots_after_fork(keep_id: i64) {
13111318
INTERP_THREAD_SLOTS.with(|slots| {
1312-
slots.borrow_mut().retain(|&id, _| id == keep_id);
1319+
slots.borrow_mut().retain(|&id, slot| {
1320+
let critical_section = slot.critical_section.load(Ordering::Relaxed);
1321+
if id != keep_id && critical_section != 0 {
1322+
FORKED_CRITICAL_SECTIONS
1323+
.with(|forked| forked.borrow_mut().insert(id, critical_section));
1324+
}
1325+
id == keep_id
1326+
});
13131327
});
13141328
}
13151329

0 commit comments

Comments
 (0)