Add minimal capi lifecycle support

bschoenmaeckers · bschoenmaeckers · commit 0a01cb4fec4c · 2026-04-21T14:34:29.000+02:00
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/capi/.cargo/config.toml b/crates/capi/.cargo/config.toml
@@ -0,0 +1,3 @@
+[env]
+PYO3_CONFIG_FILE = { value = "pyo3-rustpython.config", relative = true }
+PYO3_NO_PYTHON = { value = "1" }
diff --git a/crates/capi/Cargo.toml b/crates/capi/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "rustpython-capi"
+description = "Minimal CPython C-API compatibility exports for RustPython"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+license.workspace = true
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+rustpython-vm = { workspace = true, features = ["threading"] }
+
+[dev-dependencies]
+pyo3 = { version = "0.28", features = ["auto-initialize", "abi3"] }
+
+[lints]
+workspace = true
diff --git a/crates/capi/pyo3-rustpython.config b/crates/capi/pyo3-rustpython.config
@@ -0,0 +1,5 @@
+implementation=CPython
+version=3.14
+shared=true
+abi3=true
+suppress_build_script_link_lines=true
diff --git a/crates/capi/src/lib.rs b/crates/capi/src/lib.rs
@@ -0,0 +1,7 @@
+pub use rustpython_vm::PyObject;
+
+extern crate alloc;
+
+pub mod pylifecycle;
+pub mod pystate;
+pub mod refcount;
diff --git a/crates/capi/src/pylifecycle.rs b/crates/capi/src/pylifecycle.rs
@@ -0,0 +1,71 @@
+use crate::pystate::attach_vm_to_thread;
+use core::ffi::c_int;
+use rustpython_vm::Interpreter;
+use rustpython_vm::vm::thread::ThreadedVirtualMachine;
+use std::sync::{Once, OnceLock, mpsc};
+
+static VM_REQUEST_TX: OnceLock<mpsc::Sender<mpsc::SyncSender<ThreadedVirtualMachine>>> =
+    OnceLock::new();
+pub(crate) static INITIALIZED: Once = Once::new();
+
+/// Request a vm from the main interpreter
+pub(crate) fn request_vm_from_interpreter() -> ThreadedVirtualMachine {
+    let tx = VM_REQUEST_TX
+        .get()
+        .expect("VM request channel not initialized");
+    let (response_tx, response_rx) = mpsc::sync_channel(1);
+    tx.send(response_tx).expect("Failed to send VM request");
+    response_rx.recv().expect("Failed to receive VM response")
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_IsInitialized() -> c_int {
+    INITIALIZED.is_completed() as _
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_Initialize() {
+    Py_InitializeEx(0);
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_InitializeEx(_initsigs: c_int) {
+    if INITIALIZED.is_completed() {
+        panic!("Initialize called multiple times");
+    }
+
+    INITIALIZED.call_once(|| {
+        let (tx, rx) = mpsc::channel();
+        VM_REQUEST_TX
+            .set(tx)
+            .expect("VM request channel was already initialized");
+
+        std::thread::spawn(move || {
+            let interp = Interpreter::with_init(Default::default(), |_vm| {});
+            interp.enter(|vm| {
+                while let Ok(request) = rx.recv() {
+                    request
+                        .send(vm.new_thread())
+                        .expect("Failed to send VM response");
+                }
+            })
+        });
+    });
+
+    attach_vm_to_thread();
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_Finalize() {
+    let _ = Py_FinalizeEx();
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_FinalizeEx() -> c_int {
+    0
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_IsFinalizing() -> c_int {
+    0
+}
diff --git a/crates/capi/src/pystate.rs b/crates/capi/src/pystate.rs
@@ -0,0 +1,66 @@
+use crate::pylifecycle::request_vm_from_interpreter;
+use core::cell::RefCell;
+use core::ffi::c_int;
+use core::ptr;
+use rustpython_vm::vm::thread::ThreadedVirtualMachine;
+
+thread_local! {
+    static VM: RefCell<Option<ThreadedVirtualMachine>> = const { RefCell::new(None) };
+}
+
+#[allow(non_camel_case_types)]
+type PyGILState_STATE = c_int;
+
+#[repr(C)]
+pub struct PyThreadState {
+    _interp: *mut core::ffi::c_void,
+}
+
+pub(crate) fn attach_vm_to_thread() {
+    VM.with(|vm| {
+        vm.borrow_mut()
+            .get_or_insert_with(request_vm_from_interpreter);
+    });
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn PyGILState_Ensure() -> PyGILState_STATE {
+    attach_vm_to_thread();
+
+    0
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn PyGILState_Release(_state: PyGILState_STATE) {}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn PyEval_SaveThread() -> *mut PyThreadState {
+    ptr::null_mut()
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::pystate::VM;
+    use pyo3::prelude::*;
+
+    #[test]
+    fn test_new_thread() {
+        Python::attach(|_py| {
+            assert!(
+                VM.with(|vm| vm.borrow().is_some()),
+                "This thread did not have a vm attached"
+            );
+
+            std::thread::spawn(move || {
+                Python::attach(|_py| {
+                    assert!(
+                        VM.with(|vm| vm.borrow().is_some()),
+                        "This thread did not have a vm attached"
+                    );
+                });
+            })
+            .join()
+            .unwrap();
+        })
+    }
+}
diff --git a/crates/capi/src/refcount.rs b/crates/capi/src/refcount.rs
@@ -0,0 +1,17 @@
+use crate::PyObject;
+use core::ptr::NonNull;
+use rustpython_vm::PyObjectRef;
+
+#[unsafe(no_mangle)]
+#[allow(clippy::not_unsafe_ptr_arg_deref)]
+pub extern "C" fn _Py_DecRef(op: *mut PyObject) {
+    // By dropping PyObjectRef, we will decrement the reference count.
+    unsafe { PyObjectRef::from_raw(NonNull::new_unchecked(op)) };
+}
+
+#[unsafe(no_mangle)]
+#[allow(clippy::not_unsafe_ptr_arg_deref)]
+pub extern "C" fn _Py_IncRef(op: *mut PyObject) {
+    // Don't drop the owned value, as we just want to increment the refcount.
+    core::mem::forget(unsafe { (*op).to_owned() });
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+[env]`
	`2`	`+PYO3_CONFIG_FILE = { value = "pyo3-rustpython.config", relative = true }`
	`3`	`+PYO3_NO_PYTHON = { value = "1" }`