Skip to content

Commit 030efca

Browse files
committed
Delegate copy and pickle, mirror metaclasses and publish dunders in pyo3
- copy.copy and copy.deepcopy of an embedded object run the embedded copy module. Pickling one stores the embedded pickle bytes behind pyo3.load_embedded. - The wrapper of an immutable embedded metaclass whose metaclass is type has type as its metaclass and copies IMMUTABLETYPE; mutable ones keep forwarding through pyo3.pyo3_type. - rustpython.object proxies publish the dunder methods the wrapped class defines, decided by the first defining class in the MRO and cached per class by version tag, and fill the sequence length slot. - Embedded warnings are shown through this interpreter's warnings module while a RustPython interpreter runs on the thread, and warning categories map to the matching built-in classes. Assisted-by: Grok Assisted-by: Claude
1 parent 5635ae7 commit 030efca

8 files changed

Lines changed: 1728 additions & 236 deletions

File tree

‎.cspell.dict/python-more.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ dllhandle
5151
docstring
5252
docstrings
5353
dunder
54+
dunders
5455
endianness
5556
endpos
5657
eventmask

‎crates/module_pyo3/src/cpython_init.rs‎

Lines changed: 146 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,22 +9,31 @@
99
//! Exceptions that C code cannot propagate are reported through the embedded
1010
//! interpreter's `sys.unraisablehook`. When this thread is running RustPython,
1111
//! that hook is handed to RustPython's `sys.unraisablehook`.
12+
//!
13+
//! Warnings go the same way. The embedded `warnings.showwarning` is handed to
14+
//! this interpreter's `warnings.showwarning` while a RustPython thread is
15+
//! running, so a recorder installed here observes them. The embedded filters
16+
//! still decide whether `showwarning` is called. If the function here raises,
17+
//! that exception propagates to the embedded caller. With no RustPython thread,
18+
//! or if delivery is already in progress, the embedded function runs.
1219
20+
use core::cell::Cell;
1321
use core::ffi::CStr;
14-
use std::sync::Once;
22+
use std::sync::{Once, OnceLock};
1523

1624
use pyo3::exceptions::PyRuntimeError;
1725
use pyo3::prelude::*;
18-
use pyo3::types::{PyAnyMethods, PyCFunction, PyDict, PyTuple};
26+
use pyo3::types::{PyAnyMethods, PyCFunction, PyDict, PyTuple, PyType as CPythonType};
1927
use rustpython_vm::{
2028
AsObject, PyObjectRef, PyPayload, PyResult, VirtualMachine,
21-
builtins::{PyTuple as RustPyTuple, PyType},
29+
builtins::{PyStr, PyTuple as RustPyTuple, PyType},
30+
function::{FuncArgs, KwArgs},
2231
types::{AtomicPyTypeFlags, PyTypeFlags, PyTypeSlots},
2332
};
2433

2534
use crate::_pyo3::module_state;
2635
use crate::convert::to_rustpython;
27-
use crate::errors::cpython_exc_to_rustpython;
36+
use crate::errors::{cpython_exc_to_rustpython, matching_builtin_type, rustpython_exc_to_cpython};
2837
use crate::gil::run_rustpython;
2938

3039
/// Initialize CPython on first use and install the unraisable forwarder.
@@ -37,6 +46,7 @@ pub(crate) fn ensure_cpython_initialized() {
3746
static INIT: Once = Once::new();
3847
INIT.call_once(initialize_cpython);
3948
install_unraisable_hook();
49+
install_warning_hook();
4050
}
4151

4252
fn initialize_cpython() {
@@ -91,6 +101,138 @@ fn set_unraisable_hook(py: Python<'_>) -> pyo3::PyResult<()> {
91101
py.import("sys")?.setattr("unraisablehook", hook)
92102
}
93103

104+
static ORIGINAL_SHOWWARNING: OnceLock<Py<PyAny>> = OnceLock::new();
105+
106+
std::thread_local! {
107+
static IN_SHOWWARNING: Cell<bool> = const { Cell::new(false) };
108+
}
109+
110+
/// Set while a warning is being delivered, so a nested report uses the
111+
/// embedded function instead of calling back into the same delivery.
112+
struct ShowwarningGuard;
113+
114+
impl ShowwarningGuard {
115+
fn enter() -> Option<Self> {
116+
IN_SHOWWARNING.with(|flag| {
117+
if flag.get() {
118+
None
119+
} else {
120+
flag.set(true);
121+
Some(Self)
122+
}
123+
})
124+
}
125+
}
126+
127+
impl Drop for ShowwarningGuard {
128+
fn drop(&mut self) {
129+
IN_SHOWWARNING.with(|flag| flag.set(false));
130+
}
131+
}
132+
133+
fn install_warning_hook() {
134+
static HOOK: Once = Once::new();
135+
HOOK.call_once(|| {
136+
Python::attach(|py| {
137+
if let Err(err) = set_warning_hook(py) {
138+
panic!("failed to install the embedded warning hook: {err}");
139+
}
140+
});
141+
});
142+
}
143+
144+
fn set_warning_hook(py: Python<'_>) -> pyo3::PyResult<()> {
145+
let warnings = py.import("warnings")?;
146+
let original = warnings.getattr("showwarning")?;
147+
let _ = ORIGINAL_SHOWWARNING.set(original.unbind());
148+
let hook = PyCFunction::new_closure(py, Some(c"showwarning"), None, showwarning_hook)?;
149+
warnings.setattr("showwarning", hook)
150+
}
151+
152+
fn showwarning_hook(
153+
args: &Bound<'_, PyTuple>,
154+
kwargs: Option<&Bound<'_, PyDict>>,
155+
) -> pyo3::PyResult<()> {
156+
let Some(_guard) = ShowwarningGuard::enter() else {
157+
return call_original_showwarning(args, kwargs);
158+
};
159+
let py = args.py();
160+
let forwarded = rustpython_vm::vm::thread::try_with_current_vm(|vm| {
161+
vm.attach_for_callback(|| forward_showwarning(py, args, kwargs, vm))
162+
});
163+
match forwarded {
164+
Some(result) => result,
165+
None => call_original_showwarning(args, kwargs),
166+
}
167+
}
168+
169+
fn forward_showwarning(
170+
py: Python<'_>,
171+
args: &Bound<'_, PyTuple>,
172+
kwargs: Option<&Bound<'_, PyDict>>,
173+
vm: &VirtualMachine,
174+
) -> pyo3::PyResult<()> {
175+
let call_args =
176+
showwarning_args(args, kwargs, vm).map_err(|err| rustpython_exc_to_cpython(py, err, vm))?;
177+
run_rustpython(py, || {
178+
let warnings = vm.import("warnings", 0)?;
179+
let show = warnings.get_attr("showwarning", vm)?;
180+
show.call(call_args, vm)?;
181+
Ok(())
182+
})
183+
.map_err(|err| rustpython_exc_to_cpython(py, err, vm))
184+
}
185+
186+
fn showwarning_args(
187+
args: &Bound<'_, PyTuple>,
188+
kwargs: Option<&Bound<'_, PyDict>>,
189+
vm: &VirtualMachine,
190+
) -> PyResult<FuncArgs> {
191+
let positional = args
192+
.iter()
193+
.enumerate()
194+
.map(|(index, arg)| showwarning_value(index == 1, &arg, vm))
195+
.collect::<PyResult<Vec<_>>>()?;
196+
let kwargs = kwargs
197+
.into_iter()
198+
.flatten()
199+
.map(|(key, value)| {
200+
let key = to_rustpython(&key, vm)?;
201+
let key = key
202+
.downcast_ref::<PyStr>()
203+
.ok_or_else(|| vm.new_type_error("keywords must be strings"))?;
204+
let is_category = key.to_str() == Some("category");
205+
let value = showwarning_value(is_category, &value, vm)?;
206+
Ok((key.as_wtf8().to_owned(), value))
207+
})
208+
.collect::<PyResult<KwArgs>>()?;
209+
Ok(FuncArgs::new(positional, kwargs))
210+
}
211+
212+
/// `category` is a warning class. Hand this interpreter's class across so a
213+
/// filter or recorder here sees the class it already imported.
214+
fn showwarning_value(category: bool, value: &Bound<'_, PyAny>, vm: &VirtualMachine) -> PyResult {
215+
if category
216+
&& let Ok(ty) = value.cast::<CPythonType>()
217+
&& let Some(mapped) = matching_builtin_type(ty, vm)
218+
&& mapped.fast_issubclass(vm.ctx.exceptions.warning)
219+
{
220+
return Ok(mapped.into());
221+
}
222+
to_rustpython(value, vm)
223+
}
224+
225+
fn call_original_showwarning(
226+
args: &Bound<'_, PyTuple>,
227+
kwargs: Option<&Bound<'_, PyDict>>,
228+
) -> pyo3::PyResult<()> {
229+
let Some(original) = ORIGINAL_SHOWWARNING.get() else {
230+
return Ok(());
231+
};
232+
original.bind(args.py()).call(args, kwargs)?;
233+
Ok(())
234+
}
235+
94236
/// What happened to one unraisable exception.
95237
enum Forwarded {
96238
/// RustPython's hook ran and returned.

‎crates/module_pyo3/src/errors.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,20 @@ pub(crate) fn rustpython_exc_to_cpython(
176176
PyErr::from_value(instance)
177177
}
178178

179+
/// The built-in class on this side that corresponds to `ty`, when `ty`'s
180+
/// MRO contains a class defined in `builtins`.
181+
pub(crate) fn matching_builtin_type(
182+
ty: &Bound<'_, CPythonType>,
183+
vm: &VirtualMachine,
184+
) -> Option<rustpython_vm::PyRef<PyType>> {
185+
let (name, _) = cpython_mro_builtin(ty)?;
186+
vm.builtins
187+
.get_attr(&vm.ctx.new_str(name.as_str()), vm)
188+
.ok()?
189+
.downcast::<PyType>()
190+
.ok()
191+
}
192+
179193
/// The first class in `ty.__mro__` defined in `builtins`, and whether that is
180194
/// `ty` itself.
181195
fn cpython_mro_builtin(ty: &Bound<'_, CPythonType>) -> Option<(String, bool)> {

‎crates/module_pyo3/src/lib.rs‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,26 @@
1616
//! slots come from the methods the class actually defines. Classes that
1717
//! cannot be laid out that way stay generic proxies, and `proxy.__class__`
1818
//! is still the wrapped class. A wrapper's metaclass is the wrapper of the
19-
//! CPython metaclass, or `pyo3.pyo3_type` when that metaclass is `type`.
19+
//! CPython metaclass. When that metaclass is `type`, the class is itself a
20+
//! metaclass, and the class is immutable, the wrapper's metaclass is `type`
21+
//! too and the immutable-type flag is copied from the wrapped class. A
22+
//! mutable metaclass keeps `pyo3.pyo3_type`, so reads and writes on the
23+
//! wrapper reach the wrapped class. Every other class whose metaclass is
24+
//! `type` uses `pyo3.pyo3_type`.
2025
//! See [`convert`] for what crosses by value and what by reference,
2126
//! [`pyo3_type`] for classes, and [`errors`] for exceptions.
2227
//!
28+
//! `copy.copy` and `copy.deepcopy` of an embedded object call the embedded
29+
//! `copy` module. `pickle` stores `pyo3.load_embedded` and the bytes the
30+
//! embedded `pickle` produced; loading unpickles there and converts the
31+
//! result back. See [`pyo3_ref`].
32+
//!
33+
//! A RustPython object handed across publishes the non-slot dunder methods
34+
//! its class defines. The first definition in the MRO wins, and a name only
35+
//! `object` defines stays unpublished. Its sequence length is its `__len__`.
36+
//! See [`rustpython_object`]. Embedded `warnings.showwarning` is delivered to
37+
//! this interpreter while a thread is running it. See [`cpython_init`].
38+
//!
2339
//! `pyo3.wraps` instead runs a whole function in CPython:
2440
//!
2541
//! ```python
@@ -95,6 +111,9 @@ mod _pyo3 {
95111
types: PyMutex<HashMap<usize, (pyo3::Py<pyo3::PyAny>, PyTypeRef)>>,
96112
/// Tuple subclass passed to `sys.unraisablehook`. One per interpreter.
97113
unraisable_args: PyMutex<Option<PyTypeRef>>,
114+
/// `mro` of metaclass wrappers whose own metaclass is `type`.
115+
/// One descriptor, shared; it binds the class it is read from.
116+
published_mro_method: PyMutex<Option<PyObjectRef>>,
98117
}
99118

100119
#[pyclass(flags(DISALLOW_INSTANTIATION))]
@@ -139,6 +158,20 @@ mod _pyo3 {
139158
*slot = Some(ty.clone());
140159
ty
141160
}
161+
162+
pub(crate) fn cached_published_mro_method(&self) -> Option<PyObjectRef> {
163+
self.published_mro_method.lock().clone()
164+
}
165+
166+
/// Remember `method` unless another thread stored one first.
167+
pub(crate) fn store_published_mro_method(&self, method: PyObjectRef) -> PyObjectRef {
168+
let mut slot = self.published_mro_method.lock();
169+
if let Some(existing) = slot.clone() {
170+
return existing;
171+
}
172+
*slot = Some(method.clone());
173+
method
174+
}
142175
}
143176

144177
pub(crate) fn module_state(vm: &VirtualMachine) -> PyResult<PyRef<Pyo3State>> {
@@ -167,6 +200,22 @@ mod _pyo3 {
167200
import_module_impl(name.as_str(), vm)
168201
}
169202

203+
/// Unpickle `data` with the embedded interpreter's `pickle` and convert the result.
204+
///
205+
/// Embedded objects reduce to this function plus the bytes that
206+
/// interpreter's `pickle.dumps` produced. Protocol 5 buffers are included
207+
/// in those bytes. The failure of an object that cannot be pickled is the
208+
/// embedded failure, converted.
209+
#[pyfunction]
210+
fn load_embedded(data: PyObjectRef, vm: &VirtualMachine) -> PyResult {
211+
with_gil(vm, |py| {
212+
let pickle = run_cpython(vm, || py.import("pickle"))?;
213+
let data = to_cpython(py, &data, vm)?;
214+
let result = run_cpython(vm, || pickle.call_method1("loads", (data,)))?;
215+
to_rustpython(&result, vm)
216+
})
217+
}
218+
170219
pub(crate) fn import_module_impl(name: &str, vm: &VirtualMachine) -> PyResult {
171220
with_gil(vm, |py| {
172221
let module = run_cpython(vm, || py.import(name))?;

0 commit comments

Comments
 (0)