Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .cspell.dict/cpython.txt
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ prec
preinitialized
pybuilddir
pycore
pyinner
pydecimal
Pyfunc
pylifecycle
Expand Down
2 changes: 1 addition & 1 deletion crates/vm/src/builtins/complex.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ impl PyPayload for PyComplex {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
COMPLEX_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
2 changes: 1 addition & 1 deletion crates/vm/src/builtins/dict.rs
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ impl PyPayload for PyDict {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
DICT_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
2 changes: 1 addition & 1 deletion crates/vm/src/builtins/float.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ impl PyPayload for PyFloat {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
FLOAT_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
2 changes: 1 addition & 1 deletion crates/vm/src/builtins/int.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ impl PyPayload for PyInt {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
INT_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
2 changes: 1 addition & 1 deletion crates/vm/src/builtins/list.rs
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ impl PyPayload for PyList {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
LIST_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
2 changes: 1 addition & 1 deletion crates/vm/src/builtins/range.rs
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ impl PyPayload for PyRange {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
RANGE_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
2 changes: 1 addition & 1 deletion crates/vm/src/builtins/slice.rs
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ impl PyPayload for PySlice {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
SLICE_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
89 changes: 86 additions & 3 deletions crates/vm/src/builtins/tuple.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ use crate::{
vm::VirtualMachine,
};
use alloc::fmt;
use core::cell::Cell;
use core::ptr::NonNull;

#[pyclass(module = false, name = "tuple", traverse = "manual")]
pub struct PyTuple<R = PyObjectRef> {
Expand All @@ -53,14 +55,95 @@ unsafe impl Traverse for PyTuple {
}
}

// No freelist for PyTuple: structseq types (stat_result, struct_time, etc.)
// are static subtypes sharing the same Rust payload, making type-safe reuse
// impractical without a type-pointer comparison at push time.
// spell-checker:ignore MAXSAVESIZE
/// Per-size freelist storage for tuples, matching tuples[PyTuple_MAXSAVESIZE].
/// Each bucket caches tuples of a specific element count (index = len - 1).
struct TupleFreeList {
buckets: [Vec<NonNull<PyObject>>; Self::MAX_SAVE_SIZE],
}

impl TupleFreeList {
/// Largest tuple size to cache on the freelist (sizes 1..=20).
const MAX_SAVE_SIZE: usize = 20;
const fn new() -> Self {
Self {
buckets: [const { Vec::new() }; Self::MAX_SAVE_SIZE],
}
}
}

impl Default for TupleFreeList {
fn default() -> Self {
Self::new()
}
}

impl Drop for TupleFreeList {
fn drop(&mut self) {
// Same safety pattern as FreeList<T>::drop — free raw allocation
// without running payload destructors to avoid TLS-after-destruction panics.
let layout = crate::object::pyinner_layout::<PyTuple>();
for bucket in &mut self.buckets {
for ptr in bucket.drain(..) {
unsafe {
alloc::alloc::dealloc(ptr.as_ptr() as *mut u8, layout);
}
}
}
}
}

thread_local! {
static TUPLE_FREELIST: Cell<TupleFreeList> = const { Cell::new(TupleFreeList::new()) };
}

impl PyPayload for PyTuple {
const MAX_FREELIST: usize = 2000;
const HAS_FREELIST: bool = true;

#[inline]
fn class(ctx: &Context) -> &'static Py<PyType> {
ctx.types.tuple_type
}

#[inline]
unsafe fn freelist_push(obj: *mut PyObject) -> bool {
let len = unsafe { &*(obj as *const crate::Py<PyTuple>) }.elements.len();
if len == 0 || len > TupleFreeList::MAX_SAVE_SIZE {
return false;
}
TUPLE_FREELIST
.try_with(|fl| {
let mut list = fl.take();
let bucket = &mut list.buckets[len - 1];
let stored = if bucket.len() < Self::MAX_FREELIST {
bucket.push(unsafe { NonNull::new_unchecked(obj) });
true
} else {
false
};
fl.set(list);
stored
})
.unwrap_or(false)
}

#[inline]
unsafe fn freelist_pop(payload: &Self) -> Option<NonNull<PyObject>> {
let len = payload.elements.len();
if len == 0 || len > TupleFreeList::MAX_SAVE_SIZE {
return None;
}
TUPLE_FREELIST
.try_with(|fl| {
let mut list = fl.take();
let result = list.buckets[len - 1].pop();
fl.set(list);
result
})
.ok()
.flatten()
}
}

pub trait IntoPyTuple {
Expand Down
47 changes: 31 additions & 16 deletions crates/vm/src/object/core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -188,27 +188,32 @@ pub(super) unsafe fn default_dealloc<T: PyPayload>(obj: *mut PyObject) {
);
}

// Extract child references before deallocation to break circular refs (tp_clear)
// Try to store in freelist for reuse BEFORE tp_clear, so that
// size-based freelists (e.g. PyTuple) can read the payload directly.
// Only exact base types (not heaptype or structseq subtypes) go into the freelist.
let typ = obj_ref.class();
let pushed = if T::HAS_FREELIST
&& typ.heaptype_ext.is_none()
&& core::ptr::eq(typ, T::class(crate::vm::Context::genesis()))
{
unsafe { T::freelist_push(obj) }
} else {
false
};

// Extract child references to break circular refs (tp_clear).
// This runs regardless of freelist push — the object's children must be released.
let mut edges = Vec::new();
if let Some(clear_fn) = vtable.clear {
unsafe { clear_fn(obj, &mut edges) };
}

// Try to store in freelist for reuse; otherwise deallocate.
// Only exact types (not heaptype subclasses) go into the freelist,
// because the pop site assumes the cached typ matches the base type.
let pushed = if T::HAS_FREELIST && obj_ref.class().heaptype_ext.is_none() {
unsafe { T::freelist_push(obj) }
} else {
false
};
if !pushed {
// Deallocate the object memory (handles ObjExt prefix if present)
unsafe { PyInner::dealloc(obj as *mut PyInner<T>) };
}

// Drop child references - may trigger recursive destruction.
// The object is already deallocated, so circular refs are broken.
drop(edges);

// Trashcan: decrement depth and process deferred objects at outermost level
Expand Down Expand Up @@ -1094,6 +1099,11 @@ impl<T: PyPayload + core::fmt::Debug> PyInner<T> {
}
}

/// Returns the allocation layout for `PyInner<T>`, for use in freelist Drop impls.
pub(crate) const fn pyinner_layout<T: PyPayload>() -> core::alloc::Layout {
core::alloc::Layout::new::<PyInner<T>>()
}

/// Thread-local freelist storage for reusing object allocations.
///
/// Wraps a `Vec<*mut PyObject>`. On thread teardown, `Drop` frees raw
Expand Down Expand Up @@ -2175,9 +2185,9 @@ impl<T: PyPayload + crate::object::MaybeTraverse + core::fmt::Debug> PyRef<T> {
let has_dict = dict.is_some();
let is_heaptype = typ.heaptype_ext.is_some();

// Try to reuse from freelist (exact type only, no dict, no heaptype)
// Try to reuse from freelist (no dict, no heaptype)
let cached = if !has_dict && !is_heaptype {
unsafe { T::freelist_pop() }
unsafe { T::freelist_pop(&payload) }
} else {
None
};
Expand All @@ -2189,11 +2199,16 @@ impl<T: PyPayload + crate::object::MaybeTraverse + core::fmt::Debug> PyRef<T> {
(*inner).gc_bits.store(0, Ordering::Relaxed);
core::ptr::drop_in_place(&mut (*inner).payload);
core::ptr::write(&mut (*inner).payload, payload);
// typ, vtable, slots are preserved; dict is None, weak_list was
// cleared by drop_slow_inner before freelist push
// Freelist only stores exact base types (push-side filter),
// but subtypes sharing the same Rust payload (e.g. structseq)
// may pop entries. Update typ if it differs.
let cached_typ: *const Py<PyType> = &*(*inner).typ;
if core::ptr::eq(cached_typ, &*typ) {
drop(typ);
} else {
let _old = (*inner).typ.swap(typ);
}
}
// Drop the caller's typ since the cached object already holds one
drop(typ);
unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) }
} else {
let inner = PyInner::new(payload, typ, dict);
Expand Down
7 changes: 4 additions & 3 deletions crates/vm/src/object/payload.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,11 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static {

/// Try to push a dead object onto this type's freelist for reuse.
/// Returns true if the object was stored (caller must NOT free the memory).
/// Called before tp_clear, so the payload is still intact.
///
/// # Safety
/// `obj` must be a valid pointer to a `PyInner<Self>` with refcount 0,
/// after `drop_slow_inner` and `tp_clear` have already run.
/// `obj` must be a valid pointer to a `PyInner<Self>` with refcount 0.
/// The payload is still initialized and can be read for bucket selection.
#[inline]
unsafe fn freelist_push(_obj: *mut PyObject) -> bool {
false
Expand All @@ -75,7 +76,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static {
/// whose payload is still initialized from a previous allocation. The caller
/// will drop and overwrite `payload` before reuse.
#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
None
}

Expand Down
Loading