Skip to content

Align text and bytes argument and buffer controls #9698

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 4 commits into from
Nov 6, 2026
Merged

Align text and bytes argument and buffer controls #9698

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
24a4a64
Align text and bytes argument controls with CPython
youknowdot Nov 4, 2026
b72e268
Keep bytes conversion diagnostics WTF-8 compatible
youknowdot Nov 4, 2026
bd9380a
Merge main into text and bytes controls
youknowdot Nov 4, 2026
f539764
Share byte translation tables without buffer ownership
youknowdot Nov 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 49 additions & 65 deletions crates/vm/src/anystr.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,35 +99,43 @@ impl StartsEndsWithArgs {
(&self.affix, range)
}

pub(crate) fn get_value(self, len: usize) -> (PyObjectRef, Option<Range<usize>>) {
let range = if self.start.is_some() || self.end.is_some() {
Some(adjust_indices(
self.start.as_ref().map(AsRef::as_ref),
self.end.as_ref().map(AsRef::as_ref),
len,
))
} else {
None
};
(self.affix, range)
}

#[inline]
pub(crate) fn prepare<S, F>(self, s: &S, len: usize, substr: F) -> Option<(PyObjectRef, &S)>
pub(crate) fn matches<'a, T, F>(
&'a self,
len: usize,
func_name: &str,
py_type_name: &str,
predicate: F,
vm: &VirtualMachine,
) -> PyResult<bool>
where
S: ?Sized + AnyStr,
F: Fn(&S, Range<usize>) -> &S,
T: TryFromBorrowedObject<'a>,
F: Fn(T, Option<Range<usize>>) -> bool,
{
let (affix, range) = self.get_value(len);
let substr = if let Some(range) = range {
if !range.is_normal() {
return None;
}
substr(s, range)
} else {
s
};
Some((affix, substr))
let (affix, range) = self.parts(len);
// Validate each affix even when the requested range cannot match. A
// tuple is still short-circuited after its first successful match.
single_or_tuple_any(
affix,
&|item| Ok(predicate(item, range.clone())),
&|obj, in_tuple| {
let name = obj.class().slot_name();
if py_type_name == "str" {
let name = &name[..name.floor_char_boundary(100)];
Some(if in_tuple {
format!("tuple for {func_name} must only contain str, not {name}")
} else {
format!("{func_name} first arg must be str or a tuple of str, not {name}")
})
} else if in_tuple {
None
} else {
Some(format!(
"{func_name} first arg must be bytes or a tuple of bytes, not {name}"
))
}
},
vm,
)
}
}

Expand Down Expand Up @@ -268,35 +276,6 @@ pub(crate) trait AnyStr {
where
F: Fn(&Self) -> PyObjectRef;

#[inline]
fn py_starts_ends_with<'a, T, F>(
&self,
affix: &'a PyObject,
func_name: &str,
py_type_name: &str,
func: F,
vm: &VirtualMachine,
) -> PyResult<bool>
where
T: TryFromBorrowedObject<'a>,
F: Fn(&Self, T) -> bool,
{
single_or_tuple_any(
affix,
&|s: T| Ok(func(self, s)),
&|o| {
format!(
"{} first arg must be {} or a tuple of {}, not {}",
func_name,
py_type_name,
py_type_name,
o.class(),
)
},
vm,
)
}

#[inline]
fn py_strip<'a, S, FC, FD>(
&'a self,
Expand Down Expand Up @@ -530,13 +509,24 @@ pub(crate) fn single_or_tuple_any<'a, T, F, M>(
where
T: TryFromBorrowedObject<'a>,
F: Fn(T) -> PyResult<bool>,
M: Fn(&PyObject) -> String,
M: Fn(&PyObject, bool) -> Option<String>,
{
let convert = |item: &'a PyObject, in_tuple| {
item.try_to_value::<T>(vm).map_err(|exc| {
if exc.fast_isinstance(vm.ctx.exceptions.type_error)
&& let Some(message) = message(item, in_tuple)
{
vm.new_type_error(message)
} else {
exc
}
})
};
// _Py_bytes_tailmatch: a tuple is taken apart before anything is converted, and
// each item is converted on its own terms, so a tuple of tuples is not an affix.
if let Some(tuple) = obj.downcast_ref::<PyTuple>() {
for item in tuple {
if (predicate)(item.try_to_value::<T>(vm)?)? {
if (predicate)(convert(item, true)?)? {
return Ok(true);
}
}
Expand All @@ -545,13 +535,7 @@ where

// Only the argument simply being the wrong kind of object is reported as such;
// whatever the conversion itself raised belongs to the caller.
let single = obj.try_to_value::<T>(vm).map_err(|exc| {
if exc.fast_isinstance(vm.ctx.exceptions.type_error) {
vm.new_type_error((message)(obj))
} else {
exc
}
})?;
let single = convert(obj, false)?;
(predicate)(single)
}

Expand Down
87 changes: 47 additions & 40 deletions crates/vm/src/builtins/bytes.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,13 @@ use super::{
use crate::common::lock::LazyLock;
use crate::{
AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult,
TryFromBorrowedObject, VirtualMachine,
anystr::{self, AnyStr},
atomic_func,
TryFromBorrowedObject, VirtualMachine, anystr, atomic_func,
byte::bytes_from_object,
bytes_inner::{
ByteInnerFindOptions, ByteInnerHexOptions, ByteInnerNewOptions, ByteInnerPaddingOptions,
ByteInnerReplaceOptions, ByteInnerSplitOptions, ByteInnerStripOptions, ByteInnerSub,
ByteInnerTranslateOptions, BytesJoin, DecodeArgs, PyBytesInner, bytes_decode,
ByteInnerBuffer, ByteInnerFindOptions, ByteInnerHexOptions, ByteInnerNewOptions,
ByteInnerPaddingOptions, ByteInnerReplaceOptions, ByteInnerSplitOptions,
ByteInnerStripOptions, ByteInnerSub, ByteInnerTranslateOptions, BytesJoin, DecodeArgs,
PyBytesInner, bytes_decode,
},
class::{PyClassDef, PyClassImpl},
common::{hash::PyHash, lock::PyMutex},
Expand All @@ -31,6 +30,7 @@ use crate::{
use bstr::ByteSlice;
use core::{mem::size_of, ops::Deref};
use memchr::memchr;
use rustpython_common::wtf8::wtf8_concat;

#[pyclass(module = false, name = "bytes")]
#[derive(Clone, Debug)]
Expand Down Expand Up @@ -130,15 +130,23 @@ impl Constructor for PyBytes {
&& let Some(bytes_method) = vm.get_method(obj.clone(), identifier!(vm, __bytes__))
{
let bytes = bytes_method?.call((), vm)?;
// If exact bytes type and __bytes__ returns bytes, use it directly
if cls.is(vm.ctx.types.bytes_type)
&& let Ok(b) = bytes.clone().downcast::<Self>()
{
return Ok(b.into());
let bytes = bytes.downcast::<Self>().map_err(|result| {
match (
obj.class().fully_qualified_name(vm),
result.class().fully_qualified_name(vm),
) {
(Ok(source), Ok(result)) => vm.new_type_error(wtf8_concat!(
source,
".__bytes__() must return a bytes, not ",
result
)),
(Err(error), _) | (_, Err(error)) => error,
}
})?;
if cls.is(vm.ctx.types.bytes_type) {
return Ok(bytes.into());
}
// Otherwise convert to Vec<u8>
let inner = PyBytesInner::try_from_borrowed_object(vm, &bytes)?;
let payload = Self::py_new(&cls, inner.elements, vm)?;
let payload = Self::py_new(&cls, bytes.as_bytes().to_vec(), vm)?;
return payload.into_ref_with_type(vm, cls).map(Into::into);
}

Expand Down Expand Up @@ -423,16 +431,15 @@ impl Py<PyBytes> {

#[pymethod]
fn endswith(&self, options: anystr::StartsEndsWithArgs, vm: &VirtualMachine) -> PyResult<bool> {
let (affix, substr) =
match options.prepare(self.as_bytes(), self.len(), |s, r| s.get_bytes(r)) {
Some(x) => x,
None => return Ok(false),
};
substr.py_starts_ends_with(
&affix,
options.matches(
self.len(),
"endswith",
"bytes",
|s, x: PyBytesInner| s.ends_with(x.as_bytes()),
|affix: ByteInnerBuffer<'_>, range| {
self.as_bytes()
.get(range.unwrap_or(0..self.len()))
.is_some_and(|text| affix.with_ref(|affix| text.ends_with(affix)))
},
vm,
)
}
Expand All @@ -443,16 +450,15 @@ impl Py<PyBytes> {
options: anystr::StartsEndsWithArgs,
vm: &VirtualMachine,
) -> PyResult<bool> {
let (affix, substr) =
match options.prepare(self.as_bytes(), self.len(), |s, r| s.get_bytes(r)) {
Some(x) => x,
None => return Ok(false),
};
substr.py_starts_ends_with(
&affix,
options.matches(
self.len(),
"startswith",
"bytes",
|s, x: PyBytesInner| s.starts_with(x.as_bytes()),
|affix: ByteInnerBuffer<'_>, range| {
self.as_bytes()
.get(range.unwrap_or(0..self.len()))
.is_some_and(|text| affix.with_ref(|affix| text.starts_with(affix)))
},
vm,
)
}
Expand Down Expand Up @@ -481,15 +487,6 @@ impl Py<PyBytes> {
index.ok_or_else(|| vm.new_value_error("substring not found"))
}

#[pymethod]
fn translate(
&self,
options: ByteInnerTranslateOptions,
vm: &VirtualMachine,
) -> PyResult<PyBytes> {
Ok(self.inner.translate(options, vm)?.into())
}

#[pymethod]
fn split(
&self,
Expand Down Expand Up @@ -605,9 +602,19 @@ impl PyRef<PyBytes> {
})
}

#[pymethod]
fn translate(self, options: ByteInnerTranslateOptions, vm: &VirtualMachine) -> PyResult<Self> {
let result = self.inner.translate(options, vm)?;
if self.class().is(vm.ctx.types.bytes_type) && result == self.as_bytes() {
Ok(self)
} else {
Ok(vm.ctx.new_bytes(result))
}
}

#[pymethod]
fn __bytes__(self, vm: &VirtualMachine) -> Self {
if self.is(vm.ctx.types.bytes_type) {
if self.class().is(vm.ctx.types.bytes_type) {
self
} else {
PyBytes::from(self.payload.inner.clone()).into_ref(&vm.ctx)
Expand Down
30 changes: 12 additions & 18 deletions crates/vm/src/builtins/str.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1125,17 +1125,14 @@ impl Py<PyStr> {

#[pymethod]
fn endswith(&self, options: anystr::StartsEndsWithArgs, vm: &VirtualMachine) -> PyResult<bool> {
let (affix, substr) = match options.prepare(self.as_wtf8(), self.len(), |s, r| {
&s[self.data.char_range_to_bytes(r)]
}) {
Some(x) => x,
None => return Ok(false),
};
substr.py_starts_ends_with(
&affix,
options.matches(
self.len(),
"endswith",
"str",
|s, x: &Self| s.ends_with(x.as_wtf8()),
|affix: &Self, range| {
self.char_range_bytes(range.unwrap_or_else(|| 0..self.len()))
.is_some_and(|(_, text)| text.ends_with(affix.as_wtf8()))
},
vm,
)
}
Expand All @@ -1146,17 +1143,14 @@ impl Py<PyStr> {
options: anystr::StartsEndsWithArgs,
vm: &VirtualMachine,
) -> PyResult<bool> {
let (affix, substr) = match options.prepare(self.as_wtf8(), self.len(), |s, r| {
&s[self.data.char_range_to_bytes(r)]
}) {
Some(x) => x,
None => return Ok(false),
};
substr.py_starts_ends_with(
&affix,
options.matches(
self.len(),
"startswith",
"str",
|s, x: &Self| s.starts_with(x.as_wtf8()),
|affix: &Self, range| {
self.char_range_bytes(range.unwrap_or_else(|| 0..self.len()))
.is_some_and(|(_, text)| text.starts_with(affix.as_wtf8()))
},
vm,
)
}
Expand Down
Loading
Loading