Skip to content

Commit 8ea9a44

Browse files
youknowoneyouknowone
committed
Compose native text signatures at compile time from FromArgs
FromArgs gains a PARAMS const describing its Python parameters. The derive builds the whole internal doc, `name(sig)\n--\n\ndoc`, in a const context from those params, and `__text_signature__` / `__doc__` parse it from the doc. The macro-time string path and the runtime sig_parts composition are removed. Signatures follow clinic conventions: `$module` for module functions, Rust literal defaults rendered as Python reprs, other defaults as `<unrepresentable>`, and a new `#[pyarg(py_default = "...")]` for the few fields whose Rust default differs from the Python one. The `#[pyarg]` keys are documented on the FromArgs derive. Module functions keep their module object and return it from `__self__`. PyNativeFunction now traverses its references. os.stat, os.mkdir, os.listdir, os.open, dict.fromkeys, round, sorted, int.to_bytes and bytes.hex take arguments matching their signatures. Assisted-by: Grok:grok-4.7 Assisted-by: Claude:claude-opus-5-5
1 parent afad92b commit 8ea9a44

22 files changed

Lines changed: 811 additions & 183 deletions

File tree

‎.cspell.dict/rust-more.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ powf
6060
powi
6161
prepended
6262
punct
63+
qself
6364
replacen
6465
retag
6566
rmatch

‎Lib/test/test_inspect/test_inspect.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1363,7 +1363,6 @@ def test(): pass
13631363
spec = inspect.getfullargspec(test)
13641364
self.assertEqual(test.__annotations__, spec.annotations)
13651365

1366-
@unittest.expectedFailure # TODO: RUSTPYTHON; os.stat signature is generated as (path, dir_fd, follow_symlinks, /)
13671366
@unittest.skipIf(MISSING_C_DOCSTRINGS,
13681367
"Signature information for builtins requires docstrings")
13691368
def test_getfullargspec_builtin_methods(self):

‎Lib/test/test_pydoc/test_pydoc.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1573,7 +1573,6 @@ def test_bound_builtin_method(self):
15731573
"dump(obj, /) method of _pickle.Pickler instance")
15741574

15751575
# this should *never* include self!
1576-
@unittest.expectedFailure # TODO: RUSTPYTHON; os.stat signature is generated as (path, dir_fd, follow_symlinks, /)
15771576
@requires_docstrings
15781577
def test_module_level_callable(self):
15791578
self.assertEqual(self._get_summary_line(os.stat),

‎crates/derive-impl/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ rustpython-compiler-core = { workspace = true }
1414
rustpython-doc = { workspace = true }
1515

1616
itertools = { workspace = true }
17-
syn = { workspace = true, features = ["full", "extra-traits"] }
17+
syn = { workspace = true, features = ["full", "extra-traits", "visit", "visit-mut"] }
1818

1919
proc-macro2 = { workspace = true }
2020
quote = { workspace = true }

‎crates/derive-impl/src/from_args.rs‎

Lines changed: 126 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use proc_macro2::TokenStream;
22
use quote::{ToTokens, quote};
33
use syn::ext::IdentExt;
44
use syn::meta::ParseNestedMeta;
5-
use syn::{Attribute, Data, DeriveInput, Expr, Field, Ident, Result, Token, parse_quote};
5+
use syn::{Attribute, Data, DeriveInput, Expr, Field, Ident, Lit, Result, Token, parse_quote};
66

77
/// The kind of the python parameter, this corresponds to the value of Parameter.kind
88
/// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind)
@@ -37,6 +37,7 @@ struct ArgAttribute {
3737
name: Option<String>,
3838
kind: ParameterKind,
3939
default: Option<DefaultValue>,
40+
py_default: Option<String>,
4041
error_msg: Option<String>,
4142
}
4243

@@ -64,6 +65,7 @@ impl ArgAttribute {
6465
name: None,
6566
kind,
6667
default: None,
68+
py_default: None,
6769
error_msg: None,
6870
});
6971
return Ok(());
@@ -91,6 +93,13 @@ impl ArgAttribute {
9193
if self.default.is_none() {
9294
self.default = Some(None);
9395
}
96+
// Text-only signature default. See the `FromArgs` derive docs.
97+
} else if meta.path.is_ident("py_default") {
98+
if self.py_default.is_some() {
99+
return Err(meta.error("py_default already set"));
100+
}
101+
let val = meta.value()?.parse::<syn::LitStr>()?;
102+
self.py_default = Some(val.value());
94103
} else if meta.path.is_ident("name") {
95104
if self.name.is_some() {
96105
return Err(meta.error("already have a name"));
@@ -212,6 +221,96 @@ fn generate_field((i, field): (usize, &Field)) -> Result<TokenStream> {
212221
Ok(file_output)
213222
}
214223

224+
fn is_phantom(field: &Field) -> bool {
225+
field
226+
.ident
227+
.as_ref()
228+
.is_some_and(|ident| ident.unraw().to_string().starts_with("_phantom"))
229+
}
230+
231+
fn python_str_repr(s: &str) -> String {
232+
let mut out = String::from("'");
233+
for c in s.chars() {
234+
match c {
235+
'\\' => out.push_str("\\\\"),
236+
'\'' => out.push_str("\\'"),
237+
'\n' => out.push_str("\\n"),
238+
'\r' => out.push_str("\\r"),
239+
'\t' => out.push_str("\\t"),
240+
_ => out.push(c),
241+
}
242+
}
243+
out.push('\'');
244+
out
245+
}
246+
247+
fn python_default_repr(default: Option<&DefaultValue>, py_default: Option<&str>) -> Option<String> {
248+
if let Some(py_default) = py_default {
249+
return Some(py_default.to_owned());
250+
}
251+
match default {
252+
None => None,
253+
Some(None) => Some("<unrepresentable>".to_owned()),
254+
Some(Some(expr)) => match expr {
255+
Expr::Lit(syn::ExprLit { lit, .. }) => match lit {
256+
Lit::Bool(b) => Some(if b.value {
257+
"True".to_owned()
258+
} else {
259+
"False".to_owned()
260+
}),
261+
Lit::Int(i) => Some(i.base10_digits().to_owned()),
262+
Lit::Float(f) => Some(f.base10_digits().to_owned()),
263+
Lit::Str(s) => Some(python_str_repr(&s.value())),
264+
_ => Some("<unrepresentable>".to_owned()),
265+
},
266+
Expr::Path(path) if path.qself.is_none() && path.path.is_ident("None") => {
267+
Some("None".to_owned())
268+
}
269+
_ => Some("<unrepresentable>".to_owned()),
270+
},
271+
}
272+
}
273+
274+
fn param_token(field: &Field, attr: &ArgAttribute) -> Result<TokenStream> {
275+
if let ParameterKind::Flatten = attr.kind {
276+
let ty = &field.ty;
277+
return Ok(quote! {
278+
::rustpython_vm::function::Param::flatten(
279+
<#ty as ::rustpython_vm::function::FromArgs>::PARAMS
280+
)
281+
});
282+
}
283+
284+
let name = field.ident.as_ref().map(|ident| ident.unraw().to_string());
285+
let pyname = attr
286+
.name
287+
.clone()
288+
.or(name)
289+
.ok_or_else(|| err_span!(field, "field in tuple struct must have name attribute"))?;
290+
let default = python_default_repr(attr.default.as_ref(), attr.py_default.as_deref());
291+
let default_tok = match &default {
292+
Some(default) => quote!(Some(#default)),
293+
None => quote!(None),
294+
};
295+
let kind = match attr.kind {
296+
ParameterKind::PositionalOnly => {
297+
quote!(::rustpython_vm::function::ParamKind::PositionalOnly)
298+
}
299+
ParameterKind::PositionalOrKeyword => {
300+
quote!(::rustpython_vm::function::ParamKind::PositionalOrKeyword)
301+
}
302+
ParameterKind::KeywordOnly => quote!(::rustpython_vm::function::ParamKind::KeywordOnly),
303+
ParameterKind::Flatten => unreachable!(),
304+
};
305+
Ok(quote! {
306+
::rustpython_vm::function::Param {
307+
name: #pyname,
308+
kind: #kind,
309+
default: #default_tok,
310+
}
311+
})
312+
}
313+
215314
fn compute_arity_bounds(field_attrs: &[ArgAttribute]) -> (usize, usize) {
216315
let positional_fields = field_attrs.iter().filter(|attr| {
217316
matches!(
@@ -230,27 +329,40 @@ fn compute_arity_bounds(field_attrs: &[ArgAttribute]) -> (usize, usize) {
230329
}
231330

232331
pub(crate) fn impl_from_args(input: DeriveInput) -> Result<TokenStream> {
233-
let (fields, field_attrs) = match input.data {
234-
Data::Struct(syn::DataStruct { fields, .. }) => (
235-
fields
236-
.iter()
237-
.enumerate()
238-
.map(generate_field)
239-
.collect::<Result<TokenStream>>()?,
240-
fields
241-
.iter()
242-
.filter_map(|field| field.try_into().ok())
243-
.collect::<Vec<ArgAttribute>>(),
244-
),
245-
_ => bail_span!(input, "FromArgs input must be a struct"),
332+
let Data::Struct(syn::DataStruct {
333+
fields: struct_fields,
334+
..
335+
}) = input.data
336+
else {
337+
bail_span!(input, "FromArgs input must be a struct")
246338
};
247339

340+
let fields = struct_fields
341+
.iter()
342+
.enumerate()
343+
.map(generate_field)
344+
.collect::<Result<TokenStream>>()?;
345+
let field_attrs = struct_fields
346+
.iter()
347+
.map(ArgAttribute::try_from)
348+
.collect::<Result<Vec<_>>>()?;
248349
let (min_arity, max_arity) = compute_arity_bounds(&field_attrs);
249350

351+
let mut params = Vec::new();
352+
for field in &struct_fields {
353+
if is_phantom(field) {
354+
continue;
355+
}
356+
params.push(param_token(field, &ArgAttribute::try_from(field)?)?);
357+
}
358+
250359
let name = input.ident;
251360
let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
252361
let output = quote! {
253362
impl #impl_generics ::rustpython_vm::function::FromArgs for #name #ty_generics #where_clause {
363+
const PARAMS: Option<&'static [::rustpython_vm::function::Param]> =
364+
Some(&[#(#params),*]);
365+
254366
fn arity() -> ::std::ops::RangeInclusive<usize> {
255367
#min_arity..=#max_arity
256368
}

‎crates/derive-impl/src/pyclass.rs‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
use super::Diagnostic;
22
use crate::util::{
33
ALL_ALLOWED_NAMES, ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ExceptionItemMeta,
4-
ItemMeta, ItemMetaInner, ItemNursery, SimpleItemMeta, format_doc, infer_native_call_flags,
5-
pyclass_ident_and_attrs, pyexception_ident_and_attrs, text_signature,
4+
ItemMeta, ItemMetaInner, ItemNursery, SimpleItemMeta, infer_native_call_flags,
5+
internal_doc_tokens, pyclass_ident_and_attrs, pyexception_ident_and_attrs,
66
};
77
use core::str::FromStr;
88
use proc_macro2::{Delimiter, Group, Span, TokenStream, TokenTree};
@@ -71,6 +71,9 @@ struct ImplContext {
7171
extend_slots_items: ItemNursery,
7272
class_extensions: Vec<TokenStream>,
7373
errors: Vec<syn::Error>,
74+
/// Set when the impl has no generic parameters, so `Self` in argument
75+
/// types can be replaced before it appears in a nested const.
76+
self_ty_subst: Option<syn::Type>,
7477
}
7578

7679
fn extract_items_into_context<'a, Item>(
@@ -104,6 +107,9 @@ pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Resul
104107
let mut context = ImplContext::default();
105108
let mut tokens = match item {
106109
Item::Impl(mut imp) => {
110+
if imp.generics.params.is_empty() {
111+
context.self_ty_subst = Some((*imp.self_ty).clone());
112+
}
107113
extract_items_into_context(&mut context, imp.items.iter_mut());
108114

109115
let (impl_ty, payload_guess) = match imp.self_ty.as_ref() {
@@ -1123,7 +1129,6 @@ where
11231129
}
11241130
};
11251131
let drop_first_typed = usize::from(implicit_self.is_some());
1126-
let sig_doc = text_signature(func.sig(), &py_name, implicit_self);
11271132
let call_flags = infer_native_call_flags(func.sig(), drop_first_typed);
11281133

11291134
// Add #[allow(non_snake_case)] for setter methods like set___name__
@@ -1133,11 +1138,14 @@ where
11331138
args.attrs.push(allow_attr);
11341139
}
11351140

1136-
let doc = match (sig_doc, args.attrs.doc()) {
1137-
(Some(sig_doc), Some(doc)) => Some(format_doc(&sig_doc, &doc)),
1138-
(Some(sig_doc), None) => Some(format_doc(&sig_doc, "")),
1139-
(None, doc) => doc,
1140-
};
1141+
let doc = internal_doc_tokens(
1142+
func.sig(),
1143+
&py_name,
1144+
implicit_self,
1145+
args.attrs.doc(),
1146+
args.context.self_ty_subst.as_ref(),
1147+
None,
1148+
);
11411149
args.context.method_items.add_item(MethodNurseryItem {
11421150
py_name,
11431151
cfgs: args.cfgs.to_vec(),
@@ -1373,7 +1381,7 @@ struct MethodNurseryItem {
13731381
ident: Ident,
13741382
raw: bool,
13751383
coexist: bool,
1376-
doc: Option<String>,
1384+
doc: TokenStream,
13771385
attr_name: AttrName,
13781386
call_flags: TokenStream,
13791387
}
@@ -1401,11 +1409,7 @@ impl ToTokens for MethodNursery {
14011409
let py_name = &item.py_name;
14021410
let ident = &item.ident;
14031411
let cfgs = &item.cfgs;
1404-
let doc = if let Some(doc) = item.doc.as_ref() {
1405-
quote! { Some(#doc) }
1406-
} else {
1407-
quote! { None }
1408-
};
1412+
let doc = &item.doc;
14091413
let binding_flags = match &item.attr_name {
14101414
AttrName::Method => {
14111415
quote! { rustpython_vm::function::PyMethodFlags::METHOD }

‎crates/derive-impl/src/pymodule.rs‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ use crate::error::Diagnostic;
22
use crate::pystructseq::PyStructSequenceMeta;
33
use crate::util::{
44
ALL_ALLOWED_NAMES, AttrItemMeta, AttributeExt, ClassItemMeta, ContentItem, ContentItemInner,
5-
ErrorVec, ItemMeta, ItemNursery, ModuleItemMeta, SimpleItemMeta, format_doc,
6-
infer_native_call_flags, iter_use_idents, pyclass_ident_and_attrs, text_signature,
5+
ErrorVec, ItemMeta, ItemNursery, ModuleItemMeta, SimpleItemMeta, infer_native_call_flags,
6+
internal_doc_tokens, iter_use_idents, pyclass_ident_and_attrs,
77
};
88
use core::str::FromStr;
99
use proc_macro2::{Delimiter, Group, TokenStream, TokenTree};
@@ -524,7 +524,7 @@ struct FunctionNurseryItem {
524524
py_names: Vec<String>,
525525
cfgs: Vec<Attribute>,
526526
ident: Ident,
527-
doc: Option<String>,
527+
doc: TokenStream,
528528
call_flags: TokenStream,
529529
}
530530

@@ -556,10 +556,7 @@ impl ToTokens for ValidatedFunctionNursery {
556556
let cfgs = &item.cfgs;
557557
let cfgs = quote!(#(#cfgs)*);
558558
let py_names = &item.py_names;
559-
let doc = match &item.doc {
560-
Some(doc) => quote!(Some(#doc)),
561-
None => quote!(None),
562-
};
559+
let doc = &item.doc;
563560
let flags = &item.call_flags;
564561

565562
inner_tokens.extend(quote![
@@ -664,7 +661,6 @@ impl ModuleItem for FunctionItem {
664661
let item_meta = SimpleItemMeta::from_attr(ident.clone(), &item_attr)?;
665662

666663
let py_name = item_meta.simple_name()?;
667-
let sig_doc = text_signature(func.sig(), &py_name, None);
668664

669665
let module = args.module_name();
670666
// TODO: doc must exist at least one of code or CPython
@@ -673,11 +669,7 @@ impl ModuleItem for FunctionItem {
673669
.copied()
674670
.map(str::to_owned)
675671
});
676-
let doc = match (sig_doc, doc) {
677-
(Some(sig_doc), Some(doc)) => Some(format_doc(&sig_doc, &doc)),
678-
(Some(sig_doc), None) => Some(format_doc(&sig_doc, "")),
679-
(None, doc) => doc,
680-
};
672+
let doc = internal_doc_tokens(func.sig(), &py_name, None, doc, None, Some("$module"));
681673

682674
let py_names = {
683675
if self.py_attrs.is_empty() {

0 commit comments

Comments
 (0)