Skip to content

Commit 1613741

Browse files
youknowoneyouknowone
authored
Fix SymbolUsge::Iter, os.rename, sre_engine (RustPython#8390)
* Fix SymbolUsge::Iter * fix os.rename * fix sre * Re-export rename from crate::posix on Windows posix_windows.rs is mounted as the `posix` module on Windows, so `crate::posix_windows` does not resolve. Assisted-by: Claude * Apply rustfmt to sre_engine tests Assisted-by: Claude * Update test expectations for the sre and tokenize fixes - test_re: drop expectedFailure from test_word_boundaries, test_possessive_quantifiers and test_bug_gh101955 - test_inspect, test_pydoc: drop expectedFailure from the signature tests that now pass - test_pdb: drop expectedFailure from the two file-modification tests and +EXPECTED_FAILURE from the test_post_mortem_chained and test_pdb_asynctask doctests - test_unittest: mark test_autospec_on_bound_builtin_function as expectedFailure; inspect.signature() now succeeds on time.ctime because of its auto-generated __text_signature__ Assisted-by: Claude
1 parent 3aaec06 commit 1613741

10 files changed

Lines changed: 113 additions & 16 deletions

File tree

‎Lib/test/test_inspect/test_inspect.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5987,7 +5987,6 @@ def _strip_non_python_syntax(self, input,
59875987
self.assertEqual(computed_clean_signature, clean_signature)
59885988
self.assertEqual(computed_self_parameter, self_parameter)
59895989

5990-
@unittest.expectedFailure # TODO: RUSTPYTHON; + (module, /, path, mode, *, dir_fd=None, effective_ids=False, follow_symlinks=True)
59915990
def test_signature_strip_non_python_syntax(self):
59925991
self._strip_non_python_syntax(
59935992
"($module, /, path, mode, *, dir_fd=None, " +
@@ -6318,7 +6317,6 @@ def test_weakref_module_has_signatures(self):
63186317
no_signature = {'ReferenceType', 'ref'}
63196318
self._test_module_has_signatures(weakref, no_signature)
63206319

6321-
@unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: <function TestSignatureDefinitions.test_python_function_override_signature.<locals>.func at 0xa4c07a580> builtin has invalid signature
63226320
def test_python_function_override_signature(self):
63236321
def func(*args, **kwargs):
63246322
pass

‎Lib/test/test_pdb.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1290,7 +1290,7 @@ def test_post_mortem_chained():
12901290
... except Exception as e:
12911291
... pdb._post_mortem(e, instance)
12921292
1293-
>>> with PdbTestInput([ # TODO: RUSTPYTHON # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE +EXPECTED_FAILURE
1293+
>>> with PdbTestInput([ # TODO: RUSTPYTHON # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
12941294
... 'exceptions',
12951295
... 'exceptions 0',
12961296
... '$_exception',
@@ -2133,7 +2133,7 @@ def test_pdb_asynctask():
21332133
>>> def test_function():
21342134
... asyncio.run(test(), loop_factory=asyncio.EventLoop)
21352135
2136-
>>> with PdbTestInput([ # TODO: RUSTPYTHON # doctest: +ELLIPSIS +EXPECTED_FAILURE
2136+
>>> with PdbTestInput([ # doctest: +ELLIPSIS
21372137
... '$_asynctask',
21382138
... 'continue',
21392139
... ]):
@@ -4185,7 +4185,6 @@ def test_blocks_at_first_code_line(self):
41854185
self.assertTrue(any("__main__.py(4)<module>()"
41864186
in l for l in stdout.splitlines()), stdout)
41874187

4188-
@unittest.expectedFailure # TODO: RUSTPYTHON
41894188
def test_file_modified_after_execution(self):
41904189
script = """
41914190
print("hello")
@@ -4259,7 +4258,6 @@ def test_file_modified_after_execution_with_multiple_instances(self):
42594258
self.assertIn("WARNING:", stdout)
42604259
self.assertIn("was edited", stdout)
42614260

4262-
@unittest.expectedFailure # TODO: RUSTPYTHON
42634261
def test_file_modified_after_execution_with_restart(self):
42644262
script = """
42654263
import random

‎Lib/test/test_pydoc/test_pydoc.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1708,7 +1708,6 @@ def test_bound_builtin_classmethod_unrepresentable_default(self):
17081708
"classmeth(a, b=<x>) class method of "
17091709
"_testcapi.DocStringUnrepresentableSignatureTest")
17101710

1711-
@unittest.expectedFailure # TODO: RUSTPYTHON
17121711
def test_overridden_text_signature(self):
17131712
class C:
17141713
def meth(*args, **kwargs):

‎Lib/test/test_re.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -890,7 +890,6 @@ def test_named_unicode_escapes(self):
890890
self.checkPatternError(br'\N{LESS-THAN SIGN}', r'bad escape \N', 0)
891891
self.checkPatternError(br'[\N{LESS-THAN SIGN}]', r'bad escape \N', 1)
892892

893-
@unittest.expectedFailure # TODO: RUSTPYTHON; re.search(r"\B", "") now returns a match in CPython 3.14
894893
def test_word_boundaries(self):
895894
# See http://bugs.python.org/issue10713
896895
self.assertEqual(re.search(r"\b(abc)\b", "abc").group(1), "abc")
@@ -2493,7 +2492,6 @@ def test_search_anchor_at_beginning(self):
24932492
# With optimization -- 0.0003 seconds.
24942493
self.assertLess(stopwatch.seconds, 0.1)
24952494

2496-
@unittest.expectedFailure # TODO: RUSTPYTHON
24972495
def test_possessive_quantifiers(self):
24982496
"""Test Possessive Quantifiers
24992497
Test quantifiers of the form @+ for some repetition operator @,
@@ -2647,7 +2645,6 @@ def test_bug_gh100061(self):
26472645
self.assertEqual(re.match("(?>(?:ab?c){1,3})", "aca").span(), (0, 2))
26482646
self.assertEqual(re.match("(?:ab?c){1,3}+", "aca").span(), (0, 2))
26492647

2650-
@unittest.expectedFailure # TODO: RUSTPYTHON; self.assertEqual(re.match('((x)|y|z){3}+', 'xyz').groups(), ('z', 'x'))\n AssertionError: Tuples differ: ('x', 'x') != ('z', 'x')
26512648
def test_bug_gh101955(self):
26522649
# Possessive quantifier with nested alternative with capture groups
26532650
self.assertEqual(re.match('((x)|y|z)*+', 'xyz').groups(), ('z', 'x'))

‎Lib/test/test_unittest/testmock/testhelpers.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -929,6 +929,7 @@ def check_data_descriptor(mock_attr):
929929
check_data_descriptor(foo.desc)
930930

931931

932+
@unittest.expectedFailure # TODO: RUSTPYTHON; time.ctime has an auto-generated __text_signature__, so inspect.signature() succeeds instead of raising ValueError
932933
def test_autospec_on_bound_builtin_function(self):
933934
meth = types.MethodType(time.ctime, time.time())
934935
self.assertIsInstance(meth(), str)

‎crates/codegen/src/symboltable.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3356,7 +3356,14 @@ impl SymbolTableBuilder {
33563356
flags.insert(SymbolFlags::USE);
33573357
}
33583358
SymbolUsage::Iter => {
3359-
flags.insert(SymbolFlags::ITER | SymbolFlags::DEF_COMP_ITER);
3359+
// CPython symtable_add_def_helper() records an inlined
3360+
// comprehension target as a local definition as well as a
3361+
// comprehension iterator. Keep ITER as the internal
3362+
// re-assignment check marker; DEF_LOCAL is part of the public
3363+
// ste_symbols flags exposed by _symtable.
3364+
flags.insert(
3365+
SymbolFlags::DEF_LOCAL | SymbolFlags::ITER | SymbolFlags::DEF_COMP_ITER,
3366+
);
33603367
}
33613368
SymbolUsage::TypeParam => {
33623369
flags.insert(SymbolFlags::DEF_LOCAL | SymbolFlags::DEF_TYPE_PARAM);

‎crates/host_env/src/os.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@
44
use crate::crt_fd;
55
#[cfg(windows)]
66
use crate::fs;
7+
#[cfg(windows)]
8+
pub use crate::posix::rename;
9+
#[cfg(any(unix, target_os = "wasi"))]
10+
pub use crate::posix_unix_like::rename;
711
#[cfg(any(unix, windows))]
812
use core::ffi::CStr;
913
use core::str::Utf8Error;
@@ -28,6 +32,23 @@ use {
2832
},
2933
};
3034

35+
#[cfg(not(any(unix, windows, target_os = "wasi")))]
36+
pub fn rename(
37+
from: impl AsRef<std::path::Path>,
38+
from_fd: Option<crt_fd::Borrowed<'_>>,
39+
to: impl AsRef<std::path::Path>,
40+
to_fd: Option<crt_fd::Borrowed<'_>>,
41+
) -> io::Result<()> {
42+
if from_fd.is_none() && to_fd.is_none() {
43+
std::fs::rename(from, to)
44+
} else {
45+
Err(io::Error::new(
46+
io::ErrorKind::Unsupported,
47+
"renameat is not available on this platform",
48+
))
49+
}
50+
}
51+
3152
/// Convert exit code to std::process::ExitCode
3253
///
3354
/// On Windows, this supports the full u32 range including STATUS_CONTROL_C_EXIT (0xC000013A).

‎crates/sre_engine/src/engine.rs‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,21 @@ impl Marks {
110110
self.marks_stack.pop();
111111
}
112112

113+
fn stack_depth(&self) -> usize {
114+
self.marks_stack.len()
115+
}
116+
117+
fn discard_to(&mut self, depth: usize) {
118+
self.marks_stack.truncate(depth);
119+
}
120+
121+
fn restore_to(&mut self, depth: usize) {
122+
let (marks, last_index) = self.marks_stack[depth].clone();
123+
self.marks = marks;
124+
self.last_index = last_index;
125+
self.marks_stack.truncate(depth);
126+
}
127+
113128
fn clear(&mut self) {
114129
self.last_index = -1;
115130
self.marks.clear();
@@ -144,6 +159,7 @@ impl State {
144159
jump: Jump::OpCode,
145160
repeat_ctx_id: usize::MAX,
146161
count: -1,
162+
marks_stack_base: usize::MAX,
147163
};
148164
_match(req, self, ctx)
149165
}
@@ -165,6 +181,7 @@ impl State {
165181
jump: Jump::OpCode,
166182
repeat_ctx_id: usize::MAX,
167183
count: -1,
184+
marks_stack_base: usize::MAX,
168185
};
169186

170187
if ctx.peek_code(&req, 0) == SreOpcode::INFO as u32 {
@@ -483,6 +500,7 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
483500
}
484501
Jump::PossessiveRepeat2 => {
485502
if popped_result {
503+
ctx.cursor = state.cursor;
486504
ctx.count += 1;
487505
ctx.jump = Jump::PossessiveRepeat1;
488506
continue 'context;
@@ -495,6 +513,7 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
495513
if ((ctx.count as usize) < max_count || max_count == MAXREPEAT)
496514
&& ctx.cursor.position != state.cursor.position
497515
{
516+
ctx.marks_stack_base = state.marks.stack_depth();
498517
state.marks.push();
499518
ctx.cursor = state.cursor;
500519
let mut next = ctx.next_offset(4, Jump::PossessiveRepeat4);
@@ -507,12 +526,12 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
507526
}
508527
Jump::PossessiveRepeat4 => {
509528
if popped_result {
510-
state.marks.pop_discard();
529+
state.marks.discard_to(ctx.marks_stack_base);
511530
ctx.count += 1;
512531
ctx.jump = Jump::PossessiveRepeat3;
513532
continue 'context;
514533
}
515-
state.marks.pop();
534+
state.marks.restore_to(ctx.marks_stack_base);
516535
state.cursor = ctx.cursor;
517536
ctx.skip_code_from(req, 1);
518537
ctx.skip_code(1);
@@ -1057,6 +1076,7 @@ struct MatchContext {
10571076
jump: Jump,
10581077
repeat_ctx_id: usize,
10591078
count: isize,
1079+
marks_stack_base: usize,
10601080
}
10611081

10621082
impl MatchContext {
@@ -1147,7 +1167,9 @@ impl MatchContext {
11471167
mut word_checker: F,
11481168
) -> bool {
11491169
if self.at_beginning() && self.at_end(req) {
1150-
return false;
1170+
// Python 3.14 changed `\B` to match an empty input. Keep the
1171+
// boundary predicate false there, but its negation true.
1172+
return true;
11511173
}
11521174
let that = !self.at_beginning() && word_checker(self.back_peek_char::<S>());
11531175
let this = !self.at_end(req) && word_checker(self.peek_char::<S>());

‎crates/sre_engine/tests/tests.rs‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#[cfg(test)]
33
mod tests {
44
use rustpython_sre_engine::{Request, State, StrDrive};
5+
use rustpython_wtf8::Wtf8Buf;
56

67
struct Pattern {
78
#[expect(unused, reason = "Needed for automated script")]
@@ -44,7 +45,7 @@ mod tests {
4445
#[rustfmt::skip] let big_b = Pattern { pattern: "\\B", code: &[14, 4, 0, 0, 0, 6, 11, 1] };
4546
// END GENERATED
4647
let (req, mut state) = big_b.state("");
47-
assert!(!state.search(req));
48+
assert!(state.search(req));
4849
}
4950

5051
#[test]
@@ -169,6 +170,40 @@ mod tests {
169170
assert!(!state.py_match(&req));
170171
}
171172

173+
#[test]
174+
fn possessive_repeat_keeps_last_capture() {
175+
use optional::Optioned;
176+
177+
let single_code = &[17, 0, 24, 6, 0, 1, 16, 101, 1, 17, 1, 1];
178+
let req = Request::new("eeea", 3, usize::MAX, single_code, false);
179+
let mut single_state = State::default();
180+
assert!(single_state.py_match(&req));
181+
assert_eq!(
182+
single_state.marks.get(0),
183+
(Optioned::some(3), Optioned::some(3))
184+
);
185+
186+
// (e?){2,4}+a: the fourth successful iteration is empty, so group 1
187+
// must retain its final empty span rather than the previous "e".
188+
#[rustfmt::skip] let optional = Pattern {
189+
pattern: "(e?){2,4}+a",
190+
code: &[14, 4, 0, 1, 5, 28, 14, 2, 4, 17, 0, 24, 6, 0, 1, 16, 101, 1, 17, 1, 1, 16, 97, 1],
191+
};
192+
let (req, mut state) = optional.state("eeea");
193+
assert!(state.py_match(&req));
194+
assert_eq!(state.marks.get(0), (Optioned::some(3), Optioned::some(3)));
195+
196+
// ((x)|y|z){3}+: group 1 is the final "z"; group 2 retains "x".
197+
#[rustfmt::skip] let alternation = Pattern {
198+
pattern: "((x)|y|z){3}+",
199+
code: &[14, 4, 0, 3, 3, 28, 28, 3, 3, 17, 0, 7, 9, 17, 2, 16, 120, 17, 3, 15, 12, 5, 16, 121, 15, 7, 5, 16, 122, 15, 2, 0, 17, 1, 1, 1],
200+
};
201+
let (req, mut state) = alternation.state("xyz");
202+
assert!(state.py_match(&req));
203+
assert_eq!(state.marks.get(0), (Optioned::some(2), Optioned::some(3)));
204+
assert_eq!(state.marks.get(1), (Optioned::some(0), Optioned::some(1)));
205+
}
206+
172207
#[test]
173208
fn bug_20998() {
174209
// pattern p = re.compile('[a-c]+', re.I)
@@ -181,6 +216,23 @@ mod tests {
181216
assert_eq!(state.cursor.position, 3);
182217
}
183218

219+
#[test]
220+
fn ascii_ignore_keeps_nonascii_range_literal() {
221+
// pattern p = re.compile(r'[\u0430-\u045f]', re.I | re.A)
222+
//
223+
// ASCII-only case folding must not discard an exact non-ASCII range:
224+
// U+0450 lies in the compiled U+0430..U+045F interval.
225+
#[rustfmt::skip] let p = Pattern {
226+
pattern: "[\\u0430-\\u045f]",
227+
code: &[14, 8, 4, 1, 1, 22, 1072, 1119, 0, 13, 5, 22, 1072, 1119, 0, 1],
228+
};
229+
let (req, mut state) = p.state("\u{0450}");
230+
assert!(state.py_match(&req));
231+
let subject = Wtf8Buf::from("\u{0450}");
232+
let (req, mut state) = p.state(subject.as_ref());
233+
assert!(state.py_match(&req));
234+
}
235+
184236
#[test]
185237
fn bigcharset() {
186238
// pattern p = re.compile('[a-z]*', re.I)

‎crates/stdlib/src/_tokenize.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,9 @@ mod _tokenize {
237237
}
238238

239239
let raw_type = token_kind_value(kind);
240-
let token_type = if extra_tokens && raw_type > TOKEN_DEDENT && raw_type < TOKEN_OP {
240+
let token_type = if extra_tokens
241+
&& (kind == TokenKind::Unknown || (raw_type > TOKEN_DEDENT && raw_type < TOKEN_OP))
242+
{
241243
TOKEN_OP
242244
} else {
243245
raw_type

0 commit comments

Comments
 (0)