Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions scripts/update_lib/cmd_auto_mark.py
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,7 @@ def build_patches(
"""Convert failing tests to patch format."""
patches = {}
error_messages = error_messages or {}
for class_name, method_name in test_parts_set:
for class_name, method_name in sorted(test_parts_set):
if class_name not in patches:
patches[class_name] = {}
reason = error_messages.get((class_name, method_name), "")
Expand Down Expand Up @@ -401,6 +401,9 @@ def _method_removal_range(
and COMMENT in lines[first - 1]
):
first -= 1
# Also remove a preceding blank line to avoid double-blanks after removal
if first > 0 and not lines[first - 1].strip():
first -= 1
return range(first, func_node.end_lineno)


Expand Down Expand Up @@ -753,7 +756,11 @@ def auto_mark_file(
results = run_test(test_name, skip_build=skip_build)

# Check if test run failed entirely (e.g., import error, crash)
if not results.tests_result:
if (
not results.tests_result
and not results.tests
and not results.unexpected_successes
):
raise TestRunError(
f"Test run failed for {test_name}. "
f"Output: {results.stdout[-500:] if results.stdout else '(no output)'}"
Expand Down Expand Up @@ -870,7 +877,11 @@ def auto_mark_directory(
results = run_test(test_name, skip_build=skip_build)

# Check if test run failed entirely (e.g., import error, crash)
if not results.tests_result:
if (
not results.tests_result
and not results.tests
and not results.unexpected_successes
):
raise TestRunError(
f"Test run failed for {test_name}. "
f"Output: {results.stdout[-500:] if results.stdout else '(no output)'}"
Expand Down
5 changes: 5 additions & 0 deletions scripts/update_lib/deps.py
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,11 @@ def clear_import_graph_caches() -> None:
"test_descrtut.py",
],
},
"code": {
"test": [
"test_code_module.py",
],
},
"contextlib": {
"test": [
"test_contextlib.py",
Expand Down
4 changes: 2 additions & 2 deletions scripts/update_lib/patch_spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -282,13 +282,13 @@ def _iter_patch_lines(
yield (lineno - 1, textwrap.indent(patch_lines, indent))

# Phase 2: Iterate and mark inherited tests
for cls_name, tests in patches.items():
for cls_name, tests in sorted(patches.items()):
lineno = cache.get(cls_name)
if not lineno:
print(f"WARNING: {cls_name} does not exist in remote file", file=sys.stderr)
continue

for test_name, specs in tests.items():
for test_name, specs in sorted(tests.items()):
decorators = "\n".join(spec.as_decorator() for spec in specs)
# Check current class and ancestors for async method
is_async = False
Expand Down
185 changes: 185 additions & 0 deletions scripts/update_lib/tests/test_auto_mark.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
"""Tests for auto_mark.py - test result parsing and auto-marking."""

import ast
import pathlib
import subprocess
import tempfile
import unittest
from unittest import mock

from update_lib.cmd_auto_mark import (
Test,
TestResult,
TestRunError,
_expand_stripped_to_children,
_is_super_call_only,
apply_test_changes,
auto_mark_directory,
auto_mark_file,
collect_test_changes,
extract_test_methods,
parse_results,
Expand Down Expand Up @@ -94,6 +100,34 @@ def test_parse_tests_result(self):
result = parse_results(_make_result("== Tests result: FAILURE ==\n"))
self.assertEqual(result.tests_result, "FAILURE")

def test_parse_crashed_run_no_tests_result(self):
"""Test results are still parsed when the runner crashes (no Tests result line)."""
stdout = """\
Run 1 test sequentially in a single process
0:00:00 [1/1] test_ast
test_foo (test.test_ast.test_ast.TestA.test_foo) ... FAIL
test_bar (test.test_ast.test_ast.TestA.test_bar) ... ok
test_baz (test.test_ast.test_ast.TestB.test_baz) ... ERROR
"""
result = parse_results(_make_result(stdout))
self.assertEqual(result.tests_result, "")
self.assertEqual(len(result.tests), 2)
names = {t.name for t in result.tests}
self.assertIn("test_foo", names)
self.assertIn("test_baz", names)

def test_parse_crashed_run_has_unexpected_success(self):
"""Unexpected successes are parsed even without Tests result line."""
stdout = """\
Run 1 test sequentially in a single process
0:00:00 [1/1] test_ast
test_foo (test.test_ast.test_ast.TestA.test_foo) ... unexpected success
UNEXPECTED SUCCESS: test_foo (test.test_ast.test_ast.TestA.test_foo)
"""
result = parse_results(_make_result(stdout))
self.assertEqual(result.tests_result, "")
self.assertEqual(len(result.unexpected_successes), 1)

def test_parse_error_messages(self):
"""Single and multiple error messages are parsed from tracebacks."""
stdout = """\
Expand Down Expand Up @@ -747,5 +781,156 @@ async def test_one(self):
)


class TestAutoMarkFileWithCrashedRun(unittest.TestCase):
"""auto_mark_file should process partial results when test runner crashes."""

CRASHED_STDOUT = """\
Run 1 test sequentially in a single process
0:00:00 [1/1] test_example
test_foo (test.test_example.TestA.test_foo) ... FAIL
test_bar (test.test_example.TestA.test_bar) ... ok
======================================================================
FAIL: test_foo (test.test_example.TestA.test_foo)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test.py", line 10, in test_foo
self.assertEqual(1, 2)
AssertionError: 1 != 2
"""

def test_auto_mark_file_crashed_run(self):
"""auto_mark_file processes results even when tests_result is empty (crash)."""
test_code = f"""import unittest
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Unnecessary f-string prefix.

The string literal has no placeholders, so the f prefix is unnecessary. This was flagged by static analysis (F541).

🔧 Proposed fix
-        test_code = f"""import unittest
+        test_code = """import unittest
🧰 Tools
🪛 Flake8 (7.3.0)

[error] 803-803: f-string is missing placeholders

(F541)

🤖 Prompt for AI Agents
In `@scripts/update_lib/tests/test_auto_mark.py` at line 803, The test string
assigned to the variable test_code uses an unnecessary f-string prefix
(f"""...""") even though there are no placeholders; update the assignment in
tests/test_auto_mark.py by removing the leading "f" so the literal is a plain
triple-quoted string (change test_code = f"""import unittest to test_code =
"""import unittest), ensuring any other nearby multi-line test_code usages
follow the same pattern.


class TestA(unittest.TestCase):
def test_foo(self):
pass

def test_bar(self):
pass
"""
with tempfile.TemporaryDirectory() as tmpdir:
test_file = pathlib.Path(tmpdir) / "test_example.py"
test_file.write_text(test_code)

mock_result = TestResult()
mock_result.tests_result = ""
mock_result.tests = [
Test(
name="test_foo",
path="test.test_example.TestA.test_foo",
result="fail",
error_message="AssertionError: 1 != 2",
),
]

with mock.patch(
"update_lib.cmd_auto_mark.run_test", return_value=mock_result
):
added, removed, regressions = auto_mark_file(
test_file, mark_failure=True, verbose=False
)

self.assertEqual(added, 1)
contents = test_file.read_text()
self.assertIn("expectedFailure", contents)

def test_auto_mark_file_no_results_at_all_raises(self):
"""auto_mark_file raises TestRunError when there are zero parsed results."""
test_code = """import unittest

class TestA(unittest.TestCase):
def test_foo(self):
pass
"""
with tempfile.TemporaryDirectory() as tmpdir:
test_file = pathlib.Path(tmpdir) / "test_example.py"
test_file.write_text(test_code)

mock_result = TestResult()
mock_result.tests_result = ""
mock_result.tests = []
mock_result.stdout = "some crash output"

with mock.patch(
"update_lib.cmd_auto_mark.run_test", return_value=mock_result
):
with self.assertRaises(TestRunError):
auto_mark_file(test_file, verbose=False)


class TestAutoMarkDirectoryWithCrashedRun(unittest.TestCase):
"""auto_mark_directory should process partial results when test runner crashes."""

def test_auto_mark_directory_crashed_run(self):
"""auto_mark_directory processes results even when tests_result is empty."""
test_code = f"""import unittest
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Unnecessary f-string prefix.

Same issue as above - the string has no placeholders.

🔧 Proposed fix
-        test_code = f"""import unittest
+        test_code = """import unittest
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
test_code = f"""import unittest
test_code = """import unittest
🧰 Tools
🪛 Flake8 (7.3.0)

[error] 867-867: f-string is missing placeholders

(F541)

🤖 Prompt for AI Agents
In `@scripts/update_lib/tests/test_auto_mark.py` at line 867, The test constructs
a multi-line string assigned to test_code using an unnecessary f-string prefix;
update the assignment of test_code in test_auto_mark.py (the test_code variable)
by removing the leading "f" so it becomes a plain string literal (no
interpolation) to match other similar test strings and avoid misleading
formatting.


class TestA(unittest.TestCase):
def test_foo(self):
pass
"""
with tempfile.TemporaryDirectory() as tmpdir:
test_dir = pathlib.Path(tmpdir) / "test_example"
test_dir.mkdir()
test_file = test_dir / "test_sub.py"
test_file.write_text(test_code)

mock_result = TestResult()
mock_result.tests_result = ""
mock_result.tests = [
Test(
name="test_foo",
path="test.test_example.test_sub.TestA.test_foo",
result="fail",
error_message="AssertionError: oops",
),
]

with (
mock.patch(
"update_lib.cmd_auto_mark.run_test", return_value=mock_result
),
mock.patch(
"update_lib.cmd_auto_mark.get_test_module_name",
side_effect=lambda p: (
"test_example" if p == test_dir else "test_example.test_sub"
),
),
):
added, removed, regressions = auto_mark_directory(
test_dir, mark_failure=True, verbose=False
)

self.assertEqual(added, 1)
contents = test_file.read_text()
self.assertIn("expectedFailure", contents)

def test_auto_mark_directory_no_results_raises(self):
"""auto_mark_directory raises TestRunError when zero results."""
with tempfile.TemporaryDirectory() as tmpdir:
test_dir = pathlib.Path(tmpdir) / "test_example"
test_dir.mkdir()
test_file = test_dir / "test_sub.py"
test_file.write_text("import unittest\n")

mock_result = TestResult()
mock_result.tests_result = ""
mock_result.tests = []
mock_result.stdout = "crash"

with (
mock.patch(
"update_lib.cmd_auto_mark.run_test", return_value=mock_result
),
mock.patch(
"update_lib.cmd_auto_mark.get_test_module_name",
return_value="test_example",
),
):
with self.assertRaises(TestRunError):
auto_mark_directory(test_dir, verbose=False)


if __name__ == "__main__":
unittest.main()
Loading