Skip to content

More accurate test dependencies #7261

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 1 commit into from
Feb 28, 2026
Merged

More accurate test dependencies #7261

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions scripts/update_lib/cmd_todo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,11 @@ def get_all_tests(cpython_prefix: str) -> list[str]:
tests = set()
for entry in test_dir.iterdir():
# Skip non-test items
if not entry.name.startswith(("_test", "test_")):
if "test" not in entry.name:
continue

# Exclude special cases
if "regrtest" in entry.name:
continue

if entry.is_file() and entry.suffix == ".py":
Expand Down Expand Up @@ -333,8 +337,16 @@ def compute_test_todo_list(
# Get order from DEPENDENCIES
test_order = lib_test_order[lib_name].index(test_name)
else:
# Extract lib name from test name (test_foo -> foo)
lib_name = test_name.removeprefix("test_").removeprefix("_test")
# Extract lib name from test name:
# - test_foo -> foo
# - datetimetester -> datetime
# - xmltests -> xml
lib_name = (
test_name.removeprefix("test_")
.removeprefix("_test")
.removesuffix("tester")
.removesuffix("tests")
)
test_order = 0 # Default order for tests not in DEPENDENCIES

# Check if corresponding lib is up-to-date
Expand Down
29 changes: 12 additions & 17 deletions scripts/update_lib/deps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,7 @@ def test_imports(self) -> frozenset[str]:
@property
def lib_imports(self) -> frozenset[str]:
return frozenset(
# module.split(".", 1)[0]
module
for module in self.__imports
if not module.startswith("test.")
module for module in self.__imports if not module.startswith("test.")
)

def visit_Import(self, node):
Expand Down Expand Up @@ -119,7 +116,7 @@ def visit_Call(self, node) -> None:
self.__imports.add(f"test.{target}")


def parse_test_imports(content: str) -> set[str]:
def parse_test_imports(content: str) -> frozenset[str]:
"""Parse test file content and extract test package dependencies."""
if not (tree := safe_parse_ast(content)):
return set()
Expand All @@ -129,7 +126,7 @@ def parse_test_imports(content: str) -> set[str]:
return visitor.test_imports


def parse_lib_imports(content: str) -> set[str]:
def parse_lib_imports(content: str) -> frozenset[str]:
"""Parse library file and extract all imported module names."""
if not (tree := safe_parse_ast(content)):
return set()
Expand All @@ -147,8 +144,7 @@ def parse_lib_imports(content: str) -> set[str]:
def filter_rustpython_todo(content: str) -> str:
"""Remove lines containing RustPython TODO markers."""
lines = content.splitlines(keepends=True)
filtered = [line for line in lines if TODO_MARKER not in line]
return "".join(filtered)
return "".join(line for line in lines if TODO_MARKER not in line)


def count_rustpython_todo(content: str) -> int:
Expand Down Expand Up @@ -342,7 +338,7 @@ def clear_import_graph_caches() -> None:
},
"codecs": {
"test": [
"test_codecs.py",
"test_charmapcodec.py",
"test_codeccallbacks.py",
"test_codecencodings_cn.py",
"test_codecencodings_hk.py",
Expand All @@ -355,8 +351,9 @@ def clear_import_graph_caches() -> None:
"test_codecmaps_jp.py",
"test_codecmaps_kr.py",
"test_codecmaps_tw.py",
"test_charmapcodec.py",
"test_codecs.py",
"test_multibytecodec.py",
"testcodec.py",
],
},
# Non-pattern hard_deps (can't be auto-detected)
Expand Down Expand Up @@ -423,6 +420,7 @@ def clear_import_graph_caches() -> None:
"test_multiprocessing_forkserver",
"test_multiprocessing_spawn",
"test_multiprocessing_main_handling.py",
"_test_multiprocessing.py",
],
},
"urllib": {
Expand Down Expand Up @@ -745,12 +743,9 @@ def resolve_hard_dep_parent(name: str, cpython_prefix: str) -> str | None:
# Auto-detect _py{module} or _py_{module} patterns
# Only if the parent module actually exists
if name.startswith("_py"):
if name.startswith("_py_"):
# _py_abc -> abc
parent = name[4:]
else:
# _pydatetime -> datetime
parent = name[3:]
# _py_abc -> abc
# _pydatetime -> datetime
parent = name.removeprefix("_py_").removeprefix("_py")

# Verify the parent module exists
lib_dir = pathlib.Path(cpython_prefix) / "Lib"
Expand Down Expand Up @@ -781,7 +776,7 @@ def resolve_test_to_lib(test_name: str) -> str | None:
tests = dep_info.get("test", [])
for test_path in tests:
# test_path is like "test_urllib2.py" or "test_multiprocessing_fork"
path_stem = test_path[:-3] if test_path.endswith(".py") else test_path
path_stem = test_path.removesuffix(".py")
if path_stem == test_name:
return lib_name

Expand Down
Loading