Skip to content

Commit 792c94c

Browse files
author
jgstern-agent
committed
fix(smart-test): per-package coverage fallback for sanity check
CI's per-package manifest sanity check requires ≥1 test from packages/<P>/tests/ for every package whose source changed. The reverse-slice resolves tests via transitive imports, so when the only changed source for package P is __init__.py and no test in packages/P/tests/ imports __init__ directly, the slice correctly returns zero matches — but the sanity check still expects ≥1, blocking the merge. This recurs every release because prepare-release bumps every package's __init__.py simultaneously. Adds scripts/per_package_fallback.py: appends the alphabetically-first test_*.py / BRANCHES_test_*.py from any unrepresented affected package's tests/ directory. Same lightweight-test-per-package rule the existing version-only branch in smart-test uses; applied universally so the slice-resolved-non-empty path is also covered. Surfaced while preparing v4.1.0 (PR #3575): the release commit's manifest selected 0 tests for hypergumbo-lang-rust-analyzer because no test in that package imports __init__, breaking the merge with HTTP 405 even though pytest-retry recovered. Signed-off-by: jgstern-agent <josh-agent@iterabloom.com>
1 parent 6cb7a1b commit 792c94c

4 files changed

Lines changed: 427 additions & 3 deletions

File tree

.ci/affected-tests.txt

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
# Test selection manifest
2-
# Generated by smart-test at 2026-05-08T00:43:16-04:00
2+
# Generated by smart-test at 2026-05-08T03:10:09-04:00
33
# Mode: targeted
44
# Baseline: 63feb37916959501e7e858532b7796ba46f3b0ad
5-
# Changed files: 217
5+
# Changed files: 220
66
# Changed source files: 76
7-
# Selected tests: 170
7+
# Selected tests: 171
88
#
99
# === CHANGED_SOURCE_FILES ===
1010
packages/hypergumbo-core/src/hypergumbo_core/audit_findings.py
@@ -254,3 +254,4 @@ packages/hypergumbo-lang-mainstream/tests/test_swift.py
254254
packages/hypergumbo-lang-mainstream/tests/test_toml_analyzer.py
255255
packages/hypergumbo-lang-mainstream/tests/test_ts_decorator_edges.py
256256
tests/test_on_transcript_change.py
257+
tests/test_per_package_fallback.py

scripts/per_package_fallback.py

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
#!/usr/bin/env python3
2+
# SPDX-License-Identifier: AGPL-3.0-or-later
3+
"""Per-package fallback for ``scripts/smart-test``'s test selection.
4+
5+
CI runs each ``packages/<P>/`` in isolation, so the manifest sanity check
6+
in ``ci.yml`` requires at least one test under ``packages/<P>/tests/``
7+
whenever any source file under ``packages/<P>/src/`` is changed.
8+
``hypergumbo slice --files`` resolves tests via transitive imports, so
9+
when the only changed source for package ``P`` is ``__init__.py`` and no
10+
test in ``packages/P/tests/`` imports ``__init__`` directly, the slice
11+
correctly reports zero matches — but the sanity check still expects
12+
≥1, and the merge is blocked. This recurs every release because
13+
``prepare-release`` bumps every package's ``__init__.py`` simultaneously.
14+
15+
The fix is a per-package fallback layer applied after the slice: for any
16+
``packages/<P>/src/...`` file in the changed set whose package is not
17+
already represented in the affected-tests output, append the
18+
alphabetically-first ``test_*.py`` (or ``BRANCHES_test_*.py``) found in
19+
``packages/<P>/tests/``. The choice is alphabetical for determinism;
20+
any test in the directory satisfies the sanity check, and the
21+
"lightweight test per package" cost is the same as the existing
22+
version-only branch in ``smart-test`` (lines ~592–650), which uses the
23+
same rule.
24+
25+
Files outside ``packages/<P>/src/`` (top-level ``scripts/``, hooks,
26+
docs) are silently ignored — they don't define a per-package coverage
27+
expectation and are handled separately by
28+
``.agent/hooks/_shared/top_level_test_map.py``.
29+
30+
CLI: ``per_package_fallback.py REPO_ROOT --tests-file PATH < CHANGED``.
31+
Reads newline-separated changed source paths from stdin (relative to
32+
``REPO_ROOT``). Reads the current affected-tests selection (also
33+
newline-separated, relative paths) from ``--tests-file``; a missing
34+
file is treated as an empty selection. Writes additional test paths
35+
(one per line, sorted, deduplicated) to stdout. Always exits 0
36+
unless argv is malformed.
37+
"""
38+
from __future__ import annotations
39+
40+
import argparse
41+
import sys
42+
from pathlib import Path
43+
from typing import Iterable, List, Set
44+
45+
46+
_TEST_FILE_PREFIXES = ("test_", "BRANCHES_test_")
47+
48+
49+
def _affected_packages(changed_source_files: Iterable[str]) -> List[str]:
50+
"""Extract unique package names from ``packages/<P>/src/...`` paths.
51+
52+
Anything that doesn't match the ``packages/<P>/src/`` shape is dropped.
53+
"""
54+
seen: Set[str] = set()
55+
ordered: List[str] = []
56+
for raw in changed_source_files:
57+
path = raw.strip()
58+
if not path:
59+
continue
60+
parts = path.split("/")
61+
if len(parts) < 4 or parts[0] != "packages" or parts[2] != "src":
62+
continue
63+
pkg = parts[1]
64+
if pkg not in seen:
65+
seen.add(pkg)
66+
ordered.append(pkg)
67+
return ordered
68+
69+
70+
def _packages_already_covered(affected_tests: Iterable[str]) -> Set[str]:
71+
covered: Set[str] = set()
72+
for raw in affected_tests:
73+
path = raw.strip()
74+
if not path:
75+
continue
76+
parts = path.split("/")
77+
if len(parts) >= 3 and parts[0] == "packages" and parts[2] == "tests":
78+
covered.add(parts[1])
79+
return covered
80+
81+
82+
def _first_test_in(tests_dir: Path) -> str | None:
83+
"""Return the alphabetically-first ``test_*.py`` / ``BRANCHES_test_*.py``."""
84+
if not tests_dir.is_dir():
85+
return None
86+
candidates: List[str] = []
87+
for entry in tests_dir.iterdir():
88+
if not entry.is_file():
89+
continue
90+
name = entry.name
91+
if not name.endswith(".py"):
92+
continue
93+
if any(name.startswith(p) for p in _TEST_FILE_PREFIXES):
94+
candidates.append(name)
95+
if not candidates:
96+
return None
97+
return sorted(candidates)[0]
98+
99+
100+
def compute_fallbacks(
101+
*,
102+
repo_root: Path,
103+
changed_source_files: Iterable[str],
104+
affected_tests: Iterable[str],
105+
) -> List[str]:
106+
"""Return per-package fallback test paths, sorted and deduplicated.
107+
108+
Each returned path is relative to ``repo_root``. A path is included
109+
only if (a) the package's source changed, (b) the package is not
110+
already represented in ``affected_tests``, and (c) the package has
111+
at least one ``test_*.py`` / ``BRANCHES_test_*.py`` file.
112+
"""
113+
affected_list = list(affected_tests)
114+
covered = _packages_already_covered(affected_list)
115+
affected_set = {t.strip() for t in affected_list if t.strip()}
116+
117+
fallbacks: Set[str] = set()
118+
for pkg in _affected_packages(changed_source_files):
119+
if pkg in covered:
120+
continue
121+
tests_dir = repo_root / "packages" / pkg / "tests"
122+
first = _first_test_in(tests_dir)
123+
if first is None:
124+
continue
125+
rel = f"packages/{pkg}/tests/{first}"
126+
if rel in affected_set:
127+
continue
128+
fallbacks.add(rel)
129+
return sorted(fallbacks)
130+
131+
132+
def _read_lines(path: Path) -> List[str]:
133+
if not path.exists():
134+
return []
135+
return path.read_text().splitlines()
136+
137+
138+
def main(argv: List[str] | None = None) -> int:
139+
parser = argparse.ArgumentParser(
140+
description=(
141+
"Compute per-package fallback test paths for smart-test. Reads "
142+
"changed source files from stdin and the current affected-tests "
143+
"selection from --tests-file."
144+
)
145+
)
146+
parser.add_argument("repo_root", help="Repository root (paths resolve relative to this)")
147+
parser.add_argument(
148+
"--tests-file",
149+
default=None,
150+
help="Path to current affected-tests file (one path per line). "
151+
"Missing file is treated as an empty selection.",
152+
)
153+
args = parser.parse_args(argv)
154+
155+
repo_root = Path(args.repo_root)
156+
changed = sys.stdin.read().splitlines()
157+
affected = _read_lines(Path(args.tests_file)) if args.tests_file else []
158+
159+
fallbacks = compute_fallbacks(
160+
repo_root=repo_root,
161+
changed_source_files=changed,
162+
affected_tests=affected,
163+
)
164+
for path in fallbacks:
165+
print(path)
166+
return 0
167+
168+
169+
if __name__ == "__main__":
170+
sys.exit(main())

scripts/smart-test

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -657,6 +657,28 @@ if [[ -z "$AFFECTED_TESTS" ]]; then
657657
fi
658658
fi
659659

660+
# Per-package coverage fallback: CI runs each packages/<P>/ in isolation and
661+
# its manifest sanity check requires ≥1 test under packages/<P>/tests/ for
662+
# every package whose source changed. The reverse-slice resolves tests via
663+
# transitive imports, so when a package's only changed source is __init__.py
664+
# (the version bump pattern at every release) and no test imports __init__
665+
# directly, the slice correctly returns zero matches for that package — but
666+
# the sanity check still expects ≥1. The helper below appends the
667+
# alphabetically-first test from any unrepresented affected package's
668+
# tests/ directory. Same lightweight-test rule the version-only branch
669+
# above already uses.
670+
PER_PKG_HELPER="$REPO_ROOT/scripts/per_package_fallback.py"
671+
if [[ -f "$PER_PKG_HELPER" ]] && command -v python3 &>/dev/null && [[ -n "$CHANGED_SOURCE_FILES" ]]; then
672+
PER_PKG_TESTS_FILE=$(mktemp)
673+
echo "$AFFECTED_TESTS" > "$PER_PKG_TESTS_FILE"
674+
PER_PKG_FALLBACKS=$(echo "$CHANGED_SOURCE_FILES" | python3 "$PER_PKG_HELPER" "$REPO_ROOT" --tests-file "$PER_PKG_TESTS_FILE" 2>/dev/null || echo "")
675+
rm -f "$PER_PKG_TESTS_FILE"
676+
if [[ -n "$PER_PKG_FALLBACKS" ]]; then
677+
log "Per-package coverage fallback adding: $(echo "$PER_PKG_FALLBACKS" | tr '\n' ' ')"
678+
AFFECTED_TESTS=$(printf '%s\n%s\n' "$AFFECTED_TESTS" "$PER_PKG_FALLBACKS" | sed '/^$/d' | sort -u)
679+
fi
680+
fi
681+
660682
# Count affected tests
661683
TEST_COUNT=$(echo "$AFFECTED_TESTS" | wc -l)
662684
log "Found $TEST_COUNT affected test files"

0 commit comments

Comments
 (0)