|
4 | 4 | Automatically discovers benchmarks and integrations from the codebase structure. |
5 | 5 | Provides a high-level view of coverage by logical component rather than by file. |
6 | 6 |
|
7 | | -Usage: |
| 7 | +By default, runs all tests except ``credentialed`` and ``smoke`` (i.e. includes |
| 8 | +``slow`` and ``live`` tests that don't need API keys). Use ``--exclude`` to |
| 9 | +skip additional markers. |
| 10 | +
|
| 11 | +Usage:: |
| 12 | +
|
| 13 | + # Full coverage (default + slow + live) |
8 | 14 | uv run python scripts/coverage_by_feature.py |
| 15 | +
|
| 16 | + # Fast-only (skip slow and live tests) |
| 17 | + uv run python scripts/coverage_by_feature.py --exclude slow,live |
9 | 18 | """ |
10 | 19 |
|
| 20 | +import argparse |
11 | 21 | import json |
12 | 22 | import subprocess |
13 | 23 | import sys |
14 | 24 | from pathlib import Path |
15 | 25 | from typing import Dict, List, Set |
16 | 26 |
|
17 | 27 |
|
| 28 | +# Markers that are always excluded (need API keys or are pre-release only) |
| 29 | +ALWAYS_EXCLUDED = ["credentialed", "smoke"] |
| 30 | + |
| 31 | + |
18 | 32 | def discover_benchmarks(maseval_dir: Path) -> List[str]: |
19 | 33 | """Auto-discover benchmark implementations.""" |
20 | 34 | benchmark_dir = maseval_dir / "benchmark" |
@@ -66,14 +80,30 @@ def discover_integrations(maseval_dir: Path) -> Dict[str, Dict[str, List[str]]]: |
66 | 80 | return integrations |
67 | 81 |
|
68 | 82 |
|
69 | | -def run_coverage() -> bool: |
70 | | - """Run pytest with coverage collection.""" |
71 | | - print("Running tests with coverage...") |
72 | | - result = subprocess.run( |
73 | | - ["pytest", "--cov=maseval", "--cov-report=json", "--quiet"], |
74 | | - capture_output=True, |
75 | | - text=True, |
76 | | - ) |
| 83 | +def build_marker_expression(exclude: List[str]) -> str: |
| 84 | + """Build a pytest marker expression from the list of markers to exclude.""" |
| 85 | + all_excluded = ALWAYS_EXCLUDED + [m for m in exclude if m not in ALWAYS_EXCLUDED] |
| 86 | + return "not (" + " or ".join(all_excluded) + ")" |
| 87 | + |
| 88 | + |
| 89 | +def run_coverage(marker_expr: str) -> bool: |
| 90 | + """Run pytest with coverage collection. |
| 91 | +
|
| 92 | + Args: |
| 93 | + marker_expr: Pytest marker expression (passed via -m). |
| 94 | + """ |
| 95 | + cmd = [ |
| 96 | + "pytest", |
| 97 | + "--override-ini=addopts=", |
| 98 | + "-m", |
| 99 | + marker_expr, |
| 100 | + "--cov=maseval", |
| 101 | + "--cov-report=json", |
| 102 | + "--quiet", |
| 103 | + ] |
| 104 | + |
| 105 | + print(f"Running tests with coverage (-m '{marker_expr}') ...") |
| 106 | + result = subprocess.run(cmd, capture_output=True, text=True) |
77 | 107 | if result.returncode != 0: |
78 | 108 | print("\nTests failed:") |
79 | 109 | print(result.stdout) |
@@ -133,13 +163,32 @@ def format_coverage(label: str, stats: Dict[str, float], indent: int = 0) -> str |
133 | 163 | return f"{indent_str}{label:<30} {color}{percent:6.2f}%{reset} ({stats['covered']}/{stats['total']} lines)" |
134 | 164 |
|
135 | 165 |
|
| 166 | +def parse_args() -> argparse.Namespace: |
| 167 | + """Parse command-line arguments.""" |
| 168 | + parser = argparse.ArgumentParser( |
| 169 | + description="Generate test coverage report organized by feature area.", |
| 170 | + ) |
| 171 | + parser.add_argument( |
| 172 | + "--exclude", |
| 173 | + type=str, |
| 174 | + default="", |
| 175 | + help="Comma-separated markers to exclude (e.g. 'slow,live'). 'credentialed' and 'smoke' are always excluded.", |
| 176 | + ) |
| 177 | + return parser.parse_args() |
| 178 | + |
| 179 | + |
136 | 180 | def main(): |
137 | 181 | """Generate coverage report by feature area.""" |
| 182 | + args = parse_args() |
138 | 183 | repo_root = Path(__file__).parent.parent |
139 | 184 | maseval_dir = repo_root / "maseval" |
140 | 185 |
|
| 186 | + # Build marker expression |
| 187 | + extra_excludes = [m.strip() for m in args.exclude.split(",") if m.strip()] |
| 188 | + marker_expr = build_marker_expression(extra_excludes) |
| 189 | + |
141 | 190 | # Run coverage |
142 | | - if not run_coverage(): |
| 191 | + if not run_coverage(marker_expr): |
143 | 192 | print("\nTests failed. Coverage report may be incomplete.") |
144 | 193 |
|
145 | 194 | print("\n" + "=" * 80) |
|
0 commit comments