Skip to content

Commit f5e02b0

Browse files
committed
feat(evals): add eval:coverage human report command
1 parent b144164 commit f5e02b0

4 files changed

Lines changed: 856 additions & 0 deletions

File tree

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
"docs:keybindings": "tsx ./scripts/generate-keybindings-doc.ts",
3535
"eval:inventory": "tsx ./scripts/eval-inventory-cli.ts",
3636
"eval:inventory:json": "tsx ./scripts/eval-inventory-cli.ts --json",
37+
"eval:coverage": "tsx ./scripts/eval-coverage-cli.ts",
3738
"build": "node scripts/build.js",
3839
"build-and-start": "npm run build && npm run start --",
3940
"build:vscode": "node scripts/build_vscode_companion.js",

scripts/eval-coverage-cli.ts

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env tsx
2+
3+
/**
4+
* @license
5+
* Copyright 2026 Google LLC
6+
* SPDX-License-Identifier: Apache-2.0
7+
*/
8+
9+
import { collectInventory } from './utils/eval-inventory.js';
10+
import { buildToolRegistry } from './utils/tool-registry.js';
11+
import {
12+
computeCoverage,
13+
formatCoverageReport,
14+
} from './utils/eval-coverage.js';
15+
16+
async function main() {
17+
const rootFlagIndex = process.argv.indexOf('--root');
18+
const rootFlagValue =
19+
rootFlagIndex !== -1 ? process.argv[rootFlagIndex + 1] : undefined;
20+
21+
if (rootFlagIndex !== -1 && rootFlagValue === undefined) {
22+
console.error(
23+
'Error: --root requires a directory path argument but none was provided.',
24+
);
25+
process.exit(1);
26+
}
27+
if (rootFlagValue && rootFlagValue.startsWith('--')) {
28+
console.error(
29+
`Error: --root value "${rootFlagValue}" looks like a flag. Provide a valid directory path.`,
30+
);
31+
process.exit(1);
32+
}
33+
34+
const repoRoot = rootFlagValue ?? process.cwd();
35+
const inventory = await collectInventory(repoRoot);
36+
37+
if (inventory.totalFiles === 0) {
38+
console.error('No eval files found under evals/.');
39+
process.exit(1);
40+
}
41+
42+
const registry = buildToolRegistry();
43+
const result = computeCoverage(inventory, registry);
44+
45+
console.log(formatCoverageReport(result));
46+
}
47+
48+
main().catch((error) => {
49+
console.error('Fatal error:', error);
50+
process.exit(1);
51+
});

0 commit comments

Comments
 (0)