Skip to content

Commit c7dc070

Browse files
SutuSebastianSutuSebastian
committed
test(agent-eval): one probe per capability group on in-repo bench
Expand probes to 18 golden-backed scenarios, add capability-probes.test.mjs guard, and fix live MCP smoke to assert find-call-sites by id not index.
1 parent 2bd8071 commit c7dc070

5 files changed

Lines changed: 155 additions & 16 deletions

File tree

‎docs/plans/in-repo-test-bench.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ fixtures/
4949

5050
- [x] `src/cli/cmd-test-bench-e2e.test.ts` — show, snippet, impact, validate, `shop-symbols`, SARIF (`boundary-violations`)
5151
- [x] `cmd-cli-parity-e2e.test.ts` — trace, explore, node, context, batch, resources
52-
- [ ] Expand `test:agent-eval` to one probe per `CAPABILITIES.json` group (6/16 today)
52+
- [x] Expand `test:agent-eval` to one probe per `CAPABILITIES.json` group (18 probes; guarded by `capability-probes.test.mjs`)
5353

5454
### Phase 4 — Scale (optional, replaces Tier B′)
5555

‎docs/testing-coverage.md‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -81,16 +81,7 @@ Codemap development uses **only** the committed bench ([fixtures/README.md](../f
8181

8282
## Agent eval probes
8383

84-
`scripts/agent-eval/scenarios.json` maps probes → golden ids:
85-
86-
| Probe | Golden id |
87-
| --------------------- | ---------------------------- |
88-
| Symbol definition | `symbol-usePermissions` |
89-
| Dependency edges | `dependencies-from-consumer` |
90-
| Call sites | `find-call-sites` |
91-
| Call resolution stats | `call-resolution-stats` |
92-
| Import specifiers | `import-specifiers-consumer` |
93-
| Project-local recipe | `shop-symbols-recipe` |
84+
`scripts/agent-eval/scenarios.json` — **18 probes**, one golden id per `CAPABILITIES.json` group (except bundled-recipe guard + CLI-only groups). Enforced by `scripts/agent-eval/capability-probes.test.mjs` in `test:scripts`.
9485

9586
---
9687

‎scripts/agent-eval/capability-probes.test.mjs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import { describe, expect, it } from "bun:test";
2+
/**
3+
* Each CAPABILITIES.json group with goldenScenarios must have ≥1 agent-eval probe
4+
* pointing at one of those scenario ids.
5+
*/
6+
import { readFileSync } from "node:fs";
7+
import { join } from "node:path";
8+
9+
import { parseProbesJson } from "./schema.ts";
10+
11+
const REPO_ROOT = join(import.meta.dir, "../..");
12+
const CAPABILITIES_PATH = join(REPO_ROOT, "fixtures/CAPABILITIES.json");
13+
const PROBES_PATH = join(import.meta.dir, "scenarios.json");
14+
15+
const SKIP_CAPABILITY = new Set([
16+
"recipes.bundled",
17+
"cli.bench-smoke",
18+
"cli.mcp.http",
19+
]);
20+
21+
describe("agent-eval capability probe coverage", () => {
22+
it("every capability with goldens has a probe referencing one of its scenario ids", () => {
23+
const manifest = JSON.parse(readFileSync(CAPABILITIES_PATH, "utf-8"));
24+
const { probes } = parseProbesJson(readFileSync(PROBES_PATH, "utf-8"));
25+
const probeGoldenIds = new Set(probes.map((p) => p.goldenId));
26+
27+
const missing = [];
28+
for (const cap of manifest.capabilities ?? []) {
29+
if (SKIP_CAPABILITY.has(cap.id)) continue;
30+
const goldens = cap.goldenScenarios ?? [];
31+
if (goldens.length === 0) continue;
32+
const covered = goldens.some((id) => probeGoldenIds.has(id));
33+
if (!covered) missing.push(cap.id);
34+
}
35+
expect(missing).toEqual([]);
36+
});
37+
});

‎scripts/agent-eval/parse-agent-log.test.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -588,7 +588,10 @@ describe("run-probes smoke", () => {
588588
).probes.length;
589589
expect(parsed.scenarios).toHaveLength(probeCount);
590590
expect(parsed.summary.successCount).toBe(probeCount);
591-
expect(parsed.scenarios[2]!.mcpOn.toolSequence).toEqual(["query_recipe"]);
591+
const findCallSites = parsed.scenarios.find(
592+
(s) => s.id === "find-call-sites",
593+
);
594+
expect(findCallSites?.mcpOn.toolSequence).toEqual(["query_recipe"]);
592595
} finally {
593596
rmSync(tmp, { recursive: true, force: true });
594597
}

‎scripts/agent-eval/scenarios.json‎

Lines changed: 112 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
{
22
"version": 1,
33
"probes": [
4+
{
5+
"id": "index-table-stats",
6+
"goldenId": "index-table-stats",
7+
"traditional": {
8+
"globs": ["**/*.{ts,tsx,md,css}"],
9+
"regex": ".",
10+
"mode": "files"
11+
}
12+
},
413
{
514
"id": "symbol-usePermissions",
615
"goldenId": "symbol-usePermissions",
@@ -19,6 +28,24 @@
1928
"mode": "files"
2029
}
2130
},
31+
{
32+
"id": "import-specifiers-consumer",
33+
"goldenId": "import-specifiers-consumer",
34+
"traditional": {
35+
"globs": ["src/consumer.ts"],
36+
"regex": "import\\s+\\{[^}]+\\}\\s+from",
37+
"mode": "matches"
38+
}
39+
},
40+
{
41+
"id": "bindings-createClient",
42+
"goldenId": "bindings-createClient",
43+
"traditional": {
44+
"globs": ["**/*.{ts,tsx}"],
45+
"regex": "createClient",
46+
"mode": "matches"
47+
}
48+
},
2249
{
2350
"id": "find-call-sites",
2451
"goldenId": "find-call-sites",
@@ -38,11 +65,92 @@
3865
}
3966
},
4067
{
41-
"id": "import-specifiers-consumer",
42-
"goldenId": "import-specifiers-consumer",
68+
"id": "unresolved-call-sites",
69+
"goldenId": "unresolved-call-sites",
4370
"traditional": {
44-
"globs": ["src/consumer.ts"],
45-
"regex": "import\\s+\\{[^}]+\\}\\s+from",
71+
"globs": ["src/lib/complexity-fixture.ts"],
72+
"regex": "relay\\s*\\(",
73+
"mode": "matches"
74+
}
75+
},
76+
{
77+
"id": "type-ancestors-dog",
78+
"goldenId": "type-ancestors-dog",
79+
"traditional": {
80+
"globs": ["src/types/hierarchy.ts"],
81+
"regex": "extends\\s+\\w+",
82+
"mode": "matches"
83+
}
84+
},
85+
{
86+
"id": "find-jsx-usages",
87+
"goldenId": "find-jsx-usages",
88+
"traditional": {
89+
"globs": ["**/*.{tsx,jsx}"],
90+
"regex": "<[A-Za-z]",
91+
"mode": "matches"
92+
}
93+
},
94+
{
95+
"id": "css-variables",
96+
"goldenId": "css-variables",
97+
"traditional": {
98+
"globs": ["**/*.css"],
99+
"regex": "--[\\w-]+",
100+
"mode": "matches"
101+
}
102+
},
103+
{
104+
"id": "circular-imports",
105+
"goldenId": "circular-imports",
106+
"traditional": {
107+
"globs": ["src/lib/*.ts"],
108+
"regex": "from\\s+['\"]\\./",
109+
"mode": "files"
110+
}
111+
},
112+
{
113+
"id": "find-swallowed-errors",
114+
"goldenId": "find-swallowed-errors",
115+
"traditional": {
116+
"globs": ["src/lib/cache.ts"],
117+
"regex": "catch\\s*\\(",
118+
"mode": "matches"
119+
}
120+
},
121+
{
122+
"id": "untested-and-dead",
123+
"goldenId": "untested-and-dead",
124+
"traditional": {
125+
"globs": ["src/**/*.ts"],
126+
"regex": "export\\s+function",
127+
"mode": "files"
128+
}
129+
},
130+
{
131+
"id": "boundary-violations",
132+
"goldenId": "boundary-violations",
133+
"traditional": {
134+
"globs": ["src/components/**/*.tsx"],
135+
"regex": "from\\s+['\"]~?/api/",
136+
"mode": "matches"
137+
}
138+
},
139+
{
140+
"id": "source-fts-row-count",
141+
"goldenId": "source-fts-row-count",
142+
"traditional": {
143+
"globs": ["**/*.{ts,tsx,md,css}"],
144+
"regex": ".",
145+
"mode": "files"
146+
}
147+
},
148+
{
149+
"id": "find-skipped-tests",
150+
"goldenId": "find-skipped-tests",
151+
"traditional": {
152+
"globs": ["src/__tests__/**/*.ts"],
153+
"regex": "\\.(skip|only)\\s*\\(",
46154
"mode": "matches"
47155
}
48156
},

0 commit comments

Comments
 (0)