Skip to content

Commit a9cee96

Browse files
deepakdgupta1deepakdgupta1
committed
Document v2.3 completion and add eval smoke baseline
1 parent bb373c4 commit a9cee96

10 files changed

Lines changed: 467 additions & 25 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -476,7 +476,7 @@ See [reference_architecture.md](file:///Users/deepg/Desktop/KnowCode/docs/archit
476476
- ✅ Fix `metadata` type restriction (`dict[str, str]` → `dict[str, Any]`)
477477
- ✅ Harden configuration loading (logging, validation, strict server mode)
478478
- ✅ Decompose `KnowCodeService` and introduce `Protocol` interfaces
479-
- ✅ Add layer contract tests and harden retrieval evals (parser, store roundtrip, golden queries - see [docs/retrieval-evals.md](docs/retrieval-evals.md))
479+
- ✅ Add layer contract tests and harden retrieval evals (parser, store roundtrip, golden-query smoke baseline - see [docs/retrieval-evals.md](docs/retrieval-evals.md))
480480

481481
**Future releases:**
482482
- v2.4: Multi-level documentation synthesis

‎docs/index.md‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -357,14 +357,14 @@ See [reference_architecture.md](file:///Users/deepg/Desktop/KnowCode/docs/archit
357357
- MCP server for IDE integration
358358
- VoyageAI cross-encoder reranking
359359

360-
**Next: v2.3 — Architectural Hardening:**
360+
**v2.3 — Architectural Hardening (completed):**
361361
- ✅ Modularise dependencies into optional extras (core install stays lightweight)
362-
- Remove hidden side effects from query paths (fail fast, not auto-build)
363-
- ✅ Schema versioning on knowledge store (`schema_version: 2` is live); FAISS index metadata versioning still pending
364-
- Fix `metadata` type restriction (`dict[str, str]` → `dict[str, Any]`)
365-
- Harden configuration loading (logging, validation, strict server mode)
366-
- Decompose `KnowCodeService` and introduce `Protocol` interfaces
367-
- ✅ Add layer contract tests (parser, store roundtrip, retrieval golden queries — see [docs/retrieval-evals.md](retrieval-evals.md))
362+
- ✅ Remove hidden side effects from query paths (fail fast, not auto-build)
363+
- ✅ Schema versioning on persisted knowledge store and index artifacts
364+
- ✅ Fix `metadata` type restriction (`dict[str, str]` → `dict[str, Any]`)
365+
- ✅ Harden configuration loading (logging, validation, strict server mode)
366+
- ✅ Decompose `KnowCodeService` and introduce `Protocol` interfaces
367+
- ✅ Add layer contract tests and harden retrieval evals (parser, store roundtrip, golden-query smoke baseline — see [docs/retrieval-evals.md](retrieval-evals.md))
368368

369369
**Future releases:**
370370
- v2.4: Multi-level documentation synthesis

‎docs/retrieval-evals.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,18 @@ The golden dataset is stored in [golden_v1.0.json](file:///Users/deepg/Desktop/K
2424

2525
Metadata for the dataset version is stored in [golden_v1.0.meta.json](file:///Users/deepg/Desktop/KnowCode/tests/eval/golden/golden_v1.0.meta.json) to guard against codebase line range drift.
2626

27+
**Current status:** `golden_v1.0.json` is a committed smoke baseline, not the
28+
full Phase 1 dataset. Its metadata is marked with
29+
`"dataset_status": "smoke_baseline"` and includes per-source-file hashes so
30+
unrelated commits do not invalidate the eval gate. The full 60-record sampling
31+
plan is committed at
32+
[phase1_plan.json](file:///Users/deepg/Desktop/KnowCode/tests/eval/pipeline/phase1_plan.json).
33+
2734
### How to Add a New Query
2835
1. Open [golden_v1.0.json](file:///Users/deepg/Desktop/KnowCode/tests/eval/golden/golden_v1.0.json).
2936
2. Append a new object specifying the `query_id`, `query_text`, `task_type`, `difficulty`, `expected_entities` (as relative project paths `file_path::symbol`), and `expected_files` (as relative file paths).
3037
3. Increment the `query_count` in [golden_v1.0.meta.json](file:///Users/deepg/Desktop/KnowCode/tests/eval/golden/golden_v1.0.meta.json).
38+
4. Add or refresh any affected `source_file_hashes` in the metadata file.
3139

3240
---
3341

‎tests/eval/golden/baseline_v1.0.json‎

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
{
2+
"scores": [
3+
{
4+
"query_id": "q1",
5+
"task_type": "locate",
6+
"difficulty": "easy",
7+
"precision_at_1": 1.0,
8+
"precision_at_5": 0.4,
9+
"recall_at_10": 1.0,
10+
"mrr": 1.0,
11+
"file_coverage_at_1": 1.0,
12+
"file_coverage_at_5": 1.0,
13+
"sufficiency_score": 0.64,
14+
"routed_local": false,
15+
"narrative": {
16+
"must_mention_verdicts": [],
17+
"must_not_mention_verdicts": [],
18+
"narrative_score": null
19+
}
20+
},
21+
{
22+
"query_id": "q2",
23+
"task_type": "explain",
24+
"difficulty": "medium",
25+
"precision_at_1": 1.0,
26+
"precision_at_5": 0.2,
27+
"recall_at_10": 1.0,
28+
"mrr": 1.0,
29+
"file_coverage_at_1": 1.0,
30+
"file_coverage_at_5": 1.0,
31+
"sufficiency_score": 0.65,
32+
"routed_local": false,
33+
"narrative": {
34+
"must_mention_verdicts": [],
35+
"must_not_mention_verdicts": [],
36+
"narrative_score": null
37+
}
38+
},
39+
{
40+
"query_id": "q3",
41+
"task_type": "debug",
42+
"difficulty": "hard",
43+
"precision_at_1": 0.0,
44+
"precision_at_5": 0.4,
45+
"recall_at_10": 1.0,
46+
"mrr": 0.3333333333333333,
47+
"file_coverage_at_1": 0.0,
48+
"file_coverage_at_5": 1.0,
49+
"sufficiency_score": 0.26,
50+
"routed_local": false,
51+
"narrative": {
52+
"must_mention_verdicts": [],
53+
"must_not_mention_verdicts": [],
54+
"narrative_score": null
55+
}
56+
},
57+
{
58+
"query_id": "q4",
59+
"task_type": "dependency-traversal",
60+
"difficulty": "medium",
61+
"precision_at_1": 1.0,
62+
"precision_at_5": 0.2,
63+
"recall_at_10": 0.5,
64+
"mrr": 1.0,
65+
"file_coverage_at_1": 0.5,
66+
"file_coverage_at_5": 0.5,
67+
"sufficiency_score": 0.73,
68+
"routed_local": false,
69+
"narrative": {
70+
"must_mention_verdicts": [],
71+
"must_not_mention_verdicts": [],
72+
"narrative_score": null
73+
}
74+
},
75+
{
76+
"query_id": "q5",
77+
"task_type": "locate",
78+
"difficulty": "easy",
79+
"precision_at_1": 1.0,
80+
"precision_at_5": 0.2,
81+
"recall_at_10": 1.0,
82+
"mrr": 1.0,
83+
"file_coverage_at_1": 1.0,
84+
"file_coverage_at_5": 1.0,
85+
"sufficiency_score": 0.96,
86+
"routed_local": true,
87+
"narrative": {
88+
"must_mention_verdicts": [
89+
{
90+
"fact": "The freshness endpoint is implemented by `get_freshness`.",
91+
"verdict": null
92+
},
93+
{
94+
"fact": "The endpoint delegates to `service.get_freshness_metadata()`.",
95+
"verdict": null
96+
}
97+
],
98+
"must_not_mention_verdicts": [
99+
{
100+
"fact": "The endpoint rebuilds the knowledge store or index.",
101+
"verdict": null
102+
}
103+
],
104+
"narrative_score": null
105+
},
106+
"correct": true
107+
},
108+
{
109+
"query_id": "q6",
110+
"task_type": "explain",
111+
"difficulty": "medium",
112+
"precision_at_1": 1.0,
113+
"precision_at_5": 0.4,
114+
"recall_at_10": 1.0,
115+
"mrr": 1.0,
116+
"file_coverage_at_1": 1.0,
117+
"file_coverage_at_5": 1.0,
118+
"sufficiency_score": 0.86,
119+
"routed_local": true,
120+
"narrative": {
121+
"must_mention_verdicts": [
122+
{
123+
"fact": "`run_doctor` invokes `_check_freshness` as part of the doctor checklist.",
124+
"verdict": null
125+
},
126+
{
127+
"fact": "`_check_freshness` calls `service.get_freshness_metadata()` and reports a warning when artifacts are stale.",
128+
"verdict": null
129+
}
130+
],
131+
"must_not_mention_verdicts": [
132+
{
133+
"fact": "The doctor command silently rebuilds stale artifacts.",
134+
"verdict": null
135+
}
136+
],
137+
"narrative_score": null
138+
},
139+
"correct": true
140+
}
141+
],
142+
"summary": {
143+
"n": 6,
144+
"mean_precision_at_1": 0.8333333333333334,
145+
"mean_precision_at_5": 0.3,
146+
"mean_recall_at_10": 0.9166666666666666,
147+
"mean_mrr": 0.888888888888889,
148+
"mean_file_coverage_at_1": 0.75,
149+
"mean_file_coverage_at_5": 0.9166666666666666,
150+
"mean_sufficiency_score": 0.6833333333333335,
151+
"local_routing_rate": 0.3333333333333333,
152+
"by_task_type": {
153+
"locate": {
154+
"n": 2,
155+
"mean_precision_at_1": 1.0,
156+
"mean_precision_at_5": 0.30000000000000004,
157+
"mean_recall_at_10": 1.0,
158+
"mean_mrr": 1.0,
159+
"mean_file_coverage_at_1": 1.0,
160+
"mean_file_coverage_at_5": 1.0,
161+
"mean_sufficiency_score": 0.8,
162+
"local_routing_rate": 0.5
163+
},
164+
"explain": {
165+
"n": 2,
166+
"mean_precision_at_1": 1.0,
167+
"mean_precision_at_5": 0.30000000000000004,
168+
"mean_recall_at_10": 1.0,
169+
"mean_mrr": 1.0,
170+
"mean_file_coverage_at_1": 1.0,
171+
"mean_file_coverage_at_5": 1.0,
172+
"mean_sufficiency_score": 0.755,
173+
"local_routing_rate": 0.5
174+
},
175+
"debug": {
176+
"n": 1,
177+
"mean_precision_at_1": 0.0,
178+
"mean_precision_at_5": 0.4,
179+
"mean_recall_at_10": 1.0,
180+
"mean_mrr": 0.3333333333333333,
181+
"mean_file_coverage_at_1": 0.0,
182+
"mean_file_coverage_at_5": 1.0,
183+
"mean_sufficiency_score": 0.26,
184+
"local_routing_rate": 0.0
185+
},
186+
"dependency-traversal": {
187+
"n": 1,
188+
"mean_precision_at_1": 1.0,
189+
"mean_precision_at_5": 0.2,
190+
"mean_recall_at_10": 0.5,
191+
"mean_mrr": 1.0,
192+
"mean_file_coverage_at_1": 0.5,
193+
"mean_file_coverage_at_5": 0.5,
194+
"mean_sufficiency_score": 0.73,
195+
"local_routing_rate": 0.0
196+
}
197+
},
198+
"by_difficulty": {
199+
"easy": {
200+
"n": 2,
201+
"mean_precision_at_1": 1.0,
202+
"mean_precision_at_5": 0.30000000000000004,
203+
"mean_recall_at_10": 1.0,
204+
"mean_mrr": 1.0,
205+
"mean_file_coverage_at_1": 1.0,
206+
"mean_file_coverage_at_5": 1.0,
207+
"mean_sufficiency_score": 0.8,
208+
"local_routing_rate": 0.5
209+
},
210+
"medium": {
211+
"n": 3,
212+
"mean_precision_at_1": 1.0,
213+
"mean_precision_at_5": 0.26666666666666666,
214+
"mean_recall_at_10": 0.8333333333333334,
215+
"mean_mrr": 1.0,
216+
"mean_file_coverage_at_1": 0.8333333333333334,
217+
"mean_file_coverage_at_5": 0.8333333333333334,
218+
"mean_sufficiency_score": 0.7466666666666666,
219+
"local_routing_rate": 0.3333333333333333
220+
},
221+
"hard": {
222+
"n": 1,
223+
"mean_precision_at_1": 0.0,
224+
"mean_precision_at_5": 0.4,
225+
"mean_recall_at_10": 1.0,
226+
"mean_mrr": 0.3333333333333333,
227+
"mean_file_coverage_at_1": 0.0,
228+
"mean_file_coverage_at_5": 1.0,
229+
"mean_sufficiency_score": 0.26,
230+
"local_routing_rate": 0.0
231+
}
232+
}
233+
}
234+
}

‎tests/eval/golden/golden_v1.0.json‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,5 +54,40 @@
5454
],
5555
"must_mention_facts": [],
5656
"must_not_mention_facts": []
57+
},
58+
{
59+
"query_id": "q5",
60+
"query_text": "Where is the API freshness endpoint implemented?",
61+
"task_type": "locate",
62+
"difficulty": "easy",
63+
"expected_entities": ["src/knowcode/api/api.py::get_freshness"],
64+
"expected_files": ["src/knowcode/api/api.py"],
65+
"must_mention_facts": [
66+
"The freshness endpoint is implemented by `get_freshness`.",
67+
"The endpoint delegates to `service.get_freshness_metadata()`."
68+
],
69+
"must_not_mention_facts": [
70+
"The endpoint rebuilds the knowledge store or index."
71+
],
72+
"correct": true
73+
},
74+
{
75+
"query_id": "q6",
76+
"query_text": "Explain how the doctor command checks freshness.",
77+
"task_type": "explain",
78+
"difficulty": "medium",
79+
"expected_entities": [
80+
"src/knowcode/doctor.py::_check_freshness",
81+
"src/knowcode/doctor.py::run_doctor"
82+
],
83+
"expected_files": ["src/knowcode/doctor.py"],
84+
"must_mention_facts": [
85+
"`run_doctor` invokes `_check_freshness` as part of the doctor checklist.",
86+
"`_check_freshness` calls `service.get_freshness_metadata()` and reports a warning when artifacts are stale."
87+
],
88+
"must_not_mention_facts": [
89+
"The doctor command silently rebuilds stale artifacts."
90+
],
91+
"correct": true
5792
}
5893
]

‎tests/eval/golden/golden_v1.0.meta.json‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
11
{
2-
"source_sha": "1cb6768c7035068f42cc8c2dc159f06b4d3d26be",
3-
"created_at": "2026-06-06T19:33:00Z",
2+
"source_sha": "bb373c43a5f20c775d0c2b0ea592372dbe9feb20",
3+
"created_at": "2026-06-10T15:20:00Z",
44
"schema_version": "1.0.0",
5-
"query_count": 4
5+
"dataset_status": "smoke_baseline",
6+
"query_count": 6,
7+
"source_file_hashes": {
8+
"src/knowcode/api/api.py": "cb0ce6411ff0811c6f0e3d35052202693d0e5ef66285c66924718bd0539a0d13",
9+
"src/knowcode/doctor.py": "2d5fd42b5cd0fe36691917f34eb04c130e3261806333d76a12aff6fbcdee6117",
10+
"src/knowcode/indexing/monitor.py": "84e8453161f0db615100d7d6256e4c3a5a5cbaac3b89bfba3081507a2164f8eb",
11+
"src/knowcode/indexing/scanner.py": "739509e9e0c3781d3c4779f592bfa7ad87b0ff26b39198512dde6fb8471f7dcd",
12+
"src/knowcode/parsers/markdown_parser.py": "7e7c0981f74a20300c255a758b1d35a75e2a9e2e106dff074d675795168b6e91",
13+
"tests/eval/fixtures/mini_repo/module_a.py": "84ca71356a2fed4896569839c0c4126b3c1a45f298933f60cc1cbcaaad27986b",
14+
"tests/eval/fixtures/mini_repo/module_b.py": "e11d5c43b2ff79a23d7f8af1a343ddfc4a35a10327e5d242fcae87f943b3ee94"
15+
}
616
}

‎tests/eval/harness/scorer.py‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -248,20 +248,23 @@ def score_record(
248248
sufficiency: float = float(retrieval_result.get("sufficiency_score", 0.0))
249249
routed_local = _would_route_local(retrieval_result, sufficiency)
250250

251-
return {
251+
precision_1 = precision_at_k(retrieved_entities, expected_entity_ids, k=1)
252+
precision_5 = precision_at_k(retrieved_entities, expected_entity_ids, k=5)
253+
recall_10 = recall_at_k(retrieved_entities, expected_entity_ids, k=10)
254+
mrr = reciprocal_rank(retrieved_entities, expected_entity_ids)
255+
file_coverage_1 = file_coverage_at_k(retrieved_entities, expected_files, k=1)
256+
file_coverage_5 = file_coverage_at_k(retrieved_entities, expected_files, k=5)
257+
258+
result = {
252259
"query_id": query_id,
253260
"task_type": task_type,
254261
"difficulty": difficulty,
255-
"precision_at_1": precision_at_k(retrieved_entities, expected_entity_ids, k=1),
256-
"precision_at_5": precision_at_k(retrieved_entities, expected_entity_ids, k=5),
257-
"recall_at_10": recall_at_k(retrieved_entities, expected_entity_ids, k=10),
258-
"mrr": reciprocal_rank(retrieved_entities, expected_entity_ids),
259-
"file_coverage_at_1": file_coverage_at_k(
260-
retrieved_entities, expected_files, k=1
261-
),
262-
"file_coverage_at_5": file_coverage_at_k(
263-
retrieved_entities, expected_files, k=5
264-
),
262+
"precision_at_1": precision_1,
263+
"precision_at_5": precision_5,
264+
"recall_at_10": recall_10,
265+
"mrr": mrr,
266+
"file_coverage_at_1": file_coverage_1,
267+
"file_coverage_at_5": file_coverage_5,
265268
"sufficiency_score": sufficiency,
266269
"routed_local": routed_local,
267270
"narrative": score_narrative(
@@ -272,6 +275,13 @@ def score_record(
272275
),
273276
}
274277

278+
judged_correct = golden.get("correct")
279+
if isinstance(judged_correct, bool):
280+
structural_match = recall_10 == 1.0 and file_coverage_5 == 1.0
281+
result["correct"] = bool(judged_correct and structural_match)
282+
283+
return result
284+
275285

276286
# ---------------------------------------------------------------------------
277287
# Aggregate metrics

0 commit comments

Comments
 (0)