-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathsemantic_quality_set.json
More file actions
196 lines (196 loc) · 7.17 KB
/
Copy pathsemantic_quality_set.json
File metadata and controls
196 lines (196 loc) · 7.17 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
{
"metadata": {
"created": "2026-04-17",
"status": "historical_closed",
"closure_date": "2026-04-17",
"bridge_db_snapshot": "not_filled_historical_artifact",
"total_queries": 20,
"rater_primary": "operator",
"rater_secondary": "claude-code-blind",
"inter_rater_sample_size": 5,
"inter_rater_agreement": null,
"notes": "Historical artifact only. The semantic/vector roadmap was closed after the Phase -1 FTS5 recall dry run showed misses mostly targeted content outside bridge-db scope. expected_results remain empty by design; do not treat this file as live eval work unless the roadmap is explicitly reopened."
},
"queries": [
{
"id": "q001",
"category": "new_project_arrival",
"query": "starting a new mcp server for internal tooling",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Should surface mcpforge, bridge-db itself, and the enterprise IT MCP gap research"
},
{
"id": "q002",
"category": "new_project_arrival",
"query": "building a tauri desktop app with rust backend",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Broad but high-signal — AssistSupport, apispy, Codec, Network Protocol Decoder are all Tauri 2 + Rust"
},
{
"id": "q003",
"category": "new_project_arrival",
"query": "new explorable explanation project",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Should pull How Regex Works, How Memory Allocation Works, Neural Network Playground, GPU Pipeline Explorer"
},
{
"id": "q004",
"category": "new_project_arrival",
"query": "ios app with cloudkit sync",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Should surface Calibrate (explicit CloudKit), Ghost Routes, Nocturne; narrower than q002 on purpose"
},
{
"id": "q005",
"category": "new_project_arrival",
"query": "terraform for multi-account aws setup at work",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": true,
"notes": "Terraform is on the operator's learning path but bridge-db has no content on multi-account Terraform setups specifically. Testing empty-result path."
},
{
"id": "q006",
"category": "mid_session_deja_vu",
"query": "why did we pick sqlite-vec over chromadb",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Decision from this very plan (v1/v2) — tests whether the plan itself, once committed as a context section, surfaces"
},
{
"id": "q007",
"category": "mid_session_deja_vu",
"query": "bloomberry mcp server count 1482",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Specific factoid from MCP ecosystem research — exact-number recall test"
},
{
"id": "q008",
"category": "mid_session_deja_vu",
"query": "what was the sonnet vs opus cost decision",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Should surface 'Sonnet = Opus quality on well-specified tasks at ~60% cost' snapshot"
},
{
"id": "q009",
"category": "mid_session_deja_vu",
"query": "reasons to not publish the build log yet",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Should surface the infrastructure-mode vs publishing-mode decision from recent updates"
},
{
"id": "q010",
"category": "mid_session_deja_vu",
"query": "platform engineer reframe vs sre",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Should surface the Platform Engineer career-transition research synthesis"
},
{
"id": "q011",
"category": "cross_system_coordination",
"query": "what did claude code ship yesterday",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Temporal + system filter — tests whether activity entries are semantically findable by recency phrasing"
},
{
"id": "q012",
"category": "cross_system_coordination",
"query": "codex bridge sync status shipped events",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Should surface recent shipped-event activity from Codex via get_shipped_events tool outputs"
},
{
"id": "q013",
"category": "cross_system_coordination",
"query": "pending handoff for apispy or network protocol decoder",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Should surface any pending-handoff sections written via Auto Project Dispatch"
},
{
"id": "q014",
"category": "cross_system_coordination",
"query": "which projects has claude code touched this week",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Open-ended temporal query — tests recency-ranking without explicit time filter in the tool"
},
{
"id": "q015",
"category": "cross_system_coordination",
"query": "codex deep research query packs for geopolitics domain",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": true,
"notes": "NotebookLM Deep Research query packs live in Notion, not bridge-db. Testing empty-result path on a plausible-but-wrong-system query."
},
{
"id": "q016",
"category": "pattern_re_use",
"query": "keychain storage for daemon api keys",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Pattern that shows up in bridge-db plan + likely past AssistSupport or other local-daemon work"
},
{
"id": "q017",
"category": "pattern_re_use",
"query": "how do i structure an implementation roadmap for claude code",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Should surface the vibe-code-handoff skill outputs or past CLAUDE.md + IMPLEMENTATION-ROADMAP pairs"
},
{
"id": "q018",
"category": "pattern_re_use",
"query": "ml accuracy benchmark for support ticket classifier",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Should pull AssistSupport's 85.7% accuracy snapshot and any follow-up classifier work"
},
{
"id": "q019",
"category": "pattern_re_use",
"query": "react artifact with persistent storage via window.storage",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Should surface artifact-building skill outputs or Command Center v6 context"
},
{
"id": "q020",
"category": "pattern_re_use",
"query": "hook pattern for invalidating cached state on write",
"query_written_before_viewing_db": true,
"expected_results": [],
"expected_empty": false,
"notes": "Matches Decision 10 of this plan but also likely shows up in past Claude Code hook work and skill-forge patterns"
}
]
}