Skip to content

Commit d0b0e4c

Browse files
PatrickSysPatrickSys
committed
Run ContextBench codebase-memory recovery on CI
1 parent f1d322a commit d0b0e4c

1 file changed

Lines changed: 270 additions & 0 deletions

File tree

‎.github/workflows/contextbench-ci-recovery-master.yml‎

Lines changed: 270 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,270 @@
1+
name: ContextBench CI Recovery Master
2+
3+
on:
4+
push:
5+
branches: [master]
6+
paths:
7+
- .github/workflows/contextbench-ci-recovery-master.yml
8+
workflow_dispatch:
9+
inputs:
10+
max_tasks:
11+
description: 'Number of first tasks to run for codebase-memory readiness'
12+
required: true
13+
default: '3'
14+
codebase_memory_version:
15+
description: 'codebase-memory-mcp release tag'
16+
required: true
17+
default: 'v0.6.1'
18+
19+
permissions:
20+
contents: read
21+
22+
jobs:
23+
codebase-memory-first3-readiness:
24+
runs-on: ubuntu-latest
25+
timeout-minutes: 360
26+
env:
27+
CI_READINESS_ROOT: /tmp/contextbench-readiness
28+
TASK_PAYLOADS: /tmp/contextbench-readiness/task-payloads.json
29+
CHECKOUT_ROOT: /tmp/contextbench-checkouts
30+
CBM_VERSION: ${{ github.event.inputs.codebase_memory_version || 'v0.6.1' }}
31+
MAX_TASKS: ${{ github.event.inputs.max_tasks || '3' }}
32+
steps:
33+
- uses: actions/checkout@v4
34+
35+
- uses: pnpm/action-setup@v2
36+
with:
37+
version: 10
38+
39+
- uses: actions/setup-node@v4
40+
with:
41+
node-version: '24'
42+
cache: 'pnpm'
43+
44+
- uses: actions/setup-python@v5
45+
with:
46+
python-version: '3.11'
47+
48+
- name: Install repo dependencies
49+
run: pnpm install --frozen-lockfile
50+
51+
- name: Install official evaluator dependencies
52+
run: python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow
53+
54+
- name: Validate frozen ContextBench fixtures
55+
run: node scripts/contextbench-runner.mjs --validate-fixtures
56+
57+
- name: Materialize first task payloads and checkouts
58+
run: |
59+
mkdir -p "$CI_READINESS_ROOT" "$CHECKOUT_ROOT"
60+
node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT"
61+
node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks "$MAX_TASKS"
62+
63+
- name: Download codebase-memory-mcp
64+
run: |
65+
set -euxo pipefail
66+
mkdir -p "$CI_READINESS_ROOT/tool"
67+
curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/${CBM_VERSION}/codebase-memory-mcp-linux-amd64.tar.gz" -o "$CI_READINESS_ROOT/tool/cbm.tar.gz"
68+
tar -xzf "$CI_READINESS_ROOT/tool/cbm.tar.gz" -C "$CI_READINESS_ROOT/tool"
69+
chmod +x "$CI_READINESS_ROOT/tool/codebase-memory-mcp" || true
70+
"$CI_READINESS_ROOT/tool/codebase-memory-mcp" --version || true
71+
72+
- name: Run codebase-memory-mcp first3 readiness with official evaluator
73+
env:
74+
CBM_BIN: /tmp/contextbench-readiness/tool/codebase-memory-mcp
75+
run: |
76+
cat > "$CI_READINESS_ROOT/codebase-memory-first3-readiness.mjs" <<'NODE'
77+
import { spawnSync } from 'node:child_process';
78+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
79+
import { join } from 'node:path';
80+
81+
const root = process.env.CI_READINESS_ROOT;
82+
const payloadPath = process.env.TASK_PAYLOADS;
83+
const cbmBin = process.env.CBM_BIN;
84+
const maxTasks = Number(process.env.MAX_TASKS || '3');
85+
const payloads = JSON.parse(readFileSync(payloadPath, 'utf8'));
86+
const tasks = payloads.tasks.slice(0, maxTasks);
87+
const outRoot = join(root, 'codebase-memory-first3');
88+
mkdirSync(outRoot, { recursive: true });
89+
90+
function run(command, args, options = {}) {
91+
const started = Date.now();
92+
const result = spawnSync(command, args, {
93+
encoding: 'utf8',
94+
timeout: options.timeoutMs ?? 20 * 60 * 1000,
95+
env: options.env ?? process.env,
96+
cwd: options.cwd ?? process.cwd(),
97+
maxBuffer: 64 * 1024 * 1024
98+
});
99+
return {
100+
command: [command, ...args].join(' '),
101+
status: result.status,
102+
signal: result.signal,
103+
error: result.error?.message ?? null,
104+
durationMs: Date.now() - started,
105+
stdout: result.stdout ?? '',
106+
stderr: result.stderr ?? ''
107+
};
108+
}
109+
110+
function runCandidates(label, candidates, options = {}) {
111+
const attempts = [];
112+
for (const args of candidates) {
113+
const attempt = run(cbmBin, args, options);
114+
attempts.push(attempt);
115+
if (attempt.status === 0) return { ...attempt, label, attempts };
116+
}
117+
return { ...attempts[attempts.length - 1], label, attempts };
118+
}
119+
120+
function parseJsonish(text) {
121+
const trimmed = String(text || '').trim();
122+
if (!trimmed) return null;
123+
try { return JSON.parse(trimmed); } catch {}
124+
for (const [open, close] of [['{', '}'], ['[', ']']]) {
125+
const first = trimmed.indexOf(open);
126+
const last = trimmed.lastIndexOf(close);
127+
if (first >= 0 && last > first) {
128+
try { return JSON.parse(trimmed.slice(first, last + 1)); } catch {}
129+
}
130+
}
131+
return null;
132+
}
133+
134+
function addSpan(spans, file, start = 1, end = start) {
135+
if (typeof file !== 'string' || file.length === 0) return;
136+
const clean = file.replace(/^\/+/, '');
137+
const startLine = Number.isFinite(Number(start)) ? Math.max(1, Number(start)) : 1;
138+
const endLine = Number.isFinite(Number(end)) ? Math.max(startLine, Number(end)) : startLine;
139+
const list = spans.get(clean) || [];
140+
list.push({ start: startLine, end: endLine });
141+
spans.set(clean, list);
142+
}
143+
144+
function collectSpans(value, spans = new Map()) {
145+
if (!value || typeof value !== 'object') return spans;
146+
if (Array.isArray(value)) {
147+
for (const item of value) collectSpans(item, spans);
148+
return spans;
149+
}
150+
const file = value.file || value.path || value.file_path || value.rel_path || value.relative_path || value.filename;
151+
const start = value.start_line || value.startLine || value.line || value.line_number || value.line_start || value.start;
152+
const end = value.end_line || value.endLine || value.line_end || value.end || start;
153+
if (typeof file === 'string') addSpan(spans, file, start, end);
154+
for (const item of Object.values(value)) collectSpans(item, spans);
155+
return spans;
156+
}
157+
158+
function collectTextSpans(text, spans) {
159+
const source = String(text || '');
160+
const regex = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g;
161+
let match;
162+
while ((match = regex.exec(source)) !== null) addSpan(spans, match[1], match[2] || 1, match[2] || 1);
163+
}
164+
165+
function makeQuery(problem) {
166+
return String(problem || '')
167+
.replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ')
168+
.split(/\s+/)
169+
.filter((w) => w.length >= 4 && !/^https?$/.test(w))
170+
.slice(0, 8)
171+
.join(' ');
172+
}
173+
174+
const reports = [];
175+
let ready = true;
176+
for (const [index, task] of tasks.entries()) {
177+
const runDir = join(outRoot, `${index + 1}-${task.instance_id}`);
178+
mkdirSync(runDir, { recursive: true });
179+
const env = { ...process.env, CBM_CACHE_DIR: join(runDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' };
180+
const query = makeQuery(task.problem_statement);
181+
const setup = run(cbmBin, ['--version'], { env, timeoutMs: 60_000 });
182+
const indexRun = run(cbmBin, ['cli', 'index_repository', JSON.stringify({ repo_path: task.repo_checkout_path })], { env, timeoutMs: 45 * 60 * 1000 });
183+
const listProjects = runCandidates('list_projects', [['cli', '--raw', 'list_projects'], ['cli', 'list_projects'], ['cli', 'list_projects', '{}']], { env, timeoutMs: 120_000 });
184+
const graphSchema = runCandidates('get_graph_schema', [['cli', '--raw', 'get_graph_schema'], ['cli', 'get_graph_schema'], ['cli', 'get_graph_schema', '{}']], { env, timeoutMs: 120_000 });
185+
const graphSearch = runCandidates('search_graph', [['cli', '--raw', 'search_graph', JSON.stringify({ label: 'Function', limit: 25 })], ['cli', 'search_graph', JSON.stringify({ label: 'Function', limit: 25 })]], { env, timeoutMs: 120_000 });
186+
const codeSearch = runCandidates('search_code', [['cli', '--raw', 'search_code', JSON.stringify({ query, limit: 25 })], ['cli', 'search_code', JSON.stringify({ query, limit: 25 })]], { env, timeoutMs: 120_000 });
187+
188+
const spans = new Map();
189+
for (const text of [codeSearch.stdout, graphSearch.stdout, graphSchema.stdout, listProjects.stdout]) {
190+
const parsed = parseJsonish(text);
191+
if (parsed) collectSpans(parsed, spans);
192+
collectTextSpans(text, spans);
193+
}
194+
const predFiles = [...spans.keys()].slice(0, 20);
195+
const predSpans = Object.fromEntries([...spans.entries()].slice(0, 20));
196+
const prediction = {
197+
instance_id: task.instance_id,
198+
repo_url: task.repo_checkout_path,
199+
commit: task.base_commit,
200+
traj_data: { pred_steps: [{ files: predFiles, spans: predSpans }], pred_files: predFiles, pred_spans: predSpans },
201+
model_patch: ''
202+
};
203+
writeFileSync(join(runDir, 'prediction.json'), JSON.stringify(prediction, null, 2));
204+
for (const [name, result] of Object.entries({ setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch })) {
205+
writeFileSync(join(runDir, `${name}.stdout.log`), result.stdout || '');
206+
writeFileSync(join(runDir, `${name}.stderr.log`), result.stderr || '');
207+
writeFileSync(join(runDir, `${name}.json`), JSON.stringify(result, null, 2));
208+
}
209+
210+
const goldPath = join(runDir, 'gold.json');
211+
const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', payloadPath], { timeoutMs: 10 * 60 * 1000 });
212+
const officialRepo = join(root, 'ContextBench-official');
213+
if (!existsSync(join(officialRepo, 'contextbench', 'evaluate.py'))) run('git', ['clone', '--depth', '1', 'https://github.com/EuniAI/ContextBench.git', officialRepo], { timeoutMs: 10 * 60 * 1000 });
214+
const scorePath = join(runDir, 'official-score.jsonl');
215+
const evaluator = run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', join(runDir, 'prediction.json'), '--cache', join(runDir, 'repo-cache'), '--out', scorePath], { cwd: officialRepo, timeoutMs: 20 * 60 * 1000 });
216+
const scoreable = evaluator.status === 0 && existsSync(scorePath);
217+
const nonEmptyPrediction = predFiles.length > 0 && Object.keys(predSpans).length > 0;
218+
const report = {
219+
taskId: task.instance_id,
220+
repo: task.repo,
221+
setupStatus: setup.status,
222+
indexStatus: indexRun.status,
223+
toolCallable: [listProjects, graphSchema, graphSearch, codeSearch].some((r) => r.status === 0),
224+
nonEmptyPrediction,
225+
officialEvaluatorStatus: evaluator.status,
226+
officialEvaluatorScoreable: scoreable,
227+
laneIsolation: {
228+
allowedTool: 'codebase-memory-mcp',
229+
observedCommands: [setup.command, indexRun.command, listProjects.command, graphSchema.command, graphSearch.command, codeSearch.command],
230+
disallowedNativeReadSearchUsedForPrediction: false,
231+
note: 'Prediction spans are derived only from codebase-memory-mcp CLI outputs.'
232+
},
233+
costs: {
234+
setupDurationMs: setup.durationMs,
235+
indexDurationMs: indexRun.durationMs,
236+
queryDurationMs: listProjects.durationMs + graphSchema.durationMs + graphSearch.durationMs + codeSearch.durationMs,
237+
evaluatorDurationMs: evaluator.durationMs
238+
},
239+
query,
240+
predFiles,
241+
commands: { setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch, gold, evaluator }
242+
};
243+
writeFileSync(join(runDir, 'readiness-report.json'), JSON.stringify(report, null, 2));
244+
reports.push(report);
245+
if (!(setup.status === 0 && indexRun.status === 0 && report.toolCallable && nonEmptyPrediction && scoreable)) ready = false;
246+
}
247+
248+
const summary = {
249+
createdAt: new Date().toISOString(),
250+
lane: 'codebase-memory-mcp',
251+
ready,
252+
attemptedRows: reports.length,
253+
scoreableRows: reports.filter((r) => r.officialEvaluatorScoreable).length,
254+
nonEmptyPredictionRows: reports.filter((r) => r.nonEmptyPrediction).length,
255+
setupIndexCostReportedSeparately: true,
256+
reports
257+
};
258+
writeFileSync(join(outRoot, 'lane-readiness-codebase-memory-first3.json'), JSON.stringify(summary, null, 2));
259+
console.log(JSON.stringify(summary, null, 2));
260+
if (!ready) process.exitCode = 1;
261+
NODE
262+
node "$CI_READINESS_ROOT/codebase-memory-first3-readiness.mjs"
263+
264+
- name: Upload ContextBench recovery artifacts
265+
if: always()
266+
uses: actions/upload-artifact@v4
267+
with:
268+
name: contextbench-codebase-memory-first3-readiness
269+
path: /tmp/contextbench-readiness
270+
retention-days: 14

0 commit comments

Comments
 (0)