Skip to content

Commit ba7f2f3

Browse files
PatrickSysPatrickSys
committed
Generate ContextBench candidate pack for one task
1 parent 4b00271 commit ba7f2f3

1 file changed

Lines changed: 127 additions & 0 deletions

File tree

‎.github/workflows/contextbench-candidate-pack-one.yml‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
name: ContextBench Candidate Pack One
2+
3+
on:
4+
push:
5+
branches: [master]
6+
paths:
7+
- .github/workflows/contextbench-candidate-pack-one.yml
8+
workflow_dispatch:
9+
10+
permissions:
11+
contents: read
12+
13+
jobs:
14+
pack:
15+
runs-on: ubuntu-latest
16+
timeout-minutes: 90
17+
env:
18+
ROOT: /tmp/contextbench-candidate-pack-one
19+
TASK_PAYLOADS: /tmp/contextbench-candidate-pack-one/task-payloads.json
20+
CHECKOUT_ROOT: /tmp/contextbench-checkouts
21+
CBM_BIN: /tmp/contextbench-candidate-pack-one/tool/codebase-memory-mcp
22+
steps:
23+
- uses: actions/checkout@v4
24+
- uses: pnpm/action-setup@v2
25+
with:
26+
version: 10
27+
- uses: actions/setup-node@v4
28+
with:
29+
node-version: '24'
30+
cache: pnpm
31+
- uses: actions/setup-python@v5
32+
with:
33+
python-version: '3.11'
34+
- name: Install tools and materialize frozen Go task
35+
run: |
36+
set -euxo pipefail
37+
mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool"
38+
pnpm install --frozen-lockfile
39+
pnpm run build
40+
python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow uv codegraphcontext kuzu
41+
curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz"
42+
tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool"
43+
chmod +x "$CBM_BIN" || true
44+
curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh || true
45+
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
46+
echo "$HOME/bin" >> "$GITHUB_PATH"
47+
git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official"
48+
node scripts/contextbench-runner.mjs --validate-fixtures
49+
node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT"
50+
node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3
51+
- name: Generate candidate pack
52+
run: |
53+
cat > "$ROOT/pack.mjs" <<'NODE'
54+
import { spawnSync } from 'node:child_process';
55+
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
56+
import { basename, relative, sep } from 'node:path';
57+
const root = process.env.ROOT;
58+
const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8'));
59+
const task = payloads.tasks[2];
60+
const repo = task.repo_checkout_path;
61+
const outDir = root + '/pack';
62+
mkdirSync(outDir, { recursive: true });
63+
function run(cmd, args, opts = {}) { const started = Date.now(); const r = spawnSync(cmd, args, { cwd: opts.cwd || process.cwd(), env: opts.env || process.env, encoding: 'utf8', timeout: opts.timeoutMs || 300000, maxBuffer: 96 * 1024 * 1024 }); return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; }
64+
function queryOf(text) { const stop = new Set(['that','this','with','from','when','then','into','should','would','could','there','where','which','about','after','before','have','will','been','than','also']); return String(text || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())).slice(0, 12).join(' '); }
65+
function jsonish(text) { const raw = String(text || '').trim(); if (!raw) return null; try { return JSON.parse(raw); } catch {} for (const [a,b] of [['{','}'],['[',']']]) { const i = raw.indexOf(a), j = raw.lastIndexOf(b); if (i >= 0 && j > i) { try { return JSON.parse(raw.slice(i, j + 1)); } catch {} } } return null; }
66+
function norm(file) { let f = String(file || '').replace(/^file:\/\//, ''); if (!f) return ''; if (f.startsWith(repo)) f = relative(repo, f); f = f.replaceAll('\\\\', '/').replace(/^\/+/, '').replace(/^\.\//, ''); if (f.startsWith('tmp/') || f.includes('://') || f.includes('..')) return ''; return f; }
67+
function add(locs, file, start = 1, end = start, source = 'tool') { const clean = norm(file); if (!clean) return; const s = Math.max(1, Number(start) || 1); locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); }
68+
function walk(value, locs, source) { if (!value || typeof value !== 'object') return; if (Array.isArray(value)) { for (const item of value) walk(item, locs, source); return; } add(locs, value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, value.start_line || value.startLine || value.line || value.line_number || value.start || 1, value.end_line || value.endLine || value.end || value.line || 1, source); for (const item of Object.values(value)) walk(item, locs, source); }
69+
function collect(text, locs, source) { const parsed = jsonish(text); if (parsed) walk(parsed, locs, source); const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); }
70+
function uniq(locs, max = 60) { const seen = new Set(), out = []; for (const loc of locs) { const key = `${loc.file}:${loc.start}:${loc.end}`; if (!seen.has(key)) { seen.add(key); out.push(loc); if (out.length >= max) break; } } return out; }
71+
function laneResult(lane, commands, locs, setupStatus = 'completed', indexStatus = 'completed') { for (const [i,c] of commands.entries()) writeFileSync(`${outDir}/${lane}-command-${i + 1}.json`, JSON.stringify(c, null, 2)); return { lane, setupStatus, indexStatus, toolCallable: commands.some((c) => c.status === 0), costs: { totalMs: commands.reduce((a,c) => a + c.durationMs, 0) }, candidates: uniq(locs) }; }
72+
const query = queryOf(task.problem_statement);
73+
const lanes = [];
74+
{
75+
const commands = [], locs = [];
76+
for (const term of query.split(/\s+/).slice(0, 6)) { const r = run('rg', ['-n', '-i', '--glob', '!.git', '--glob', '!vendor/**', '--glob', '!node_modules/**', term, '.'], { cwd: repo, timeoutMs: 60000 }); commands.push(r); collect(r.stdout, locs, 'raw-native'); collect(r.stderr, locs, 'raw-native'); }
77+
lanes.push(laneResult('raw-native', commands, locs));
78+
}
79+
{
80+
const commands = [], locs = []; const env = { ...process.env, CODEBASE_ROOT: repo, CODEBASE_CONTEXT_ASCII: '1' };
81+
const v = run('node', ['dist/index.js', '--version'], { env, timeoutMs: 60000 }); commands.push(v);
82+
const idx = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 1200000 }); commands.push(idx);
83+
const s = run('node', ['dist/index.js', 'search', '--query', query, '--intent', 'edit', '--limit', '30', '--json'], { env, timeoutMs: 300000 }); commands.push(s); collect(s.stdout, locs, 'codebase-context'); collect(s.stderr, locs, 'codebase-context');
84+
lanes.push(laneResult('codebase-context', commands, locs, 'completed', idx.status === 0 ? 'completed' : 'index_failed'));
85+
}
86+
{
87+
const commands = [], locs = []; const env = { ...process.env, CBM_CACHE_DIR: outDir + '/cbm-cache', CBM_DIAGNOSTICS: '1' };
88+
const v = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); commands.push(v);
89+
const idx = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 2700000 }); commands.push(idx);
90+
const project = (jsonish(idx.stdout) || jsonish(idx.stderr) || {}).project || basename(repo);
91+
const g = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 30 })], { cwd: repo, env, timeoutMs: 120000 }); commands.push(g);
92+
const c = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 30 })], { cwd: repo, env, timeoutMs: 120000 }); commands.push(c);
93+
for (const r of [g, c]) { collect(r.stdout, locs, 'codebase-memory-mcp'); collect(r.stderr, locs, 'codebase-memory-mcp'); }
94+
lanes.push(laneResult('codebase-memory-mcp', commands, locs, 'completed', idx.status === 0 ? 'completed' : 'index_failed'));
95+
}
96+
{
97+
const commands = [], locs = [];
98+
const v = run('grepai', ['version'], { timeoutMs: 60000 }); commands.push(v);
99+
const init = run('grepai', ['init', '--yes', '--provider', 'synthetic', '--backend', 'gob'], { cwd: repo, timeoutMs: 120000 }); commands.push(init);
100+
const watch = run('grepai', ['watch', '--background'], { cwd: repo, timeoutMs: 120000 }); commands.push(watch);
101+
const status = run('grepai', ['watch', '--status'], { cwd: repo, timeoutMs: 60000 }); commands.push(status);
102+
const s = run('grepai', ['search', query, '--json', '--compact'], { cwd: repo, timeoutMs: 180000 }); commands.push(s); collect(s.stdout, locs, 'grepai'); collect(s.stderr, locs, 'grepai');
103+
commands.push(run('grepai', ['watch', '--stop'], { cwd: repo, timeoutMs: 60000 }));
104+
lanes.push(laneResult('grepai', commands, locs, init.status === 0 ? 'completed' : 'setup_failed', (watch.status === 0 || status.status === 0) ? 'completed' : 'index_failed'));
105+
}
106+
{
107+
const commands = [], locs = [];
108+
const v = run('cgc', ['--help'], { timeoutMs: 60000 }); commands.push(v);
109+
const idx = run('cgc', ['index', repo], { cwd: repo, timeoutMs: 1200000 }); commands.push(idx);
110+
const a = run('cgc', ['analyze', 'complexity', '--limit', '30'], { cwd: repo, timeoutMs: 180000 }); commands.push(a);
111+
const d = run('cgc', ['analyze', 'dead-code', '--limit', '30'], { cwd: repo, timeoutMs: 180000 }); commands.push(d);
112+
const call = run('cgc', ['analyze', 'callers', query.split(/\s+/)[0] || 'main'], { cwd: repo, timeoutMs: 120000 }); commands.push(call);
113+
for (const r of [a, d, call]) { collect(r.stdout, locs, 'codegraphcontext'); collect(r.stderr, locs, 'codegraphcontext'); }
114+
lanes.push(laneResult('codegraphcontext', commands, locs, 'completed', idx.status === 0 ? 'completed' : 'index_failed'));
115+
}
116+
const pack = { createdAt: new Date().toISOString(), task: { instance_id: task.instance_id, repo: task.repo, base_commit: task.base_commit, problem_statement: task.problem_statement }, query, lanes };
117+
writeFileSync(`${outDir}/candidate-pack.json`, JSON.stringify(pack, null, 2));
118+
console.log(JSON.stringify({ task: pack.task, query, lanes: lanes.map((l) => ({ lane: l.lane, setupStatus: l.setupStatus, indexStatus: l.indexStatus, toolCallable: l.toolCallable, candidateCount: l.candidates.length, candidates: l.candidates.slice(0, 30) })) }, null, 2));
119+
NODE
120+
node "$ROOT/pack.mjs"
121+
- name: Upload candidate pack
122+
if: always()
123+
uses: actions/upload-artifact@v4
124+
with:
125+
name: contextbench-candidate-pack-one
126+
path: /tmp/contextbench-candidate-pack-one/pack
127+
retention-days: 14

0 commit comments

Comments
 (0)