Skip to content

Commit 66a522b

Browse files
PatrickSysPatrickSys
committed
Call CBM ContextBench candidate extractor from script
1 parent 2048fee commit 66a522b

1 file changed

Lines changed: 2 additions & 102 deletions

File tree

‎.github/workflows/contextbench-cbm-candidates-one.yml‎

Lines changed: 2 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ on:
55
branches: [master]
66
paths:
77
- .github/workflows/contextbench-cbm-candidates-one.yml
8+
- scripts/contextbench-cbm-candidate-pack.mjs
89
workflow_dispatch:
910

1011
permissions:
@@ -48,108 +49,7 @@ jobs:
4849
echo "quiet_setup_completed"
4950
- name: Print compact CBM candidates
5051
shell: bash
51-
run: |
52-
cat > "$ROOT/cbm-pack.mjs" <<'NODE'
53-
import { spawnSync } from 'node:child_process';
54-
import { basename, join, relative } from 'node:path';
55-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
56-
const root = process.env.ROOT;
57-
const outDir = join(root, 'pack');
58-
mkdirSync(outDir, { recursive: true });
59-
const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8'));
60-
const task = payloads.tasks[2];
61-
const repo = task.repo_checkout_path;
62-
function run(cmd, args, opts = {}) {
63-
const started = Date.now();
64-
const r = spawnSync(cmd, args, {
65-
cwd: opts.cwd || process.cwd(),
66-
env: opts.env || process.env,
67-
encoding: 'utf8',
68-
timeout: opts.timeoutMs || 600000,
69-
maxBuffer: 128 * 1024 * 1024
70-
});
71-
return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' };
72-
}
73-
function queryOf(text) {
74-
const stop = new Set(['that','this','with','from','when','then','into','should','would','could','there','where','which','about','after','before','have','will','been','than','also','only','some','using']);
75-
return String(text || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())).slice(0, 14).join(' ');
76-
}
77-
function jsonish(s) {
78-
const t = String(s || '').trim();
79-
if (!t) return null;
80-
try { return JSON.parse(t); } catch {}
81-
for (const [a,b] of [['{','}'],['[',']']]) {
82-
const i = t.indexOf(a), j = t.lastIndexOf(b);
83-
if (i >= 0 && j > i) { try { return JSON.parse(t.slice(i, j + 1)); } catch {} }
84-
}
85-
return null;
86-
}
87-
function norm(file) {
88-
let f = String(file || '').replace(/^file:\/\//, '').replaceAll('\\', '/');
89-
if (!f) return '';
90-
if (f.startsWith(repo)) f = relative(repo, f).replaceAll('\\', '/');
91-
f = f.replace(/^\/+/, '').replace(/^\.\//, '');
92-
if (!f || f.includes('://') || f.includes('..') || f.startsWith('tmp-contextbench') || f.includes('/tmp-contextbench')) return '';
93-
if (!existsSync(join(repo, f))) return '';
94-
return f;
95-
}
96-
function add(locs, file, start = 1, end = start, source = 'codebase-memory-mcp') {
97-
const clean = norm(file);
98-
if (!clean) return;
99-
const s = Math.max(1, Number(start) || 1);
100-
locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source });
101-
}
102-
function walk(value, locs) {
103-
if (!value || typeof value !== 'object') return;
104-
if (Array.isArray(value)) { for (const item of value) walk(item, locs); return; }
105-
add(locs, value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, value.start_line || value.startLine || value.line || value.line_number || value.start || 1, value.end_line || value.endLine || value.end || value.line || 1);
106-
for (const item of Object.values(value)) walk(item, locs);
107-
}
108-
function collect(text, locs) {
109-
const parsed = jsonish(text);
110-
if (parsed) walk(parsed, locs);
111-
const re = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp))(?::|#L|\s+line\s+)?(\d+)?/g;
112-
let m;
113-
while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1);
114-
}
115-
function uniq(locs, max = 80) {
116-
const seen = new Set(), out = [];
117-
for (const loc of locs) {
118-
const key = `${loc.file}:${loc.start}:${loc.end}`;
119-
if (!seen.has(key)) { seen.add(key); out.push(loc); if (out.length >= max) break; }
120-
}
121-
return out;
122-
}
123-
const query = queryOf(task.problem_statement);
124-
const env = { ...process.env, CBM_CACHE_DIR: join(outDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' };
125-
const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 });
126-
const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 2700000 });
127-
const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo);
128-
const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 50 })], { cwd: repo, env, timeoutMs: 120000 });
129-
const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 50 })], { cwd: repo, env, timeoutMs: 120000 });
130-
for (const [name, value] of Object.entries({ setup, index, graph, code })) writeFileSync(join(outDir, `${name}.json`), JSON.stringify({ ...value, stdout: value.stdout.slice(0, 120000), stderr: value.stderr.slice(0, 120000) }, null, 2));
131-
const locs = [];
132-
for (const r of [graph, code]) { collect(r.stdout, locs); collect(r.stderr, locs); }
133-
const candidates = uniq(locs);
134-
const pack = {
135-
task: { instance_id: task.instance_id, repo: task.repo, base_commit: task.base_commit, problem_statement: task.problem_statement },
136-
query,
137-
lane: {
138-
lane: 'codebase-memory-mcp',
139-
setupStatus: setup.status === 0 ? 'completed' : 'setup_failed',
140-
indexStatus: index.status === 0 ? 'completed' : 'index_failed',
141-
toolCallable: graph.status === 0 || code.status === 0,
142-
candidateCount: candidates.length,
143-
setupIndex: { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: graph.durationMs + code.durationMs },
144-
candidates
145-
}
146-
};
147-
writeFileSync(join(outDir, 'cbm-candidate-pack.json'), JSON.stringify(pack, null, 2));
148-
console.log('CBM_CANDIDATE_PACK_JSON_START');
149-
console.log(JSON.stringify(pack, null, 2));
150-
console.log('CBM_CANDIDATE_PACK_JSON_END');
151-
NODE
152-
node "$ROOT/cbm-pack.mjs"
52+
run: node scripts/contextbench-cbm-candidate-pack.mjs
15353
- name: Upload CBM candidate pack
15454
if: always()
15555
uses: actions/upload-artifact@v4

0 commit comments

Comments
 (0)