Skip to content

Commit 3d6ac27

Browse files
PatrickSysPatrickSys
committed
Make CBM ContextBench candidate pack logs compact
1 parent f4e5bf9 commit 3d6ac27

1 file changed

Lines changed: 100 additions & 31 deletions

File tree

‎.github/workflows/contextbench-cbm-candidates-one.yml‎

Lines changed: 100 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -31,53 +31,122 @@ jobs:
3131
- uses: actions/setup-python@v5
3232
with:
3333
python-version: '3.11'
34-
- name: Install and materialize Go task
34+
- name: Install and materialize Go task quietly
35+
shell: bash
3536
run: |
36-
set -euxo pipefail
37-
mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool"
38-
pnpm install --frozen-lockfile
39-
python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow
37+
set -euo pipefail
38+
mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/pack" "$ROOT/logs"
39+
pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1
40+
python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-install.log" 2>&1
4041
curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz"
41-
tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool"
42+
tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" > "$ROOT/logs/cbm-extract.log" 2>&1
4243
chmod +x "$CBM_BIN" || true
43-
git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official"
44-
node scripts/contextbench-runner.mjs --validate-fixtures
45-
node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT"
46-
node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3
47-
- name: Print CBM candidates
44+
git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" > "$ROOT/logs/contextbench-clone.log" 2>&1
45+
node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1
46+
node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1
47+
node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 > "$ROOT/logs/materialize.log" 2>&1
48+
echo "quiet_setup_completed"
49+
- name: Print compact CBM candidates
50+
shell: bash
4851
run: |
4952
cat > "$ROOT/cbm-pack.mjs" <<'NODE'
5053
import { spawnSync } from 'node:child_process';
51-
import { basename, relative } from 'node:path';
52-
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
54+
import { basename, join, relative } from 'node:path';
55+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
5356
const root = process.env.ROOT;
54-
const outDir = root + '/pack';
57+
const outDir = join(root, 'pack');
5558
mkdirSync(outDir, { recursive: true });
5659
const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8'));
5760
const task = payloads.tasks[2];
5861
const repo = task.repo_checkout_path;
59-
function run(cmd, args, opts = {}) { const started = Date.now(); const r = spawnSync(cmd, args, { cwd: opts.cwd || process.cwd(), env: opts.env || process.env, encoding: 'utf8', timeout: opts.timeoutMs || 600000, maxBuffer: 128 * 1024 * 1024 }); return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; }
60-
function q(s) { return String(s || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4).slice(0, 10).join(' '); }
61-
function jsonish(s) { const t = String(s || '').trim(); if (!t) return null; try { return JSON.parse(t); } catch {} for (const [a,b] of [['{','}'],['[',']']]) { const i=t.indexOf(a), j=t.lastIndexOf(b); if (i>=0 && j>i) { try { return JSON.parse(t.slice(i,j+1)); } catch {} } } return null; }
62-
function norm(file) { let f = String(file || '').replace(/^file:\/\//, ''); if (!f) return ''; if (f.startsWith(repo)) f = relative(repo, f); f = f.replaceAll('\\\\','/').replace(/^\/+/, '').replace(/^\.\//, ''); if (!f || f.startsWith('tmp/') || f.includes('://') || f.includes('..')) return ''; return f; }
63-
function add(locs, file, start = 1, end = start) { const clean = norm(file); if (!clean) return; const s = Math.max(1, Number(start) || 1); locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source: 'codebase-memory-mcp' }); }
64-
function walk(v, locs) { if (!v || typeof v !== 'object') return; if (Array.isArray(v)) { for (const x of v) walk(x, locs); return; } add(locs, v.file || v.path || v.file_path || v.relative_path || v.filename || v.source_path, v.start_line || v.line || 1, v.end_line || v.line || 1); for (const x of Object.values(v)) walk(x, locs); }
65-
function collect(text, locs) { const parsed = jsonish(text); if (parsed) walk(parsed, locs); const re = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1); }
66-
function uniq(locs) { const seen = new Set(), out = []; for (const loc of locs) { const k = `${loc.file}:${loc.start}:${loc.end}`; if (!seen.has(k)) { seen.add(k); out.push(loc); if (out.length >= 80) break; } } return out; }
67-
const query = q(task.problem_statement);
68-
const env = { ...process.env, CBM_CACHE_DIR: outDir + '/cbm-cache', CBM_DIAGNOSTICS: '1' };
62+
function run(cmd, args, opts = {}) {
63+
const started = Date.now();
64+
const r = spawnSync(cmd, args, {
65+
cwd: opts.cwd || process.cwd(),
66+
env: opts.env || process.env,
67+
encoding: 'utf8',
68+
timeout: opts.timeoutMs || 600000,
69+
maxBuffer: 128 * 1024 * 1024
70+
});
71+
return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' };
72+
}
73+
function queryOf(text) {
74+
const stop = new Set(['that','this','with','from','when','then','into','should','would','could','there','where','which','about','after','before','have','will','been','than','also','only','some','using']);
75+
return String(text || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())).slice(0, 14).join(' ');
76+
}
77+
function jsonish(s) {
78+
const t = String(s || '').trim();
79+
if (!t) return null;
80+
try { return JSON.parse(t); } catch {}
81+
for (const [a,b] of [['{','}'],['[',']']]) {
82+
const i = t.indexOf(a), j = t.lastIndexOf(b);
83+
if (i >= 0 && j > i) { try { return JSON.parse(t.slice(i, j + 1)); } catch {} }
84+
}
85+
return null;
86+
}
87+
function norm(file) {
88+
let f = String(file || '').replace(/^file:\/\//, '').replaceAll('\\', '/');
89+
if (!f) return '';
90+
if (f.startsWith(repo)) f = relative(repo, f).replaceAll('\\', '/');
91+
f = f.replace(/^\/+/, '').replace(/^\.\//, '');
92+
if (!f || f.includes('://') || f.includes('..') || f.startsWith('tmp-contextbench') || f.includes('/tmp-contextbench')) return '';
93+
if (!existsSync(join(repo, f))) return '';
94+
return f;
95+
}
96+
function add(locs, file, start = 1, end = start, source = 'codebase-memory-mcp') {
97+
const clean = norm(file);
98+
if (!clean) return;
99+
const s = Math.max(1, Number(start) || 1);
100+
locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source });
101+
}
102+
function walk(value, locs) {
103+
if (!value || typeof value !== 'object') return;
104+
if (Array.isArray(value)) { for (const item of value) walk(item, locs); return; }
105+
add(locs, value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, value.start_line || value.startLine || value.line || value.line_number || value.start || 1, value.end_line || value.endLine || value.end || value.line || 1);
106+
for (const item of Object.values(value)) walk(item, locs);
107+
}
108+
function collect(text, locs) {
109+
const parsed = jsonish(text);
110+
if (parsed) walk(parsed, locs);
111+
const re = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp))(?::|#L|\s+line\s+)?(\d+)?/g;
112+
let m;
113+
while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1);
114+
}
115+
function uniq(locs, max = 80) {
116+
const seen = new Set(), out = [];
117+
for (const loc of locs) {
118+
const key = `${loc.file}:${loc.start}:${loc.end}`;
119+
if (!seen.has(key)) { seen.add(key); out.push(loc); if (out.length >= max) break; }
120+
}
121+
return out;
122+
}
123+
const query = queryOf(task.problem_statement);
124+
const env = { ...process.env, CBM_CACHE_DIR: join(outDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' };
69125
const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 });
70126
const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 2700000 });
71127
const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo);
72-
const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 40 })], { cwd: repo, env, timeoutMs: 120000 });
73-
const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 40 })], { cwd: repo, env, timeoutMs: 120000 });
74-
for (const [name, value] of Object.entries({ setup, index, graph, code })) writeFileSync(`${outDir}/${name}.json`, JSON.stringify(value, null, 2));
75-
const locs = []; for (const r of [graph, code]) { collect(r.stdout, locs); collect(r.stderr, locs); }
128+
const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 50 })], { cwd: repo, env, timeoutMs: 120000 });
129+
const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 50 })], { cwd: repo, env, timeoutMs: 120000 });
130+
for (const [name, value] of Object.entries({ setup, index, graph, code })) writeFileSync(join(outDir, `${name}.json`), JSON.stringify({ ...value, stdout: value.stdout.slice(0, 120000), stderr: value.stderr.slice(0, 120000) }, null, 2));
131+
const locs = [];
132+
for (const r of [graph, code]) { collect(r.stdout, locs); collect(r.stderr, locs); }
76133
const candidates = uniq(locs);
77-
const pack = { task: { instance_id: task.instance_id, repo: task.repo, base_commit: task.base_commit, problem_statement: task.problem_statement }, query, lane: { lane: 'codebase-memory-mcp', setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', indexStatus: index.status === 0 ? 'completed' : 'index_failed', toolCallable: graph.status === 0 || code.status === 0, candidateCount: candidates.length, setupIndex: { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: graph.durationMs + code.durationMs }, candidates } };
78-
writeFileSync(`${outDir}/cbm-candidate-pack.json`, JSON.stringify(pack, null, 2));
134+
const pack = {
135+
task: { instance_id: task.instance_id, repo: task.repo, base_commit: task.base_commit, problem_statement: task.problem_statement },
136+
query,
137+
lane: {
138+
lane: 'codebase-memory-mcp',
139+
setupStatus: setup.status === 0 ? 'completed' : 'setup_failed',
140+
indexStatus: index.status === 0 ? 'completed' : 'index_failed',
141+
toolCallable: graph.status === 0 || code.status === 0,
142+
candidateCount: candidates.length,
143+
setupIndex: { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: graph.durationMs + code.durationMs },
144+
candidates
145+
}
146+
};
147+
writeFileSync(join(outDir, 'cbm-candidate-pack.json'), JSON.stringify(pack, null, 2));
79148
console.log('CBM_CANDIDATE_PACK_JSON_START');
80-
console.log(JSON.stringify({ ...pack, lane: { ...pack.lane, candidates: candidates.slice(0, 60) } }, null, 2));
149+
console.log(JSON.stringify(pack, null, 2));
81150
console.log('CBM_CANDIDATE_PACK_JSON_END');
82151
NODE
83152
node "$ROOT/cbm-pack.mjs"

0 commit comments

Comments
 (0)