Skip to content

Commit 441d855

Browse files
andre15silvaclaude
andcommitted
feat(probe): add tool-output NLL correlation ablation
Measures whether the model's surprise at tool/environment output (mean NLL of bash output tokens) predicts probe difficulty. Adds: - build_tool_nll_index.py: GPU forward-pass script that computes per-turn mean NLL of tool output tokens and saves a {sample_id: {turn_k: nll}} index (no cache rebuild needed) - run_eval: optional tool_nll_run_id param; computes Spearman correlation between per-step tool NLL and Brier score, saved to nll_corr.pt - run_probe.py: --tool-nll-run-id CLI flag - plot_tool_nll_correlation: scatter + LOWESS figure with ρ annotated - run_figures.py: --tool-nll-run-id / --tool-nll-filename-suffix flags - slurm/build_tool_nll.sh: GPU SLURM script (same config as extraction) - 8 unit tests covering NLL computation, normalization, Brier, Spearman Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent fa669d9 commit 441d855

7 files changed

Lines changed: 604 additions & 1 deletion

File tree

build_tool_nll_index.py

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
"""Compute per-turn mean NLL of tool output tokens for SWE-bench trajectories.
2+
3+
For each assistant turn k > 0, the model's mean negative log-likelihood over the
4+
tool/environment output tokens that precede turn k is stored. This quantifies how
5+
"surprised" the model was by the test runner or compiler response.
6+
7+
Output: <output> (a .pt file)
8+
= {sample_id: {turn_k (int): mean_nll (float)}}
9+
10+
Turn 0 is excluded because it has no preceding tool output (only the initial task).
11+
12+
Usage (array job, 8 shards):
13+
sbatch --array=0-7 slurm/build_tool_nll.sh \\
14+
--model-config configs/models/laguna_xs2.yaml \\
15+
--generation-config configs/generation_laguna_xs2.yaml \\
16+
--traj-dir generations/swebench/laguna_xs2_full \\
17+
--output cache/swebench/laguna_xs2_full/tool_nll_index.pt
18+
"""
19+
from __future__ import annotations
20+
21+
import argparse
22+
from pathlib import Path
23+
24+
import torch
25+
26+
from src.configs import GenerationConfig, ModelConfig, load_config
27+
from src.tasks.swe_bench_extract import load_trajectories
28+
29+
30+
def compute_trajectory_nll(
31+
token_ids: list[int],
32+
segments: list[dict],
33+
step_segment_indices: list[int],
34+
logits: torch.Tensor,
35+
) -> dict[int, float]:
36+
"""Per-turn mean NLL of tool output tokens for turns k > 0.
37+
38+
logits[i] is the model's prediction for token i+1, so the NLL of a tool
39+
output spanning token positions [start, end) is:
40+
mean(-log softmax(logits[start-1 : end-1])[:, token_ids[start:end]])
41+
42+
Args:
43+
token_ids: full token sequence (length = seq_len)
44+
segments: tokenization.segments list (role, start_token, end_token)
45+
step_segment_indices: indices into *segments* for each assistant turn
46+
logits: [seq_len, vocab_size] — output of one full forward pass
47+
48+
Returns:
49+
{turn_k: mean_nll} for turns that have a preceding user (tool output) segment
50+
"""
51+
turn_nll: dict[int, float] = {}
52+
for k, asst_seg_idx in enumerate(step_segment_indices):
53+
if k == 0 or asst_seg_idx == 0:
54+
continue # turn 0 is preceded by the initial task prompt, not a tool response
55+
tool_seg = segments[asst_seg_idx - 1]
56+
if tool_seg.get("role") != "user":
57+
continue
58+
start = tool_seg["start_token"]
59+
end = tool_seg["end_token"]
60+
n_tokens = end - start
61+
if n_tokens <= 0 or start == 0:
62+
continue
63+
# logits[start-1] predicts token[start], …, logits[end-2] predicts token[end-1]
64+
tool_logits = logits[start - 1 : end - 1] # [n_tokens, vocab_size]
65+
if tool_logits.shape[0] != n_tokens:
66+
continue
67+
actual = torch.tensor(token_ids[start:end], dtype=torch.long, device=logits.device)
68+
log_probs = torch.log_softmax(tool_logits.float(), dim=-1)
69+
token_nll = -log_probs[torch.arange(n_tokens, device=logits.device), actual]
70+
turn_nll[k] = float(token_nll.mean().item())
71+
return turn_nll
72+
73+
74+
def _load_model_adapter(adapter_name: str):
75+
if adapter_name == "laguna":
76+
from src.models.laguna import LagunaAdapter
77+
return LagunaAdapter()
78+
if adapter_name == "qwen":
79+
from src.models.qwen import QwenAdapter
80+
return QwenAdapter()
81+
if adapter_name == "qwen35":
82+
from src.models.qwen35 import Qwen35Adapter
83+
return Qwen35Adapter()
84+
if adapter_name == "cwm":
85+
from src.models.cwm import CwmAdapter
86+
return CwmAdapter()
87+
raise ValueError(f"Unknown model adapter: {adapter_name!r}")
88+
89+
90+
def main() -> None:
91+
parser = argparse.ArgumentParser(description=__doc__)
92+
parser.add_argument("--model-config", required=True)
93+
parser.add_argument("--generation-config", required=True)
94+
parser.add_argument("--traj-dir", required=True, help="Directory of trajectory JSON files")
95+
parser.add_argument("--output", required=True, help="Output .pt path")
96+
parser.add_argument("--shard-rank", type=int, default=0)
97+
parser.add_argument("--num-shards", type=int, default=1)
98+
args = parser.parse_args()
99+
100+
model_config: ModelConfig = load_config(args.model_config, ModelConfig)
101+
gen_config: GenerationConfig = load_config(args.generation_config, GenerationConfig)
102+
103+
out_path = Path(args.output)
104+
# If sharded, accumulate into a shard-specific temp file; merge manually afterwards.
105+
# For simplicity we write the whole shard's results to the output path (caller merges).
106+
if args.num_shards > 1:
107+
out_path = out_path.with_suffix(f".shard{args.shard_rank}.pt")
108+
109+
print(f"Loading trajectories from {args.traj_dir}...")
110+
all_trajs = load_trajectories(args.traj_dir)
111+
trajs = all_trajs[args.shard_rank::args.num_shards]
112+
print(f" {len(trajs)} trajectories (shard {args.shard_rank}/{args.num_shards})")
113+
114+
model_adapter = _load_model_adapter(model_config.adapter)
115+
model_adapter.load_for_extraction(model_config, gen_config)
116+
device = next(model_adapter._model.parameters()).device
117+
118+
index: dict[str, dict[int, float]] = {}
119+
120+
for i, traj in enumerate(trajs):
121+
print(f" [{i + 1}/{len(trajs)}] {traj.sample_id} ({len(traj.token_ids)} tokens)...", flush=True)
122+
input_ids = torch.tensor([traj.token_ids], dtype=torch.long, device=device)
123+
try:
124+
with torch.no_grad():
125+
out = model_adapter._model(input_ids=input_ids, output_hidden_states=False)
126+
logits = out.logits[0].cpu() # [seq_len, vocab_size] — move to CPU immediately
127+
del out
128+
torch.cuda.empty_cache()
129+
except torch.cuda.OutOfMemoryError:
130+
torch.cuda.empty_cache()
131+
print(f" OOM — skipping {traj.sample_id}")
132+
continue
133+
134+
turn_nll = compute_trajectory_nll(
135+
traj.token_ids, traj.segments, traj.step_segment_indices, logits
136+
)
137+
del logits
138+
index[traj.sample_id] = turn_nll
139+
print(f" {len(turn_nll)} turns with tool NLL (out of {len(traj.step_segment_indices)} total turns)")
140+
141+
out_path.parent.mkdir(parents=True, exist_ok=True)
142+
torch.save(index, out_path)
143+
print(f"Saved tool NLL index ({len(index)} samples) → {out_path}")
144+
145+
146+
if __name__ == "__main__":
147+
main()

run_figures.py

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import argparse
22
from src.configs import ModelConfig, load_config
3-
from src.figures import plot_probe_heatmap, plot_lookahead_horizon
3+
from src.figures import plot_probe_heatmap, plot_lookahead_horizon, plot_tool_nll_correlation
44

55

66
def main():
@@ -22,6 +22,10 @@ def main():
2222
"Defaults to all probes. Exclude static-label probes (e.g. will_resolve).")
2323
parser.add_argument("--lookahead-filename-suffix", default="",
2424
help="Suffix appended to the output filename, e.g. 'max50' → probe_lookahead_max50.png")
25+
parser.add_argument("--tool-nll-run-id", default=None,
26+
help="Run ID whose nll_corr.pt to load for the tool NLL correlation figure.")
27+
parser.add_argument("--tool-nll-filename-suffix", default="",
28+
help="Suffix for the tool NLL figure filename.")
2529
args = parser.parse_args()
2630

2731
model_cfg = load_config(args.model_config, ModelConfig)
@@ -37,6 +41,16 @@ def main():
3741
)
3842

3943
lookahead_probes = args.lookahead_probes or args.probe
44+
if args.tool_nll_run_id:
45+
plot_tool_nll_correlation(
46+
run_id=args.tool_nll_run_id,
47+
probe_name=probe_name,
48+
probe_layers=model_cfg.probe_layers,
49+
results_dir=args.results_dir,
50+
figures_dir=args.figures_dir,
51+
filename_suffix=args.tool_nll_filename_suffix,
52+
)
53+
4054
if args.lookahead_shift_run_ids and args.lookahead_k_values and probe_name in lookahead_probes:
4155
if len(args.lookahead_shift_run_ids) != len(args.lookahead_k_values):
4256
raise ValueError("--lookahead-shift-run-ids and --lookahead-k-values must have the same length")

run_probe.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ def main():
2727
help="Restrict eval (and training) to tokens on the first turn after a code edit.")
2828
parser.add_argument("--edit-index-run-id", default=None,
2929
help="Run ID whose edit_step_index.pt to load (defaults to cache-run-id or run-id).")
30+
parser.add_argument("--tool-nll-run-id", default=None,
31+
help="Run ID whose tool_nll_index.pt to load from cache dir for NLL correlation analysis.")
3032

3133
subparsers = parser.add_subparsers(dest="mode", required=True)
3234

@@ -106,6 +108,7 @@ def main():
106108
eval_bin_axis=args.eval_bin_axis,
107109
after_edit_only=args.after_edit_only,
108110
edit_index_run_id=args.edit_index_run_id,
111+
tool_nll_run_id=args.tool_nll_run_id,
109112
)
110113
else:
111114
run_final(

slurm/build_tool_nll.sh

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
# Compute per-turn mean NLL of tool output tokens for SWE-bench trajectories.
3+
# Runs one forward pass per trajectory (no sampling); same GPU requirements as extraction.
4+
#
5+
# Laguna XS2 (single shard):
6+
# sbatch slurm/build_tool_nll.sh \
7+
# --model-config configs/models/laguna_xs2.yaml \
8+
# --generation-config configs/generation_laguna_xs2.yaml \
9+
# --traj-dir generations/swebench/laguna_xs2_full \
10+
# --output cache/swebench/laguna_xs2_full/tool_nll_index.pt
11+
#
12+
# Laguna XS2 (array job, 8 shards):
13+
# sbatch --array=0-7 slurm/build_tool_nll.sh \
14+
# --model-config configs/models/laguna_xs2.yaml \
15+
# --generation-config configs/generation_laguna_xs2.yaml \
16+
# --traj-dir generations/swebench/laguna_xs2_full \
17+
# --output cache/swebench/laguna_xs2_full/tool_nll_index.pt
18+
#
19+
# After all shards complete, merge with:
20+
# uv run python -c "
21+
# import torch; from pathlib import Path
22+
# shards = sorted(Path('cache/swebench/laguna_xs2_full').glob('tool_nll_index.pt.shard*.pt'))
23+
# merged = {}
24+
# for p in shards:
25+
# merged.update(torch.load(p, weights_only=False))
26+
# torch.save(merged, 'cache/swebench/laguna_xs2_full/tool_nll_index.pt')
27+
# print(f'Merged {len(shards)} shards → {len(merged)} samples')
28+
# "
29+
#
30+
#SBATCH -J pp-tool-nll
31+
#SBATCH -p berzelius
32+
#SBATCH --gpus=4
33+
#SBATCH -t 06:00:00
34+
#SBATCH -o logs/build_tool_nll_%j.out
35+
#SBATCH -e logs/build_tool_nll_%j.err
36+
37+
set -euo pipefail
38+
mkdir -p logs
39+
40+
module load buildenv-gcccuda/12.4.1-gcc13.3.0
41+
unset CPATH
42+
export LIBRARY_PATH="/usr/local/cuda/lib64:${LIBRARY_PATH:-}"
43+
export CUDA_HOME=/usr/local/cuda
44+
export PATH="/usr/local/cuda/bin:$PATH"
45+
46+
RANK=${SLURM_ARRAY_TASK_ID:-0}
47+
NUM_SHARDS=${SLURM_ARRAY_TASK_COUNT:-1}
48+
49+
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
50+
51+
uv run python build_tool_nll_index.py \
52+
--shard-rank "$RANK" \
53+
--num-shards "$NUM_SHARDS" \
54+
"$@"

src/figures.py

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,3 +199,93 @@ def plot_lookahead_horizon(
199199
suffix = f"_{filename_suffix}" if filename_suffix else ""
200200
plt.savefig(out_dir / f"{probe_name}_lookahead{suffix}.png", dpi=150, bbox_inches="tight")
201201
plt.close()
202+
203+
204+
def plot_tool_nll_correlation(
205+
run_id: str,
206+
probe_name: str,
207+
probe_layers: list[int],
208+
results_dir: str = "results",
209+
figures_dir: str = "figures",
210+
filename_suffix: str = "",
211+
) -> None:
212+
"""Scatter plot of tool output NLL vs per-step Brier score with LOWESS trend.
213+
214+
Reads <results_dir>/<run_id>/<probe>/nll_corr.pt and produces one figure
215+
per layer showing the correlation. Spearman ρ is annotated on the plot.
216+
"""
217+
import matplotlib.pyplot as plt
218+
from pathlib import Path
219+
220+
nll_path = Path(results_dir) / run_id / probe_name / "nll_corr.pt"
221+
if not nll_path.exists():
222+
print(f"[figures] nll_corr.pt not found at {nll_path}, skipping")
223+
return
224+
nll_corr = torch.load(nll_path, weights_only=False)
225+
226+
out_dir = Path(figures_dir) / run_id
227+
out_dir.mkdir(parents=True, exist_ok=True)
228+
229+
title = _PROBE_DISPLAY_NAME.get(probe_name, probe_name)
230+
suffix = f"_{filename_suffix}" if filename_suffix else ""
231+
232+
for layer_idx in probe_layers:
233+
if layer_idx not in nll_corr:
234+
continue
235+
d = nll_corr[layer_idx]
236+
nll_vals = d["tool_nll"]
237+
brier_vals = d["brier"]
238+
rho = d["spearman_nll_brier"]
239+
n_with = d["n_with_nll"]
240+
241+
valid = ~np.isnan(nll_vals)
242+
if valid.sum() < 2:
243+
continue
244+
245+
x = nll_vals[valid]
246+
y = brier_vals[valid]
247+
248+
plt.style.use("seaborn-v0_8-white")
249+
fig, ax = plt.subplots(figsize=(6, 5))
250+
251+
# Scatter (subsampled for readability if very large)
252+
max_points = 5000
253+
if len(x) > max_points:
254+
rng = np.random.default_rng(42)
255+
idx = rng.choice(len(x), max_points, replace=False)
256+
xs, ys = x[idx], y[idx]
257+
else:
258+
xs, ys = x, y
259+
260+
ax.scatter(xs, ys, alpha=0.15, s=8, color="#3a7fc1", linewidths=0)
261+
262+
# LOWESS smoothed trend line
263+
try:
264+
from statsmodels.nonparametric.smoothers_lowess import lowess
265+
order = np.argsort(x)
266+
smoothed = lowess(y[order], x[order], frac=0.3, return_sorted=True)
267+
ax.plot(smoothed[:, 0], smoothed[:, 1], color="#c0392b", lw=2, label=f"LOWESS")
268+
except ImportError:
269+
# Fallback: binned means
270+
n_bins = 20
271+
edges = np.percentile(x, np.linspace(0, 100, n_bins + 1))
272+
bin_x, bin_y = [], []
273+
for i in range(n_bins):
274+
m = (x >= edges[i]) & (x < edges[i + 1])
275+
if m.sum() > 0:
276+
bin_x.append(x[m].mean())
277+
bin_y.append(y[m].mean())
278+
ax.plot(bin_x, bin_y, color="#c0392b", lw=2, marker="o", ms=4)
279+
280+
rho_str = f"ρ = {rho:.3f}" if not np.isnan(rho) else "ρ = n/a"
281+
ax.set_xlabel("Tool output NLL (model surprise)", fontsize=11)
282+
ax.set_ylabel("Brier score (per step)", fontsize=11)
283+
ax.set_title(f"{title} — layer {layer_idx}\n{rho_str} (n = {n_with:,})", fontsize=12)
284+
ax.spines["top"].set_visible(False)
285+
ax.spines["right"].set_visible(False)
286+
287+
plt.tight_layout()
288+
fname = f"{probe_name}_tool_nll_layer{layer_idx}{suffix}.png"
289+
plt.savefig(out_dir / fname, dpi=150, bbox_inches="tight")
290+
plt.close()
291+
print(f"[figures] saved {out_dir / fname}")

0 commit comments

Comments
 (0)