Skip to content

Commit 24f7874

Browse files
andre15silvaclaude
andcommitted
fix(paper): drop train/val/test state columns from dataset stats table
Avoids exposing the repo-level vs instance-level split inconsistency between Verified (12 repos) and Pro (434 instances). Table is now: Model / Dataset / #Traj. / #Tokens / >=15 turns / >=50 turns. Fits single column again (table, not table*). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 563e82d commit 24f7874

1 file changed

Lines changed: 11 additions & 25 deletions

File tree

run_paper_figures.py

Lines changed: 11 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1255,23 +1255,20 @@ def build_dataset_stats_table(
12551255
r"Dataset statistics per (model, dataset) combination. "
12561256
r"\#Traj.\ counts all agent runs including those that hit the step limit. "
12571257
r"\#Tokens is the total token count across all runs. "
1258-
r"Train, Val, and Test report hidden-state positions (stride $s=5$, "
1259-
r"70/15/15\,\% split by task) for the syntactic-correctness probe; "
1260-
r"other probes use the same split. "
12611258
r"$\geq$15 and $\geq$50 count trajectories reaching those turn thresholds, "
12621259
r"the length filters used in the lookahead experiments. "
12631260
r"The $k_{\max}=15$ filter is nearly lossless (90--99\,\% of runs); "
12641261
r"the $k_{\max}=50$ filter retains 40--60\,\%."
12651262
)
12661263
header = (
1267-
r"\begin{table*}[t]" "\n"
1264+
r"\begin{table}[t]" "\n"
12681265
r"\centering" "\n"
12691266
r"\small" "\n"
12701267
r"\caption{" + caption + r"}" "\n"
12711268
r"\label{tab:dataset_stats}" "\n"
1272-
r"\begin{tabular}{llrrrrrrr}" "\n"
1269+
r"\begin{tabular}{llrrrr}" "\n"
12731270
r"\toprule" "\n"
1274-
r"Model & Dataset & \#Traj. & \#Tokens & Train states & Val states & Test states"
1271+
r"Model & Dataset & \#Traj. & \#Tokens"
12751272
r" & \#Traj.\ $\geq$15 turns & \#Traj.\ $\geq$50 turns \\" "\n"
12761273
r"\midrule" "\n"
12771274
)
@@ -1280,37 +1277,26 @@ def build_dataset_stats_table(
12801277
if cfg.get("placeholder"):
12811278
rows.append(
12821279
f"{cfg['model_label']} & {cfg['dataset_label']} & "
1283-
r"\multicolumn{7}{c}{---} \\"
1280+
r"\multicolumn{4}{c}{---} \\"
12841281
)
12851282
continue
1286-
stats = cfg["gen_stats"]
1287-
n_traj = len(stats)
1288-
n_tok = sum(s.get("n_tokens", 0) for s in stats)
1289-
gt15 = sum(1 for s in stats if s["turns"] >= 15)
1290-
gt50 = sum(1 for s in stats if s["turns"] >= 50)
1291-
1292-
n_train = n_val = n_test = "—"
1293-
rpt = cfg.get("results_pt_path")
1294-
if rpt and Path(rpt).exists():
1295-
r = torch.load(rpt, weights_only=False)
1296-
layer = list(r.keys())[0]
1297-
res0 = r[layer][0]
1298-
n_train = f"{res0.n_train:,}"
1299-
n_val = f"{res0.n_val:,}"
1300-
n_test = f"{res0.n_test:,}"
1283+
stats = cfg["gen_stats"]
1284+
n_traj = len(stats)
1285+
n_tok = sum(s.get("n_tokens", 0) for s in stats)
1286+
gt15 = sum(1 for s in stats if s["turns"] >= 15)
1287+
gt50 = sum(1 for s in stats if s["turns"] >= 50)
13011288

13021289
rows.append(
13031290
f"{cfg['model_label']} & {cfg['dataset_label']} & "
1304-
f"{n_traj:,} & {n_tok:,} & {n_train} & {n_val} & {n_test}"
1305-
f" & {gt15:,} & {gt50:,} \\\\"
1291+
f"{n_traj:,} & {n_tok:,} & {gt15:,} & {gt50:,} \\\\"
13061292
)
13071293

13081294
body = "\n".join(rows)
13091295
footer = (
13101296
"\n"
13111297
r"\bottomrule" "\n"
13121298
r"\end{tabular}" "\n"
1313-
r"\end{table*}"
1299+
r"\end{table}"
13141300
)
13151301
tex = header + body + footer
13161302
out_path.parent.mkdir(parents=True, exist_ok=True)

0 commit comments

Comments
 (0)