feat: [US-007] - [Add transcript discovery abstraction for non-Claude artifacts]

Author: sjarmak

.beads/issues.jsonl

@@ -128,12 +128,13 @@
 {"id":"CodeContextBench-pdk","title":"Fix instruction contamination + re-extract all token metrics","status":"closed","priority":1,"issue_type":"bug","owner":"locobench@anthropic.com","created_at":"2026-02-07T16:04:36.872665156Z","created_by":"LoCoBench Bot","updated_at":"2026-02-07T16:11:05.467083794Z","closed_at":"2026-02-07T16:11:05.467083794Z","close_reason":"Fixed 6 contaminated files (RepoQA template, 5 LargeRepo CLAUDE.md); re-extracted 197 task_metrics.json fixing 139 inflated costs (8.7K -\u003e 50 actual total). Created scripts/reextract_all_metrics.py."}
 {"id":"CodeContextBench-pss","title":"US-006a: Scaffold 3 architectural understanding tasks (Tier A)","status":"closed","priority":1,"issue_type":"task","owner":"locobench@anthropic.com","created_at":"2026-02-15T23:04:39.305991853Z","created_by":"LoCoBench Bot","updated_at":"2026-02-15T23:10:55.114709246Z","closed_at":"2026-02-15T23:10:55.114709246Z","close_reason":"US-006a complete: 3 Tier A architectural understanding tasks scaffolded"}
 {"id":"CodeContextBench-q27","title":"Add canary guardrails + subscription enforcement to _common.sh","status":"closed","priority":1,"issue_type":"task","owner":"locobench@anthropic.com","created_at":"2026-02-08T03:36:11.761732866Z","created_by":"LoCoBench Bot","updated_at":"2026-02-08T03:37:32.353683884Z","closed_at":"2026-02-08T03:37:32.353683884Z","close_reason":"Added enforce_subscription_mode, validate_canary_result, run_canary_then_batch, check_token_health; removed API key branch from setup_multi_accounts"}
-{"id":"CodeContextBench-qtf","title":"US-007 Add transcript discovery abstraction for non-Claude artifacts","status":"open","priority":2,"issue_type":"task","owner":"locobench@anthropic.com","created_at":"2026-02-17T03:33:11.290586345Z","created_by":"LoCoBench Bot","updated_at":"2026-02-17T03:33:11.290586345Z"}
+{"id":"CodeContextBench-qtf","title":"US-007 Add transcript discovery abstraction for non-Claude artifacts","status":"closed","priority":2,"issue_type":"task","owner":"locobench@anthropic.com","created_at":"2026-02-17T03:33:11.290586345Z","created_by":"LoCoBench Bot","updated_at":"2026-02-17T04:00:12.261541882Z","closed_at":"2026-02-17T04:00:12.261541882Z","close_reason":"done"}
 {"id":"CodeContextBench-r71","title":"CrossRepo: all runs invalid due to verifier path bug","description":"All 8 CrossRepo runs (4 tasks × 2 configs) crashed because test.sh referenced /task/tests/expected_changes.json instead of /tests/expected_changes.json. Verifier is now fixed locally but all existing runs predate the fix. Agents produced meaningful output (261-line patch, 224-line analysis, 497-line reasoning). All 4 tasks need reruns.","status":"closed","priority":1,"issue_type":"bug","owner":"locobench@anthropic.com","created_at":"2026-02-06T22:03:15.909834308Z","created_by":"LoCoBench Bot","updated_at":"2026-02-07T18:39:18.97810564Z","closed_at":"2026-02-07T18:39:18.97810564Z","close_reason":"CrossRepo all 3 configs rerun complete: baseline avg=0.571, SG_base avg=0.587, SG_full avg=0.387"}
 {"id":"CodeContextBench-rch","title":"US-019: Scaffold enterprise multi-team and conflicting-docs tasks","status":"closed","priority":1,"issue_type":"task","owner":"locobench@anthropic.com","created_at":"2026-02-15T15:00:15.881711075Z","created_by":"LoCoBench Bot","updated_at":"2026-02-15T15:08:20.111033768Z","closed_at":"2026-02-15T15:08:20.111033768Z","close_reason":"US-019 complete: 3 enterprise tasks scaffolded"}
 {"id":"CodeContextBench-rej","title":"Generate aggregate CCB evaluation report (updated: 12 benchmarks, no LoCoBench)","description":"After all benchmark runs complete and MANIFEST is clean, generate the aggregate evaluation report using python3 scripts/generate_report.py. Should cover all 13 benchmarks with 3-config comparison (baseline vs SG_base vs SG_full), MCP impact analysis, and per-benchmark breakdowns.","status":"closed","priority":2,"issue_type":"task","owner":"locobench@anthropic.com","created_at":"2026-02-06T14:50:31.544649793Z","created_by":"LoCoBench Bot","updated_at":"2026-02-16T01:39:42.151139942Z","closed_at":"2026-02-16T01:39:42.151139942Z","close_reason":"Stale - aggregate report design predates enterprise task expansion and IR pipeline work. Needs redesign.","dependencies":[{"issue_id":"CodeContextBench-rej","depends_on_id":"CodeContextBench-aot","type":"blocks","created_at":"2026-02-06T14:50:47.565065613Z","created_by":"LoCoBench Bot"},{"issue_id":"CodeContextBench-rej","depends_on_id":"CodeContextBench-kph","type":"blocks","created_at":"2026-02-06T14:50:47.632620141Z","created_by":"LoCoBench Bot"},{"issue_id":"CodeContextBench-rej","depends_on_id":"CodeContextBench-yk3","type":"blocks","created_at":"2026-02-06T14:50:47.689660185Z","created_by":"LoCoBench Bot"},{"issue_id":"CodeContextBench-rej","depends_on_id":"CodeContextBench-9r9","type":"blocks","created_at":"2026-02-06T14:50:47.744576933Z","created_by":"LoCoBench Bot"},{"issue_id":"CodeContextBench-rej","depends_on_id":"CodeContextBench-05n","type":"blocks","created_at":"2026-02-06T14:50:47.799295655Z","created_by":"LoCoBench Bot"},{"issue_id":"CodeContextBench-rej","depends_on_id":"CodeContextBench-99h","type":"blocks","created_at":"2026-02-06T14:50:47.854278452Z","created_by":"LoCoBench Bot"},{"issue_id":"CodeContextBench-rej","depends_on_id":"CodeContextBench-dfp","type":"blocks","created_at":"2026-02-06T14:50:47.909843823Z","created_by":"LoCoBench Bot"},{"issue_id":"CodeContextBench-rej","depends_on_id":"CodeContextBench-8t7","type":"blocks","created_at":"2026-02-08T01:13:11.737288558Z","created_by":"LoCoBench Bot"},{"issue_id":"CodeContextBench-rej","depends_on_id":"CodeContextBench-nj3","type":"blocks","created_at":"2026-02-08T02:54:34.482946883Z","created_by":"LoCoBench Bot"},{"issue_id":"CodeContextBench-rej","depends_on_id":"CodeContextBench-36d","type":"blocks","created_at":"2026-02-08T02:54:34.538759931Z","created_by":"LoCoBench Bot"},{"issue_id":"CodeContextBench-rej","depends_on_id":"CodeContextBench-1up","type":"blocks","created_at":"2026-02-08T02:54:34.594203831Z","created_by":"LoCoBench Bot"},{"issue_id":"CodeContextBench-rej","depends_on_id":"CodeContextBench-0y8","type":"blocks","created_at":"2026-02-08T02:54:35.06846015Z","created_by":"LoCoBench Bot"},{"issue_id":"CodeContextBench-rej","depends_on_id":"CodeContextBench-24z","type":"blocks","created_at":"2026-02-08T02:54:35.124147115Z","created_by":"LoCoBench Bot"}]}
 {"id":"CodeContextBench-rf3","title":"US-002: Fix protonmail Docker environment","status":"closed","priority":2,"issue_type":"task","owner":"locobench@anthropic.com","created_at":"2026-02-11T23:31:45.49023811Z","created_by":"LoCoBench Bot","updated_at":"2026-02-11T23:39:17.748141376Z","closed_at":"2026-02-11T23:39:17.748141376Z","close_reason":"Fixed protonmail Docker Node.js v16→v18 in local + cached Dockerfiles"}
 {"id":"CodeContextBench-rxg","title":"Rerun 7 LoCoBench SG_base zero-token gap-fill tasks","description":"7 LoCoBench tasks in locobench_gapfill_opus_20260209_010036/sourcegraph_base have zero tokens (auth failure). Tasks: c_api_graphql_expert_079 (arch+cross_file), rust_microservice_expert_008, csharp_warehouse_expert_012 (2), python_streaming_expert_085, python_desktop_expert. Current SG_base mean=0.504 (18 valid) but MANIFEST shows 0.363 including errored. Fix errored classification is done but these need actual reruns for complete data.","status":"closed","priority":1,"issue_type":"task","owner":"locobench@anthropic.com","created_at":"2026-02-10T11:28:20.889991278Z","created_by":"LoCoBench Bot","updated_at":"2026-02-15T19:31:57.593499773Z","closed_at":"2026-02-15T19:31:57.593499773Z","close_reason":"SG_base config dropped from official runs"}
+{"id":"CodeContextBench-s00t","title":"US-007 - Add transcript discovery abstraction for non-Claude artifacts","status":"closed","priority":2,"issue_type":"task","owner":"locobench@anthropic.com","created_at":"2026-02-17T03:57:12.383536394Z","created_by":"LoCoBench Bot","updated_at":"2026-02-17T03:57:29.113635367Z","closed_at":"2026-02-17T03:57:29.113635367Z","close_reason":"duplicate"}
 {"id":"CodeContextBench-si6","title":"US-003: Retrieval-to-outcome correlation analysis","status":"closed","priority":1,"issue_type":"task","owner":"locobench@anthropic.com","created_at":"2026-02-16T01:06:44.928313476Z","created_by":"LoCoBench Bot","updated_at":"2026-02-16T01:34:05.037762777Z","closed_at":"2026-02-16T01:34:05.037762777Z","close_reason":"Done - compute_retrieval_outcome_correlation() and compute_mcp_value_scores() already on main. Spearman correlation with scatter plot output."}
 {"id":"CodeContextBench-szi","title":"Fix judge JSON parsing: strip markdown code fences","status":"closed","priority":0,"issue_type":"bug","owner":"locobench@anthropic.com","created_at":"2026-02-07T19:44:44.132260075Z","created_by":"LoCoBench Bot","updated_at":"2026-02-07T20:03:44.608413235Z","closed_at":"2026-02-07T20:03:44.608413235Z","close_reason":"Added code fence stripping to CodeReview test.sh (3 tasks) and RepoQA test.sh (10 tasks + template). Agents that wrap review.json or solution.json in markdown fences now get parsed correctly."}
 {"id":"CodeContextBench-szv","title":"US-002: Create inv-deep-002 Istio control plane deep causal chain task","status":"closed","priority":2,"issue_type":"task","owner":"locobench@anthropic.com","created_at":"2026-02-16T15:14:17.279712198Z","created_by":"LoCoBench Bot","updated_at":"2026-02-16T15:27:43.319743852Z","closed_at":"2026-02-16T15:27:43.319743852Z","close_reason":"US-002 complete: inv-deep-002 Istio deep causal chain task created and committed"}

ralph-multi-harness/prd.json

@@ -106,7 +106,7 @@
         "python3 -m py_compile scripts/ccb_metrics/discovery.py scripts/ccb_metrics/judge_context.py scripts/ccb_metrics/extractors.py succeeds"
       ],
       "priority": 7,
-      "passes": false,
+      "passes": true,
       "notes": ""
     },
     {

ralph-multi-harness/progress.txt

@@ -8,6 +8,7 @@
 - Harness registry entries should be keyed by stable harness IDs (`codex`, `cursor`, `gemini`, `copilot`, `openhands`) and constrain `allowed_mcp_modes` to exactly `none` and `sourcegraph_full` for this rollout.
 - Non-Claude harness runners should still source `configs/_common.sh` for validation/parallel helpers, but must not depend on Claude OAuth refresh flows.
 - In sandboxed environments, `runs/staging` may resolve to an external symlink target; use a writable `--category` override when dry-running scaffolds locally.
+- In `scripts/ccb_metrics`, resolve transcript artifacts through a shared candidate list (not hardcoded `agent/claude-code.txt`) so non-Claude harness outputs are discoverable.
 
 ## Progress
 
@@ -83,3 +84,16 @@
   - Useful context (e.g., "the evaluation panel is in component X")
     - `runs/staging` is a symlink in this repo setup; local dry runs may need a different `--category` to avoid permission issues when the symlink target is outside writable roots.
 ---
+
+## 2026-02-17 03:59:26 UTC - US-007
+- Implemented multi-harness transcript discovery abstraction with shared candidate-path resolution and wired it into run discovery + judge context generation.
+- Updated transcript-driven extractors to recover gracefully when `agent/claude-code.txt` is missing by resolving harness-appropriate alternatives from the task directory.
+- Files changed: `scripts/ccb_metrics/transcript_paths.py`, `scripts/ccb_metrics/discovery.py`, `scripts/ccb_metrics/judge_context.py`, `scripts/ccb_metrics/extractors.py`, `ralph-multi-harness/prd.json`, `ralph-multi-harness/progress.txt`
+- **Learnings for future iterations:**
+  - Patterns discovered (e.g., "this codebase uses X for Y")
+    - Keep transcript filename fallbacks centralized (`scripts/ccb_metrics/transcript_paths.py`) so discovery, judge context, and extractors stay consistent as harness artifacts evolve.
+  - Gotchas encountered (e.g., "don't forget to update Z when changing W")
+    - `scripts/ccb_metrics/extractors.py` cannot import from `discovery.py` due dependency direction, so shared transcript-path utilities should live in a separate helper module.
+  - Useful context (e.g., "the evaluation panel is in component X")
+    - `extract_run_config` now reads init metadata from whichever transcript candidate resolves first, improving MCP-mode inference for non-Claude harness runs.
+---

scripts/ccb_metrics/discovery.py

@@ -16,6 +16,10 @@
 
 from .models import TaskMetrics, RunMetrics
 from .task_selection import normalize_benchmark_name
+from .transcript_paths import (
+    TRANSCRIPT_CANDIDATE_RELATIVE_PATHS,
+    resolve_task_transcript_path as _resolve_task_transcript_path,
+)
 from .extractors import (
     extract_task_from_result_json,
     extract_task_tokens_from_transcript,
@@ -41,6 +45,14 @@
 )
 
 
+def resolve_task_transcript_path(task_dir: Path) -> Path:
+    """Resolve transcript path by trying multiple harness artifact names."""
+    # Keep helper in this module so discovery users can share one resolver.
+    # This list intentionally covers non-Claude harness artifact variants.
+    _ = TRANSCRIPT_CANDIDATE_RELATIVE_PATHS
+    return _resolve_task_transcript_path(task_dir)
+
+
 def _infer_benchmark(run_name: str) -> str:
     """Infer benchmark name from run directory name.
 
@@ -151,7 +163,7 @@ def _process_task_dir(
     # Token data: prefer transcript (actual API usage) over result.json
     # (result.json n_input_tokens can include cumulative MCP result tokens,
     # inflating counts by 100x for MCP-enabled runs)
-    transcript_path = task_dir / "agent" / "claude-code.txt"
+    transcript_path = resolve_task_transcript_path(task_dir)
     tokens = extract_task_tokens_from_transcript(transcript_path)
     if tokens.get("input_tokens") is not None:
         tm.input_tokens = tokens["input_tokens"]
@@ -189,7 +201,7 @@ def _process_task_dir(
 
     # Tool usage — prefer trajectory, fall back to transcript
     trajectory_path = task_dir / "agent" / "trajectory.json"
-    transcript_path = task_dir / "agent" / "claude-code.txt"
+    transcript_path = resolve_task_transcript_path(task_dir)
     tool_usage = extract_tool_usage_from_trajectory(trajectory_path)
     if tool_usage.get("tool_calls_total") is None:
         tool_usage = extract_tool_usage_from_transcript(transcript_path)
@@ -351,7 +363,7 @@ def discover_runs(runs_dir: str | Path) -> list[RunMetrics]:
                 _transcript_for_config = None
                 for _td in sorted(batch_dir.iterdir()):
                     if _is_task_dir(_td):
-                        _candidate = _td / "agent" / "claude-code.txt"
+                        _candidate = resolve_task_transcript_path(_td)
                         if _candidate.is_file():
                             _transcript_for_config = _candidate
                             break

scripts/ccb_metrics/extractors.py

@@ -14,6 +14,7 @@
 from typing import Optional
 
 from .models import TaskMetrics
+from .transcript_paths import infer_task_dir_from_transcript_path, resolve_task_transcript_path
 
 
 def _parse_iso(ts: Optional[str]) -> Optional[datetime]:
@@ -36,6 +37,20 @@ def _seconds_between(start: Optional[str], end: Optional[str]) -> Optional[float
     return (e - s).total_seconds()
 
 
+def _resolve_existing_transcript_path(transcript_path: str | Path) -> Path:
+    """Resolve transcript path with task-level fallback candidates."""
+    path = Path(transcript_path)
+    if path.is_file():
+        return path
+
+    task_dir = infer_task_dir_from_transcript_path(path)
+    if task_dir is not None:
+        candidate = resolve_task_transcript_path(task_dir)
+        if candidate.is_file():
+            return candidate
+    return path
+
+
 def extract_task_from_result_json(
     result_json_path: str | Path,
     benchmark: str = "",
@@ -174,7 +189,7 @@ def extract_task_tokens_from_transcript(
         "cache_read_input_tokens": None,
         "total_cost_usd": None,
     }
-    path = Path(claude_code_txt_path)
+    path = _resolve_existing_transcript_path(claude_code_txt_path)
     if not path.is_file():
         return empty
 
@@ -346,7 +361,7 @@ def extract_tool_usage_from_transcript(
         tool_calls_by_name (Counter dict), mcp_ratio (mcp/total).
         Returns None-valued dict if file is missing or unparseable.
     """
-    path = Path(claude_code_txt_path)
+    path = _resolve_existing_transcript_path(claude_code_txt_path)
     if not path.is_file():
         return _empty_tool_usage()
 
@@ -436,7 +451,7 @@ def extract_run_config(
 
     # --- Extract from claude-code.txt init line ---
     if transcript_path is not None:
-        tp = Path(transcript_path)
+        tp = _resolve_existing_transcript_path(transcript_path)
         if tp.is_file():
             try:
                 for line in tp.open():
@@ -625,7 +640,7 @@ def extract_search_patterns_from_transcript(
     Returns:
         Dict with same schema as extract_search_patterns_from_trajectory.
     """
-    path = Path(claude_code_txt_path)
+    path = _resolve_existing_transcript_path(claude_code_txt_path)
     if not path.is_file():
         return _empty_search_patterns()
 
@@ -755,7 +770,7 @@ def extract_code_changes_from_transcript(
     Returns:
         Dict with same schema as extract_code_changes_from_trajectory.
     """
-    path = Path(claude_code_txt_path)
+    path = _resolve_existing_transcript_path(claude_code_txt_path)
     if not path.is_file():
         return _empty_code_changes()
 
@@ -1095,7 +1110,7 @@ def extract_conversation_analysis_from_transcript(
         "backtrack_count": None,
         "context_window_peak_pct": None,
     }
-    path = Path(claude_code_txt_path)
+    path = _resolve_existing_transcript_path(claude_code_txt_path)
     if not path.is_file():
         return empty
 

scripts/ccb_metrics/judge_context.py

@@ -16,7 +16,13 @@
 from pathlib import Path
 from typing import Optional
 
-from .discovery import discover_runs, _extract_task_id, _is_batch_dir, _is_task_dir
+from .discovery import (
+    discover_runs,
+    _extract_task_id,
+    _is_batch_dir,
+    _is_task_dir,
+    resolve_task_transcript_path,
+)
 from .extractors import (
     extract_tool_usage_from_trajectory,
     extract_tool_usage_from_transcript,
@@ -111,7 +117,7 @@ def _extract_agent_output(task_dir: Path) -> Optional[str]:
         return text
 
     # Fallback: last assistant message text from transcript
-    transcript_path = task_dir / "agent" / "claude-code.txt"
+    transcript_path = resolve_task_transcript_path(task_dir)
     if not transcript_path.is_file():
         return None
 
@@ -146,7 +152,7 @@ def _extract_agent_output(task_dir: Path) -> Optional[str]:
 def _extract_tool_usage_summary(task_dir: Path) -> Optional[dict]:
     """Extract tool usage summary: total calls, MCP calls, top 5 tools."""
     trajectory_path = task_dir / "agent" / "trajectory.json"
-    transcript_path = task_dir / "agent" / "claude-code.txt"
+    transcript_path = resolve_task_transcript_path(task_dir)
 
     usage = extract_tool_usage_from_trajectory(trajectory_path)
     if usage.get("tool_calls_total") is None:
@@ -173,7 +179,7 @@ def _extract_code_changes(task_dir: Path) -> Optional[list[dict]]:
     Returns a list of {file, action} dicts, or None if not available.
     """
     trajectory_path = task_dir / "agent" / "trajectory.json"
-    transcript_path = task_dir / "agent" / "claude-code.txt"
+    transcript_path = resolve_task_transcript_path(task_dir)
 
     changes: list[dict] = []
 
@@ -506,7 +512,7 @@ def generate_judge_contexts(
         ground_truth = _read_ground_truth(benchmarks_dir, benchmark, task_id)
 
         # Extract transcript summary
-        transcript_path = task_dir / "agent" / "claude-code.txt"
+        transcript_path = resolve_task_transcript_path(task_dir)
         transcript_summary = _extract_transcript_summary(transcript_path)
 
         # Extract agent output