Add token and cost charts to launch readiness

Author: aseembits93

experiment/locality-mcp-comparison/README.md

@@ -320,6 +320,22 @@ Important artifacts:
 - `scenarios/<scenario>/locality.folded` and `scenarios/<scenario>/notion-mcp.folded` - FlameGraph-compatible folded stacks generated from the same timing spans.
 - `scenarios/<scenario>/locality.snakeviz.prof` and `scenarios/<scenario>/notion-mcp.snakeviz.prof` - SnakeViz-compatible synthetic pstats profiles.
 - `scenarios/<scenario>/locality.snakeviz.stats.md` and `scenarios/<scenario>/notion-mcp.snakeviz.stats.md` - text summary of the SnakeViz profile frames.
+- `token-usage/by-trial-scenario/*.svg` - stacked token-usage charts with one
+  Locality bar and one MCP bar for each trial/scenario pair.
+- `token-usage/average.svg` - stacked token-usage chart averaged over paired
+  scenarios and trials.
+- `token-usage/cost/by-trial-scenario/*.svg` - stacked cost charts using the
+  same token buckets and one Locality/MCP bar pair per trial/scenario.
+- `token-usage/cost/average.svg` - stacked cost chart averaged over paired
+  scenarios and trials.
+- `token-usage/token-usage.tsv`, `token-usage/cost-usage.tsv`, and
+  `token-usage/token-usage.json` - chart data, cost data, pricing, and manifest.
+
+Cost charts default to the `gpt-5.6-luna` Standard short-context rates used by
+the benchmark harness. Override them for Azure/internal billing with
+`CODEX_COST_INPUT_USD_PER_1M`, `CODEX_COST_CACHED_INPUT_USD_PER_1M`,
+`CODEX_COST_CACHE_WRITE_INPUT_USD_PER_1M`, `CODEX_COST_OUTPUT_USD_PER_1M`, and
+`CODEX_COST_REASONING_OUTPUT_USD_PER_1M`.
 - `locality-traces/*.jsonl` - raw Locality command and pull/hydration spans.
 - `locality-traces/*-summary.json` - top Locality spans by duration.
 - `locality-traces/*-spans.tsv` - tabular Locality span data.

experiment/locality-mcp-comparison/run-agent-comparison.sh

@@ -384,6 +384,9 @@ run_launch_strategy "$LOCALITY_SANDBOX" "locality" "$LOCALITY_REMOTE_OUT_DIR" "$
 run_launch_strategy "$MCP_SANDBOX" "notion-mcp" "$MCP_REMOTE_OUT_DIR" "$@"
 sync_artifacts "$LOCALITY_SANDBOX" "locality" "$LOCALITY_REMOTE_OUT_DIR"
 sync_artifacts "$MCP_SANDBOX" "notion-mcp" "$MCP_REMOTE_OUT_DIR"
+if [ "$SYNC_ARTIFACTS" = "1" ]; then
+  python3 "$SCRIPT_DIR/scripts/token-usage-charts.py" "$LOCAL_OUT_DIR/artifacts" "$LOCAL_OUT_DIR/token-usage" >/dev/null
+fi
 
 cat > "$LOCAL_OUT_DIR/artifacts.tsv" <<EOF
 strategy	sandbox	remote_out_dir	local_stdout	local_stderr	local_artifact_dir
@@ -396,4 +399,5 @@ echo "Locality artifacts: $LOCALITY_SANDBOX:$LOCALITY_REMOTE_OUT_DIR"
 echo "MCP artifacts: $MCP_SANDBOX:$MCP_REMOTE_OUT_DIR"
 if [ "$SYNC_ARTIFACTS" = "1" ]; then
   echo "Local copies: $LOCAL_OUT_DIR/artifacts/locality and $LOCAL_OUT_DIR/artifacts/notion-mcp"
+  echo "Token usage charts: $LOCAL_OUT_DIR/token-usage"
 fi

experiment/locality-mcp-comparison/run-launch-readiness-benchmark.sh

@@ -1675,3 +1675,6 @@ summary = {
 Path(summary_path).write_text(json.dumps(summary, indent=2) + "\n")
 print(json.dumps(summary, indent=2))
 PY
+
+python3 "$SCRIPT_DIR/scripts/token-usage-charts.py" "$OUT_DIR" "$OUT_DIR/token-usage" >/dev/null
+echo "Token usage charts: $OUT_DIR/token-usage"

experiment/locality-mcp-comparison/scripts/summarize-runs.py

@@ -1,18 +1,40 @@
 #!/usr/bin/env python3
 import json
+import subprocess
 import statistics
 import sys
 from pathlib import Path
 
 root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("experiment/runs")
 summaries = sorted(root.glob("**/summary.json"))
+STRATEGY_DIRS = {"locality", "notion-mcp"}
+
+
+def run_id_for_summary(path):
+    parent = path.parent
+    try:
+        rel_parts = parent.relative_to(root).parts
+    except ValueError:
+        rel_parts = parent.parts
+
+    if not rel_parts:
+        return root.name
+    if len(rel_parts) >= 2 and rel_parts[-2] == "artifacts" and rel_parts[-1] in STRATEGY_DIRS:
+        trial_parts = rel_parts[:-2]
+        return "/".join(trial_parts) if trial_parts else root.parent.name
+    if len(rel_parts) >= 1 and rel_parts[-1] in STRATEGY_DIRS:
+        trial_parts = rel_parts[:-1]
+        return "/".join(trial_parts) if trial_parts else root.name
+    return "/".join(rel_parts)
+
+
 rows = []
 for path in summaries:
     data = json.loads(path.read_text())
     metrics = data.get("metrics", [])
     if not metrics:
         continue
-    run_id = str(path.parent.relative_to(root))
+    run_id = run_id_for_summary(path)
     for metric in metrics:
         rows.append(
             {
@@ -48,3 +70,12 @@
 out = root / "repeated-summary.md"
 out.write_text("\n".join(lines) + "\n")
 print(out)
+
+chart_script = Path(__file__).with_name("token-usage-charts.py")
+if rows and chart_script.exists():
+    subprocess.run(
+        [sys.executable, str(chart_script), str(root), str(root / "token-usage")],
+        check=True,
+        stdout=subprocess.DEVNULL,
+    )
+    print(root / "token-usage")