Add baseline-vs-MCP IR aggregates and per-suite cost tables

Author: sjarmak

docs/BLOG_POST.md

@@ -155,6 +155,13 @@ I built an information retrieval evaluation pipeline alongside the task scoring
 
 The refreshed retrieval pipeline run confirms moderate retrieval quality overall (file recall 0.460, MRR 0.364), but a large fraction of traces still lack mapped ground truth files (488/799), which limits configuration-level retrieval comparisons.
 
+On the computable subset, aggregated baseline vs MCP retrieval metrics are:
+
+| Config Type | n | File Recall | MRR | MAP | Context Efficiency |
+|-------------|---|-------------|-----|-----|--------------------|
+| baseline | 132 | 0.330 | 0.346 | 0.231 | 0.184 |
+| mcp | 179 | 0.556 | 0.378 | 0.267 | 0.204 |
+
 But better retrieval doesn't always mean better outcomes. Still investigating this but likely finding the right files is necessary but not sufficient. The agent still has to correctly apply what it finds, and in some tasks the local code modification step is where removing local code availability from the MCP run environment hurts more than others.
 
 ## Patterns in the Retrieval-Outcome Pairing Data
@@ -175,6 +182,8 @@ Let's take a break from whatever voodoo variables control reward outcomes and ta
 
 This updated snapshot indicates MCP token/tool usage overhead is currently dominating cost in the analysis set.
 
+Suite-level cost is mixed: MCP is cheaper on several Org suites (for example crossorg −$0.062/task and incident −$0.048/task) but more expensive on some SDLC suites (refactor +$0.398/task, feature +$0.211/task). The full per-suite cost table is in the technical report.
+
 Speed tells an even cleaner story:
 
 | Metric | Baseline Mean | MCP Mean | Delta |

docs/analysis/analysis_refresh_tables_20260303.json

@@ -433,5 +433,179 @@
       "mean_reward_delta": 0.048,
       "reward_delta_variance": 0.059717
     }
+  },
+  "ir_by_type": {
+    "baseline": {
+      "n": 132,
+      "file_recall": 0.3295,
+      "mrr": 0.3462,
+      "map_score": 0.2307,
+      "context_efficiency": 0.1843
+    },
+    "mcp": {
+      "n": 179,
+      "file_recall": 0.5558,
+      "mrr": 0.3778,
+      "map_score": 0.2667,
+      "context_efficiency": 0.2043
+    }
+  },
+  "ir_by_config": {
+    "baseline-local-direct": {
+      "n": 132,
+      "file_recall": 0.3295,
+      "mrr": 0.3462,
+      "map_score": 0.2307,
+      "context_efficiency": 0.1843
+    },
+    "mcp-remote-direct": {
+      "n": 179,
+      "file_recall": 0.5558,
+      "mrr": 0.3778,
+      "map_score": 0.2667,
+      "context_efficiency": 0.2043
+    }
+  },
+  "cost_by_suite": {
+    "csb_org_compliance": {
+      "n": 18,
+      "baseline_cost_mean_usd": 0.2679,
+      "mcp_cost_mean_usd": 0.2521,
+      "delta_cost_mean_usd": -0.0158,
+      "delta_cost_variance": 0.003486
+    },
+    "csb_org_crossorg": {
+      "n": 15,
+      "baseline_cost_mean_usd": 0.2756,
+      "mcp_cost_mean_usd": 0.2136,
+      "delta_cost_mean_usd": -0.062,
+      "delta_cost_variance": 0.017902
+    },
+    "csb_org_crossrepo": {
+      "n": 14,
+      "baseline_cost_mean_usd": 0.2575,
+      "mcp_cost_mean_usd": 0.2523,
+      "delta_cost_mean_usd": -0.0052,
+      "delta_cost_variance": 0.005375
+    },
+    "csb_org_crossrepo_tracing": {
+      "n": 22,
+      "baseline_cost_mean_usd": 0.2478,
+      "mcp_cost_mean_usd": 0.2187,
+      "delta_cost_mean_usd": -0.0292,
+      "delta_cost_variance": 0.003282
+    },
+    "csb_org_domain": {
+      "n": 20,
+      "baseline_cost_mean_usd": 0.2108,
+      "mcp_cost_mean_usd": 0.2258,
+      "delta_cost_mean_usd": 0.015,
+      "delta_cost_variance": 0.003268
+    },
+    "csb_org_incident": {
+      "n": 20,
+      "baseline_cost_mean_usd": 0.2465,
+      "mcp_cost_mean_usd": 0.1989,
+      "delta_cost_mean_usd": -0.0476,
+      "delta_cost_variance": 0.007914
+    },
+    "csb_org_migration": {
+      "n": 26,
+      "baseline_cost_mean_usd": 0.2534,
+      "mcp_cost_mean_usd": 0.2501,
+      "delta_cost_mean_usd": -0.0033,
+      "delta_cost_variance": 0.009546
+    },
+    "csb_org_onboarding": {
+      "n": 28,
+      "baseline_cost_mean_usd": 0.1029,
+      "mcp_cost_mean_usd": 0.1049,
+      "delta_cost_mean_usd": 0.002,
+      "delta_cost_variance": 0.00086
+    },
+    "csb_org_org": {
+      "n": 15,
+      "baseline_cost_mean_usd": 0.2362,
+      "mcp_cost_mean_usd": 0.2193,
+      "delta_cost_mean_usd": -0.0169,
+      "delta_cost_variance": 0.00171
+    },
+    "csb_org_platform": {
+      "n": 18,
+      "baseline_cost_mean_usd": 0.194,
+      "mcp_cost_mean_usd": 0.2149,
+      "delta_cost_mean_usd": 0.0209,
+      "delta_cost_variance": 0.001999
+    },
+    "csb_org_security": {
+      "n": 24,
+      "baseline_cost_mean_usd": 0.2167,
+      "mcp_cost_mean_usd": 0.2146,
+      "delta_cost_mean_usd": -0.002,
+      "delta_cost_variance": 0.003105
+    },
+    "csb_sdlc_debug": {
+      "n": 18,
+      "baseline_cost_mean_usd": 0.3669,
+      "mcp_cost_mean_usd": 0.4569,
+      "delta_cost_mean_usd": 0.0901,
+      "delta_cost_variance": 0.02381
+    },
+    "csb_sdlc_design": {
+      "n": 14,
+      "baseline_cost_mean_usd": 0.41,
+      "mcp_cost_mean_usd": 0.359,
+      "delta_cost_mean_usd": -0.051,
+      "delta_cost_variance": 0.097988
+    },
+    "csb_sdlc_document": {
+      "n": 13,
+      "baseline_cost_mean_usd": 0.2669,
+      "mcp_cost_mean_usd": 0.2974,
+      "delta_cost_mean_usd": 0.0305,
+      "delta_cost_variance": 0.01439
+    },
+    "csb_sdlc_feature": {
+      "n": 23,
+      "baseline_cost_mean_usd": 0.4965,
+      "mcp_cost_mean_usd": 0.7079,
+      "delta_cost_mean_usd": 0.2114,
+      "delta_cost_variance": 0.183988
+    },
+    "csb_sdlc_fix": {
+      "n": 26,
+      "baseline_cost_mean_usd": 0.5997,
+      "mcp_cost_mean_usd": 0.7057,
+      "delta_cost_mean_usd": 0.1059,
+      "delta_cost_variance": 0.06587
+    },
+    "csb_sdlc_refactor": {
+      "n": 15,
+      "baseline_cost_mean_usd": 0.3194,
+      "mcp_cost_mean_usd": 0.7173,
+      "delta_cost_mean_usd": 0.398,
+      "delta_cost_variance": 0.147469
+    },
+    "csb_sdlc_secure": {
+      "n": 12,
+      "baseline_cost_mean_usd": 0.4825,
+      "mcp_cost_mean_usd": 0.5657,
+      "delta_cost_mean_usd": 0.0832,
+      "delta_cost_variance": 0.030859
+    },
+    "csb_sdlc_test": {
+      "n": 18,
+      "baseline_cost_mean_usd": 0.2641,
+      "mcp_cost_mean_usd": 0.2976,
+      "delta_cost_mean_usd": 0.0335,
+      "delta_cost_variance": 0.015625
+    },
+    "csb_sdlc_understand": {
+      "n": 10,
+      "baseline_cost_mean_usd": 0.3519,
+      "mcp_cost_mean_usd": 0.4475,
+      "delta_cost_mean_usd": 0.0956,
+      "delta_cost_variance": 0.022037
+    }
   }
 }

docs/analysis/analysis_set_metrics_20260303.json

@@ -433,5 +433,179 @@
       "mean_reward_delta": 0.048,
       "reward_delta_variance": 0.059717
     }
+  },
+  "ir_by_type": {
+    "baseline": {
+      "n": 132,
+      "file_recall": 0.3295,
+      "mrr": 0.3462,
+      "map_score": 0.2307,
+      "context_efficiency": 0.1843
+    },
+    "mcp": {
+      "n": 179,
+      "file_recall": 0.5558,
+      "mrr": 0.3778,
+      "map_score": 0.2667,
+      "context_efficiency": 0.2043
+    }
+  },
+  "ir_by_config": {
+    "baseline-local-direct": {
+      "n": 132,
+      "file_recall": 0.3295,
+      "mrr": 0.3462,
+      "map_score": 0.2307,
+      "context_efficiency": 0.1843
+    },
+    "mcp-remote-direct": {
+      "n": 179,
+      "file_recall": 0.5558,
+      "mrr": 0.3778,
+      "map_score": 0.2667,
+      "context_efficiency": 0.2043
+    }
+  },
+  "cost_by_suite": {
+    "csb_org_compliance": {
+      "n": 18,
+      "baseline_cost_mean_usd": 0.2679,
+      "mcp_cost_mean_usd": 0.2521,
+      "delta_cost_mean_usd": -0.0158,
+      "delta_cost_variance": 0.003486
+    },
+    "csb_org_crossorg": {
+      "n": 15,
+      "baseline_cost_mean_usd": 0.2756,
+      "mcp_cost_mean_usd": 0.2136,
+      "delta_cost_mean_usd": -0.062,
+      "delta_cost_variance": 0.017902
+    },
+    "csb_org_crossrepo": {
+      "n": 14,
+      "baseline_cost_mean_usd": 0.2575,
+      "mcp_cost_mean_usd": 0.2523,
+      "delta_cost_mean_usd": -0.0052,
+      "delta_cost_variance": 0.005375
+    },
+    "csb_org_crossrepo_tracing": {
+      "n": 22,
+      "baseline_cost_mean_usd": 0.2478,
+      "mcp_cost_mean_usd": 0.2187,
+      "delta_cost_mean_usd": -0.0292,
+      "delta_cost_variance": 0.003282
+    },
+    "csb_org_domain": {
+      "n": 20,
+      "baseline_cost_mean_usd": 0.2108,
+      "mcp_cost_mean_usd": 0.2258,
+      "delta_cost_mean_usd": 0.015,
+      "delta_cost_variance": 0.003268
+    },
+    "csb_org_incident": {
+      "n": 20,
+      "baseline_cost_mean_usd": 0.2465,
+      "mcp_cost_mean_usd": 0.1989,
+      "delta_cost_mean_usd": -0.0476,
+      "delta_cost_variance": 0.007914
+    },
+    "csb_org_migration": {
+      "n": 26,
+      "baseline_cost_mean_usd": 0.2534,
+      "mcp_cost_mean_usd": 0.2501,
+      "delta_cost_mean_usd": -0.0033,
+      "delta_cost_variance": 0.009546
+    },
+    "csb_org_onboarding": {
+      "n": 28,
+      "baseline_cost_mean_usd": 0.1029,
+      "mcp_cost_mean_usd": 0.1049,
+      "delta_cost_mean_usd": 0.002,
+      "delta_cost_variance": 0.00086
+    },
+    "csb_org_org": {
+      "n": 15,
+      "baseline_cost_mean_usd": 0.2362,
+      "mcp_cost_mean_usd": 0.2193,
+      "delta_cost_mean_usd": -0.0169,
+      "delta_cost_variance": 0.00171
+    },
+    "csb_org_platform": {
+      "n": 18,
+      "baseline_cost_mean_usd": 0.194,
+      "mcp_cost_mean_usd": 0.2149,
+      "delta_cost_mean_usd": 0.0209,
+      "delta_cost_variance": 0.001999
+    },
+    "csb_org_security": {
+      "n": 24,
+      "baseline_cost_mean_usd": 0.2167,
+      "mcp_cost_mean_usd": 0.2146,
+      "delta_cost_mean_usd": -0.002,
+      "delta_cost_variance": 0.003105
+    },
+    "csb_sdlc_debug": {
+      "n": 18,
+      "baseline_cost_mean_usd": 0.3669,
+      "mcp_cost_mean_usd": 0.4569,
+      "delta_cost_mean_usd": 0.0901,
+      "delta_cost_variance": 0.02381
+    },
+    "csb_sdlc_design": {
+      "n": 14,
+      "baseline_cost_mean_usd": 0.41,
+      "mcp_cost_mean_usd": 0.359,
+      "delta_cost_mean_usd": -0.051,
+      "delta_cost_variance": 0.097988
+    },
+    "csb_sdlc_document": {
+      "n": 13,
+      "baseline_cost_mean_usd": 0.2669,
+      "mcp_cost_mean_usd": 0.2974,
+      "delta_cost_mean_usd": 0.0305,
+      "delta_cost_variance": 0.01439
+    },
+    "csb_sdlc_feature": {
+      "n": 23,
+      "baseline_cost_mean_usd": 0.4965,
+      "mcp_cost_mean_usd": 0.7079,
+      "delta_cost_mean_usd": 0.2114,
+      "delta_cost_variance": 0.183988
+    },
+    "csb_sdlc_fix": {
+      "n": 26,
+      "baseline_cost_mean_usd": 0.5997,
+      "mcp_cost_mean_usd": 0.7057,
+      "delta_cost_mean_usd": 0.1059,
+      "delta_cost_variance": 0.06587
+    },
+    "csb_sdlc_refactor": {
+      "n": 15,
+      "baseline_cost_mean_usd": 0.3194,
+      "mcp_cost_mean_usd": 0.7173,
+      "delta_cost_mean_usd": 0.398,
+      "delta_cost_variance": 0.147469
+    },
+    "csb_sdlc_secure": {
+      "n": 12,
+      "baseline_cost_mean_usd": 0.4825,
+      "mcp_cost_mean_usd": 0.5657,
+      "delta_cost_mean_usd": 0.0832,
+      "delta_cost_variance": 0.030859
+    },
+    "csb_sdlc_test": {
+      "n": 18,
+      "baseline_cost_mean_usd": 0.2641,
+      "mcp_cost_mean_usd": 0.2976,
+      "delta_cost_mean_usd": 0.0335,
+      "delta_cost_variance": 0.015625
+    },
+    "csb_sdlc_understand": {
+      "n": 10,
+      "baseline_cost_mean_usd": 0.3519,
+      "mcp_cost_mean_usd": 0.4475,
+      "delta_cost_mean_usd": 0.0956,
+      "delta_cost_variance": 0.022037
+    }
   }
 }