refactor(agentic): consume producer KV capacity (#547)

Author: cquil11

packages/app/src/components/inference/agentic-point/server-metric-cards.tsx

@@ -416,7 +416,7 @@ export function InflightUniqueTokensCard({
 }: {
   phaseTimeline: RequestTimeline | null;
   timelineLoading: boolean;
-  /** KV-cache pool size in tokens (vLLM only) — drawn as a constant ceiling. */
+  /** KV-cache pool size in tokens (vLLM/SGLang) — drawn as a constant ceiling. */
   kvCachePoolTokens: number | null;
 }) {
   return (
@@ -437,7 +437,7 @@ export function InflightUniqueTokensCard({
         // so brief turn-handoff dips don't dominate the chart.
         const raw = inflightUniqueTokens(phaseTimeline.requests);
         const smoothed = timeRollingAverage(raw, 30);
-        // KV-cache pool size (vLLM only) drawn as a constant ceiling so
+        // KV-cache pool size (vLLM/SGLang) drawn as a constant ceiling so
         // you can see how close the working set gets to eviction
         // pressure. Phase-independent — it's a static config value.
         const pool = kvCachePoolTokens;

packages/app/src/hooks/api/use-trace-server-metrics.ts

@@ -78,8 +78,8 @@ export interface TraceServerMetrics {
    */
   kvCacheUsageByEngine: { engineLabel: string; points: TimeSeriesPoint[] }[];
   /**
-   * Total KV-cache pool size in tokens (num_gpu_blocks × block_size, summed
-   * across engines). vLLM only — null for SGLang/TRT or older rows.
+   * Deployment-wide KV-cache pool size in tokens emitted by InferenceX.
+   * Available for vLLM/SGLang agentic rows; null for unsupported or older rows.
    */
   kvCachePoolTokens: number | null;
   /** Orchestrator-normalized metrics grouped by endpoint/worker. */

packages/constants/src/metric-keys.ts

@@ -112,6 +112,8 @@ export const METRIC_KEYS = new Set([
   // profiling window (agentic aiperf; flat in v2 artifacts, mapped from
   // server_metrics.kv_cache.gpu_usage_pct in v3)
   'gpu_kv_cache_usage_pct',
+  // Deployment-wide KV-cache capacity emitted by InferenceX agentic processing.
+  'kv_cache_pool_tokens',
   // measured power / energy (emitted by runner's aggregate_power.py)
   // avg_power_w:             mean per-GPU draw (W) during the load window
   // joules_per_output_token: energy / total_output_tokens. CLUSTER-WIDE on

packages/db/src/etl/benchmark-ingest.ts

@@ -4,7 +4,6 @@
 
 import type postgres from 'postgres';
 import type { BenchmarkParams } from './benchmark-mapper';
-import { kvCachePoolTokensFromServerLog } from './server-log-metrics';
 
 type Sql = ReturnType<typeof postgres>;
 
@@ -74,12 +73,16 @@ export async function bulkIngestBenchmarkRows(
       unnest(${sql.array(workersJsons)}::jsonb[])
     on conflict (workflow_run_id, config_id, benchmark_type, isl, osl, conc, offload_mode)
     do update set
-      -- Replace metrics with the fresh artifact values, but carry over
-      -- kv_cache_pool_tokens: it is derived from the server log at
-      -- insertServerLog time (not present in any artifact JSON), so a later
-      -- upsert from the aggregated results_bmk artifact would silently wipe it.
+      -- Producer artifacts now carry kv_cache_pool_tokens. Preserve an existing
+      -- backfill only when replaying a legacy artifact that predates the field.
       metrics = excluded.metrics || jsonb_strip_nulls(
-        jsonb_build_object('kv_cache_pool_tokens', benchmark_results.metrics->'kv_cache_pool_tokens')
+        jsonb_build_object(
+          'kv_cache_pool_tokens',
+          coalesce(
+            excluded.metrics->'kv_cache_pool_tokens',
+            benchmark_results.metrics->'kv_cache_pool_tokens'
+          )
+        )
       ),
       image = excluded.image,
       workers = excluded.workers
@@ -113,18 +116,9 @@ export async function insertServerLog(
     insert into server_logs (server_log) values (${serverLog})
     returning id
   `;
-  // Derive the KV-cache pool size (tokens) from the log's authoritative
-  // "GPU KV cache size: N tokens" line(s) and stash it on the result's metrics
-  // JSON, mirroring how trace-replay-ingest derives cache-hit rates. The
-  // scraped vllm:cache_config_info metric can't reconstruct this for MLA models.
-  const kvCachePoolTokens = kvCachePoolTokensFromServerLog(serverLog);
   await sql`
     update benchmark_results
-    set server_log_id = ${logId}${
-      kvCachePoolTokens === null
-        ? sql``
-        : sql`, metrics = jsonb_set(metrics, '{kv_cache_pool_tokens}', to_jsonb(${kvCachePoolTokens}::bigint))`
-    }
+    set server_log_id = ${logId}
     where id = any(${sql.array(unlinked.map((r) => r.id))}::bigint[])
   `;
 }

packages/db/src/etl/benchmark-mapper.test.ts

@@ -655,6 +655,7 @@ function makeV3AgenticRow(overrides: Record<string, any> = {}): Record<string, a
     kv_offload_backend: '',
     num_requests_total: 1648,
     num_requests_successful: 1648,
+    kv_cache_pool_tokens: 11_500_000,
     dataset: {
       source_type: 'public_dataset',
       hf_dataset_name: 'semianalysisai/cc-traces-weka-062126',
@@ -817,6 +818,7 @@ describe('mapBenchmarkRow — v3 agentic nested agg schema', () => {
     expect(m.server_gpu_cache_hit_rate).toBeCloseTo(0.78539, 6);
     expect(m.server_external_cache_hit_rate).toBe(0);
     expect(m.gpu_kv_cache_usage_pct).toBeCloseTo(0.82134, 6);
+    expect(m.kv_cache_pool_tokens).toBe(11_500_000);
     expect(m.total_prompt_tokens).toBe(261750519);
     expect(m.total_generation_tokens).toBe(1422696);
     expect(m.total_requests_completed).toBe(1648);

packages/db/src/etl/server-log-metrics.test.ts

@@ -83,8 +83,8 @@ export interface TraceServerMetrics {
    */
   kvCacheUsageByEngine: { engineLabel: string; points: TimeSeriesPoint[] }[];
   /**
-   * Total KV-cache pool size in tokens (num_gpu_blocks × block_size, summed
-   * across engines). vLLM only — null for SGLang/TRT or older rows.
+   * Deployment-wide KV-cache pool size in tokens emitted by InferenceX.
+   * Available for vLLM/SGLang agentic rows; null for unsupported or older rows.
    */
   kvCachePoolTokens: number | null;
   /** Orchestrator-normalized metrics grouped by endpoint/worker. */
@@ -95,7 +95,7 @@ interface RawMetaRow extends PointMeta {
   trace_replay_id: number | null;
   has_blob: boolean;
   chart_series: ChartSeries | null;
-  /** Derived at server-log ingest from "GPU KV cache size: N tokens" lines. */
+  /** Producer-emitted deployment-wide KV-cache capacity. */
   kv_cache_pool_tokens: string | null;
 }