Skip to content

Commit 01af32e

Browse files
authored
feat(agentic): emit KV cache pool tokens (#2142)
1 parent 2993627 commit 01af32e

2 files changed

Lines changed: 27 additions & 0 deletions

File tree

utils/agentic/aggregation/process_agentic_result.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,7 @@ def build_agg(
175175

176176
agg["request_metrics"] = request_nested
177177
agg["server_metrics"] = server_nested
178+
agg["kv_cache_pool_tokens"] = server_nested["kv_cache"]["gpu_total_tokens"]
178179
if warnings:
179180
agg["warnings"] = warnings
180181
return agg

utils/agentic/aggregation/test_process_agentic_result.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
"request_accounting",
5858
"request_metrics",
5959
"server_metrics",
60+
"kv_cache_pool_tokens",
6061
}
6162
REQUEST_ACCOUNTING_KEYS = {
6263
"records_total",
@@ -604,6 +605,7 @@ def test_processor_handles_missing_server_metrics(tmp_path: Path):
604605
server_metrics = agg["server_metrics"]
605606
assert server_metrics["cache"]["gpu_cache_hit_rate"] is None
606607
assert server_metrics["kv_cache"]["gpu_total_tokens"] is None
608+
assert agg["kv_cache_pool_tokens"] is None
607609
assert agg["request_metrics"]["cache"]["theoretical_cache_hit_rate"] is None
608610
# Non-server-derived totals fall back to per-record sums.
609611
assert server_metrics["tokens"]["prompt_total"] == 100 + 180 + 120 + 200 + 240
@@ -628,6 +630,30 @@ def test_processor_reads_gpu_kv_cache_capacity_from_server_log(tmp_path: Path):
628630
agg = _run_processor(result_dir, tmp_path / "out")
629631

630632
assert agg["server_metrics"]["kv_cache"]["gpu_total_tokens"] == 11_500_000
633+
assert agg["kv_cache_pool_tokens"] == 11_500_000
634+
_assert_stable_server_metrics_schema(agg)
635+
636+
637+
def test_processor_emits_sglang_kv_pool_from_server_log(tmp_path: Path):
638+
result_dir = _write_fixture(tmp_path)
639+
(result_dir / "server.log").write_text(
640+
"\n".join(
641+
[
642+
"[2026-07-08 16:43:35] server_args=ServerArgs(tp_size=4, dp_size=4)",
643+
"[2026-07-08 16:49:59 DP0 TP0 EP0] "
644+
"max_total_num_tokens=4602880, chunked_prefill_size=4096",
645+
]
646+
)
647+
)
648+
649+
agg = _run_processor(
650+
result_dir,
651+
tmp_path / "out",
652+
env_overrides={"FRAMEWORK": "sglang"},
653+
)
654+
655+
assert agg["server_metrics"]["kv_cache"]["gpu_total_tokens"] == 18_411_520
656+
assert agg["kv_cache_pool_tokens"] == 18_411_520
631657
_assert_stable_server_metrics_schema(agg)
632658

633659

0 commit comments

Comments
 (0)