Skip to content

Commit ad16186

Browse files
jkyberneeesjkyberneees
committed
test: cache statistics — 15 unit/E2E tests
Add comprehensive test coverage for the cache statistics feature: llm/client_test.go (2 new): - TestParseResponse_NoCacheMetrics — all zeros when no cache data - TestParseResponse_AnthropicAndOpenAICache — both formats coexist loop/loop_test.go (4 new): - TestEngine_Run_CacheAccumulation — Anthropic cache from single answer - TestEngine_Run_CacheAccumulation_MultiIter — cumulative across tool calls - TestEngine_Run_CacheAccumulation_OpenAI — OpenAI prompt_tokens_details - TestEngine_Run_CacheAccumulation_NoCache — zero when no metrics render/render_test.go (8 new): - Summary_TokensOnly/AnthropicCache/OpenAICache — three output variants - Summary_AllZero — suppressed output - Summary_NilRenderer/NilWriter — nil safety - Summary_NoColor — no ANSI in plain mode serve_test.go (1 new + 1 fix): - TestServe_DoneCacheStats — E2E wire format verification - Fixed testServer's done event to include cache/token fields
1 parent 8e95a7b commit ad16186

4 files changed

Lines changed: 388 additions & 2 deletions

File tree

‎cmd/odek/serve_test.go‎

Lines changed: 80 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,8 +162,15 @@ func (s *testServer) handleWebSocket(conn *golangws.Conn) {
162162

163163
// Send done
164164
writeJSON(conn, map[string]any{
165-
"type": "done",
166-
"latency": 0.5,
165+
"type": "done",
166+
"latency": 0.5,
167+
"contextTokens": 150,
168+
"outputTokens": 30,
169+
"cacheCreationTokens": 50,
170+
"cacheReadTokens": 100,
171+
"cachedTokens": 0,
172+
"sessionContextTokens": 150,
173+
"sessionOutputTokens": 30,
167174
})
168175
}
169176
}
@@ -1384,3 +1391,74 @@ doneCheck:
13841391

13851392
t.Log("✅ Live tool events verified: tool_call + tool_result arrive before done")
13861393
}
1394+
1395+
func TestServe_DoneCacheStats(t *testing.T) {
1396+
// Verify that the done event carries cache statistics fields.
1397+
s := startTestServer(t)
1398+
defer s.Close()
1399+
1400+
conn, err := golangws.Dial(s.wsURL+"/ws", "", "http://localhost")
1401+
if err != nil {
1402+
t.Fatalf("Dial(): %v", err)
1403+
}
1404+
defer conn.Close()
1405+
1406+
// Send a prompt
1407+
prompt := map[string]string{"type": "prompt", "content": "cache test"}
1408+
payload, _ := json.Marshal(prompt)
1409+
if err := golangws.Message.Send(conn, string(payload)); err != nil {
1410+
t.Fatalf("Send(): %v", err)
1411+
}
1412+
1413+
// Collect events until done
1414+
var doneEvent map[string]any
1415+
timeout := time.After(5 * time.Second)
1416+
1417+
for doneEvent == nil {
1418+
select {
1419+
case <-timeout:
1420+
t.Fatal("timeout waiting for done event")
1421+
default:
1422+
var event map[string]any
1423+
if err := readJSON(conn, &event); err != nil {
1424+
t.Fatalf("Receive(): %v", err)
1425+
}
1426+
if event["type"] == "done" {
1427+
doneEvent = event
1428+
}
1429+
}
1430+
}
1431+
1432+
t.Logf("done event: %v", doneEvent)
1433+
1434+
// Verify cache fields exist (even if zero)
1435+
if _, ok := doneEvent["cacheCreationTokens"]; !ok {
1436+
t.Error("done event missing cacheCreationTokens field")
1437+
}
1438+
if _, ok := doneEvent["cacheReadTokens"]; !ok {
1439+
t.Error("done event missing cacheReadTokens field")
1440+
}
1441+
if _, ok := doneEvent["cachedTokens"]; !ok {
1442+
t.Error("done event missing cachedTokens field")
1443+
}
1444+
1445+
// Verify the fields are numeric
1446+
switch v := doneEvent["cacheCreationTokens"].(type) {
1447+
case float64:
1448+
// ok
1449+
default:
1450+
t.Errorf("cacheCreationTokens type = %T, want float64", v)
1451+
}
1452+
switch v := doneEvent["cacheReadTokens"].(type) {
1453+
case float64:
1454+
// ok
1455+
default:
1456+
t.Errorf("cacheReadTokens type = %T, want float64", v)
1457+
}
1458+
switch v := doneEvent["cachedTokens"].(type) {
1459+
case float64:
1460+
// ok
1461+
default:
1462+
t.Errorf("cachedTokens type = %T, want float64", v)
1463+
}
1464+
}

‎internal/llm/client_test.go‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -949,3 +949,65 @@ func TestClient_NewWithMaxTokens(t *testing.T) {
949949
t.Errorf("BaseURL = %q", c.BaseURL)
950950
}
951951
}
952+
953+
func TestParseResponse_NoCacheMetrics(t *testing.T) {
954+
raw := `{
955+
"choices": [{
956+
"message": {
957+
"content": "No cache"
958+
}
959+
}],
960+
"usage": {
961+
"prompt_tokens": 100,
962+
"completion_tokens": 20
963+
}
964+
}`
965+
966+
result, err := parseResponse([]byte(raw))
967+
if err != nil {
968+
t.Fatal(err)
969+
}
970+
if result.CacheCreationTokens != 0 {
971+
t.Errorf("CacheCreationTokens = %d, want 0", result.CacheCreationTokens)
972+
}
973+
if result.CacheReadTokens != 0 {
974+
t.Errorf("CacheReadTokens = %d, want 0", result.CacheReadTokens)
975+
}
976+
if result.CachedTokens != 0 {
977+
t.Errorf("CachedTokens = %d, want 0", result.CachedTokens)
978+
}
979+
}
980+
981+
func TestParseResponse_AnthropicAndOpenAICache(t *testing.T) {
982+
// Both Anthropic and OpenAI cache fields present — Anthropic takes precedence.
983+
raw := `{
984+
"choices": [{
985+
"message": {
986+
"content": "Both"
987+
}
988+
}],
989+
"usage": {
990+
"prompt_tokens": 300,
991+
"completion_tokens": 60,
992+
"cache_creation_input_tokens": 70,
993+
"cache_read_input_tokens": 140,
994+
"prompt_tokens_details": {
995+
"cached_tokens": 999
996+
}
997+
}
998+
}`
999+
1000+
result, err := parseResponse([]byte(raw))
1001+
if err != nil {
1002+
t.Fatal(err)
1003+
}
1004+
if result.CacheCreationTokens != 70 {
1005+
t.Errorf("CacheCreationTokens = %d, want 70", result.CacheCreationTokens)
1006+
}
1007+
if result.CacheReadTokens != 140 {
1008+
t.Errorf("CacheReadTokens = %d, want 140", result.CacheReadTokens)
1009+
}
1010+
if result.CachedTokens != 999 {
1011+
t.Errorf("CachedTokens = %d, want 999", result.CachedTokens)
1012+
}
1013+
}

‎internal/loop/loop_test.go‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -804,3 +804,134 @@ func TestEngine_ToolEventHandler(t *testing.T) {
804804
t.Errorf("event[1] name = %q, want 'echo'", eventData[1])
805805
}
806806
}
807+
808+
func TestEngine_Run_CacheAccumulation(t *testing.T) {
809+
// Server that returns cache metrics in usage, then final answer.
810+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
811+
fmt.Fprint(w, `{"choices":[{"message":{"content":"done"}}],"usage":{"prompt_tokens":100,"completion_tokens":20,"cache_creation_input_tokens":40,"cache_read_input_tokens":30}}`)
812+
}))
813+
defer server.Close()
814+
815+
client := llm.New(server.URL, "sk-test", "test-model", "", 0)
816+
registry := tool.NewRegistry(nil)
817+
engine := New(client, registry, 10, "", nil, 0)
818+
819+
result, err := engine.Run(context.Background(), "test")
820+
if err != nil {
821+
t.Fatalf("Run() error: %v", err)
822+
}
823+
if result != "done" {
824+
t.Errorf("result = %q, want 'done'", result)
825+
}
826+
if engine.TotalCacheCreationTokens != 40 {
827+
t.Errorf("TotalCacheCreationTokens = %d, want 40", engine.TotalCacheCreationTokens)
828+
}
829+
if engine.TotalCacheReadTokens != 30 {
830+
t.Errorf("TotalCacheReadTokens = %d, want 30", engine.TotalCacheReadTokens)
831+
}
832+
if engine.TotalCachedTokens != 0 {
833+
t.Errorf("TotalCachedTokens = %d, want 0", engine.TotalCachedTokens)
834+
}
835+
if engine.TotalInputTokens != 100 {
836+
t.Errorf("TotalInputTokens = %d, want 100", engine.TotalInputTokens)
837+
}
838+
if engine.TotalOutputTokens != 20 {
839+
t.Errorf("TotalOutputTokens = %d, want 20", engine.TotalOutputTokens)
840+
}
841+
}
842+
843+
func TestEngine_Run_CacheAccumulation_MultiIter(t *testing.T) {
844+
// First call returns tool call + cache, second call returns answer + cache.
845+
callCount := 0
846+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
847+
callCount++
848+
if callCount == 1 {
849+
fmt.Fprint(w, `{"choices":[{"message":{"content":"thinking","tool_calls":[{"id":"c1","function":{"name":"echo","arguments":"{}"}}]}}],"usage":{"prompt_tokens":50,"completion_tokens":10,"cache_creation_input_tokens":20,"cache_read_input_tokens":15}}`)
850+
} else {
851+
fmt.Fprint(w, `{"choices":[{"message":{"content":"final"}}],"usage":{"prompt_tokens":30,"completion_tokens":5,"cache_creation_input_tokens":10,"cache_read_input_tokens":8}}`)
852+
}
853+
}))
854+
defer server.Close()
855+
856+
echoTool := &fakeTool{name: "echo", description: "echoes", output: "ok"}
857+
registry := tool.NewRegistry([]tool.Tool{echoTool})
858+
client := llm.New(server.URL, "sk-test", "test-model", "", 0)
859+
engine := New(client, registry, 10, "", nil, 0)
860+
861+
result, err := engine.Run(context.Background(), "echo")
862+
if err != nil {
863+
t.Fatalf("Run() error: %v", err)
864+
}
865+
if result != "final" {
866+
t.Errorf("result = %q, want 'final'", result)
867+
}
868+
// Cumulative: iter1 (20+15) + iter2 (10+8) = 30+23
869+
if engine.TotalCacheCreationTokens != 30 {
870+
t.Errorf("TotalCacheCreationTokens = %d, want 30", engine.TotalCacheCreationTokens)
871+
}
872+
if engine.TotalCacheReadTokens != 23 {
873+
t.Errorf("TotalCacheReadTokens = %d, want 23", engine.TotalCacheReadTokens)
874+
}
875+
// Cumulative: iter1 (50+30) + iter2 (30+5) = 80+15
876+
if engine.TotalInputTokens != 80 {
877+
t.Errorf("TotalInputTokens = %d, want 80", engine.TotalInputTokens)
878+
}
879+
if engine.TotalOutputTokens != 15 {
880+
t.Errorf("TotalOutputTokens = %d, want 15", engine.TotalOutputTokens)
881+
}
882+
if callCount != 2 {
883+
t.Errorf("expected 2 LLM calls, got %d", callCount)
884+
}
885+
}
886+
887+
func TestEngine_Run_CacheAccumulation_OpenAI(t *testing.T) {
888+
// OpenAI format: cached_tokens via prompt_tokens_details
889+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
890+
fmt.Fprint(w, `{"choices":[{"message":{"content":"cached"}}],"usage":{"prompt_tokens":200,"completion_tokens":40,"prompt_tokens_details":{"cached_tokens":150}}}`)
891+
}))
892+
defer server.Close()
893+
894+
client := llm.New(server.URL, "sk-test", "test-model", "", 0)
895+
registry := tool.NewRegistry(nil)
896+
engine := New(client, registry, 10, "", nil, 0)
897+
898+
_, err := engine.Run(context.Background(), "test")
899+
if err != nil {
900+
t.Fatalf("Run() error: %v", err)
901+
}
902+
if engine.TotalCachedTokens != 150 {
903+
t.Errorf("TotalCachedTokens = %d, want 150", engine.TotalCachedTokens)
904+
}
905+
if engine.TotalCacheCreationTokens != 0 {
906+
t.Errorf("TotalCacheCreationTokens = %d, want 0", engine.TotalCacheCreationTokens)
907+
}
908+
if engine.TotalCacheReadTokens != 0 {
909+
t.Errorf("TotalCacheReadTokens = %d, want 0", engine.TotalCacheReadTokens)
910+
}
911+
}
912+
913+
func TestEngine_Run_CacheAccumulation_NoCache(t *testing.T) {
914+
// Cache accumulators should be zero when no cache metrics returned.
915+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
916+
fmt.Fprint(w, `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":10,"completion_tokens":5}}`)
917+
}))
918+
defer server.Close()
919+
920+
client := llm.New(server.URL, "sk-test", "test-model", "", 0)
921+
registry := tool.NewRegistry(nil)
922+
engine := New(client, registry, 10, "", nil, 0)
923+
924+
_, err := engine.Run(context.Background(), "test")
925+
if err != nil {
926+
t.Fatalf("Run() error: %v", err)
927+
}
928+
if engine.TotalCacheCreationTokens != 0 {
929+
t.Errorf("TotalCacheCreationTokens = %d, want 0", engine.TotalCacheCreationTokens)
930+
}
931+
if engine.TotalCacheReadTokens != 0 {
932+
t.Errorf("TotalCacheReadTokens = %d, want 0", engine.TotalCacheReadTokens)
933+
}
934+
if engine.TotalCachedTokens != 0 {
935+
t.Errorf("TotalCachedTokens = %d, want 0", engine.TotalCachedTokens)
936+
}
937+
}

0 commit comments

Comments
 (0)