Vercel no longer seems to include cached tokens in input tokens for /generation endpoint (#3690)

* Vercel no longer seems to include cached tokens in input tokens for /generation endpoint

See here: https://us.posthog.com/project/141915/insights/oskDcDiF

The cache hit tokens number is often larger than the input tokens number

* Typecheck fixes

Author: chrarnoldus

apps/web/src/lib/ai-gateway/processUsage.test.ts

@@ -248,13 +248,7 @@ describe('mapToUsageStats approval tests', () => {
   test(claudeSonnetGeneration, async () => {
     const inputFile = join(sampleDir, claudeSonnetGeneration);
     const generationData = JSON.parse(await readFile(inputFile, 'utf-8')) as OpenRouterGeneration;
-    const result = mapToUsageStats(
-      generationData,
-      'nonsense',
-      'fake-user-id',
-      'fake-model',
-      'openrouter'
-    );
+    const result = mapToUsageStats(generationData, 'nonsense', 'fake-user-id', 'openrouter');
     const resultString = JSON.stringify(result, null, 2);
     const approvalFilePath = inputFile + '.mapToUsageStats.approved.json';
     await verifyApproval(resultString, approvalFilePath);
@@ -292,13 +286,7 @@ describe('mapToUsageStats', () => {
     };
 
     // Call mapToUsageStats with the BYOK generation
-    const result = mapToUsageStats(
-      byokGeneration,
-      'test response',
-      'fake-user-id',
-      'fake-model',
-      'openrouter'
-    );
+    const result = mapToUsageStats(byokGeneration, 'test response', 'fake-user-id', 'openrouter');
 
     // Verify that the cost is multiplied by OPENROUTER_BYOK_COST_MULTIPLIER
     expect(result.cost_mUsd).toBe(toMicrodollars(0.1 * 20.0)); // 0.1 * 20 = 2, then convert to microdollars
@@ -326,7 +314,6 @@ describe('mapToUsageStats', () => {
       nonByokGeneration,
       'test response',
       ' fake-user-id',
-      'fake-model',
       'openrouter'
     );
 

apps/web/src/lib/ai-gateway/processUsage.ts

@@ -61,8 +61,6 @@ import {
   drainSseStream,
   extractVercelIsByok,
 } from '@/lib/ai-gateway/processUsage.shared';
-import { isClaudeModel } from '@/lib/ai-gateway/providers/anthropic.constants';
-import { isMinimaxModel } from '@/lib/ai-gateway/providers/minimax';
 import type { KiloExclusiveModel } from '@/lib/ai-gateway/providers/kilo-exclusive-model';
 
 const posthogClient = PostHogClient();
@@ -987,10 +985,19 @@ export async function processTokenData(
       generation,
       usageStats.responseContent,
       usageContext.kiloUserId,
-      usageContext.requested_model,
       usageContext.provider
     );
 
+    if (usageContext.provider === 'vercel' && usageStats.inputTokens > 0) {
+      // It seems Vercel's /generation result does not include cache hit tokens in input tokens, unlike OpenRouter.
+      // Since it's not completely clear this is the case and in the past the numbers were inconsistent
+      // we keep the response usage data if we have it.
+      genStats.inputTokens = usageStats.inputTokens;
+      genStats.outputTokens = usageStats.outputTokens;
+      genStats.cacheHitTokens = usageStats.cacheHitTokens;
+      genStats.cacheWriteTokens = usageStats.cacheWriteTokens;
+    }
+
     genStats.model = usageStats.model; // openrouter bug?
     genStats.hasError = usageStats.hasError; // retain by choice
     genStats.status_code = usageStats.status_code; // retain by choice
@@ -1003,11 +1010,6 @@ export async function processTokenData(
         [genStats.cacheDiscount_mUsd, usageStats.cacheDiscount_mUsd]
       );
     }
-    if (genStats.inputTokens < usageStats.inputTokens) {
-      console.warn(
-        'Suspicious: fewer input tokens in generation data compared to usage stats. Did provider return Anthropic-style token counts?'
-      );
-    }
     usageStats = genStats;
   }
 
@@ -1043,10 +1045,6 @@ export async function processTokenData(
   return logMicrodollarUsage(usageStats, usageContext);
 }
 
-function useAnthropicStyleTokenCounting(requestedModel: string, provider: ProviderId) {
-  return provider === 'vercel' && (isClaudeModel(requestedModel) || isMinimaxModel(requestedModel));
-}
-
 async function useGenerationLookup(
   usageStats: MicrodollarUsageStats | null,
   usageContext: MicrodollarUsageContext
@@ -1071,7 +1069,6 @@ export const mapToUsageStats = (
   { data }: OpenRouterGeneration,
   responseContent: string,
   kiloUserId: string,
-  requestedModel: string,
   provider: ProviderId
 ): MicrodollarUsageStats => {
   let llmCostUsd;
@@ -1094,16 +1091,18 @@ export const mapToUsageStats = (
     hasError: false,
     model: data.model,
     responseContent,
-    inputTokens: useAnthropicStyleTokenCounting(requestedModel, provider)
-      ? (data.native_tokens_prompt ?? 0) +
-        (data.native_tokens_cached ?? 0) +
-        (data.native_tokens_cache_creation ?? 0)
-      : (data.native_tokens_prompt ?? 0),
+    inputTokens:
+      provider === 'vercel'
+        ? (data.native_tokens_prompt ?? 0) +
+          (data.native_tokens_cached ?? 0) +
+          (data.native_tokens_cache_creation ?? 0)
+        : (data.native_tokens_prompt ?? 0),
     cacheHitTokens: data.native_tokens_cached ?? 0,
     cacheWriteTokens: data.native_tokens_cache_creation ?? 0,
-    outputTokens: useAnthropicStyleTokenCounting(requestedModel, provider)
-      ? (data.native_tokens_completion ?? 0) + (data.native_tokens_reasoning ?? 0)
-      : (data.native_tokens_completion ?? 0),
+    outputTokens:
+      provider === 'vercel'
+        ? (data.native_tokens_completion ?? 0) + (data.native_tokens_reasoning ?? 0)
+        : (data.native_tokens_completion ?? 0),
     cost_mUsd: toMicrodollars(llmCostUsd),
     is_byok: data.is_byok ?? null,
     cacheDiscount_mUsd: