Allow document/pdf upload for Claude, GPT and Grok 4 models (#2848)

chrarnoldus · web-flow · commit 0e7fd0e7856b · 2026-04-28T14:06:25.000+02:00
* Allow document/pdf upload for Claude, GPT and Grok 4 models

* Move
diff --git a/apps/web/src/lib/ai-gateway/kilo-auto/index.ts b/apps/web/src/lib/ai-gateway/kilo-auto/index.ts
@@ -19,6 +19,7 @@ type AutoModel = {
   input_cache_read_price: string | undefined;
   input_cache_write_price: string | undefined;
   supports_images: boolean;
+  supports_pdf: boolean;
   opencode_settings: OpenCodeSettings | undefined;
 };
 
@@ -109,6 +110,7 @@ export const KILO_AUTO_FRONTIER_MODEL: AutoModel = {
   input_cache_read_price: '0.0000005',
   input_cache_write_price: '0.00000625',
   supports_images: true,
+  supports_pdf: true,
   opencode_settings: {
     family: 'claude',
     prompt: 'anthropic',
@@ -127,6 +129,7 @@ export const KILO_AUTO_FREE_MODEL: AutoModel = {
   input_cache_read_price: '0',
   input_cache_write_price: '0',
   supports_images: false,
+  supports_pdf: false,
   opencode_settings: undefined,
 };
 
@@ -141,6 +144,7 @@ export const KILO_AUTO_BALANCED_MODEL: AutoModel = {
   input_cache_read_price: '0.0000000325',
   input_cache_write_price: '0.00000040625',
   supports_images: true,
+  supports_pdf: false,
   opencode_settings: {
     ai_sdk_provider: 'openai-compatible',
   },
@@ -157,6 +161,7 @@ export const KILO_AUTO_SMALL_MODEL: AutoModel = {
   input_cache_read_price: '0.000000005',
   input_cache_write_price: undefined,
   supports_images: true,
+  supports_pdf: false,
   opencode_settings: undefined,
 };
 
diff --git a/apps/web/src/lib/ai-gateway/models.ts b/apps/web/src/lib/ai-gateway/models.ts
@@ -25,7 +25,12 @@ import { morph_warp_grep_free_model } from '@/lib/ai-gateway/providers/morph';
 import { gemma_4_26b_a4b_it_free_model } from '@/lib/ai-gateway/providers/google';
 import { qwen36_plus_model } from '@/lib/ai-gateway/providers/qwen';
 import { stepfun_35_flash_free_model } from '@/lib/ai-gateway/providers/stepfun';
-import { grok_code_fast_1_optimized_free_model } from '@/lib/ai-gateway/providers/xai';
+import {
+  grok_code_fast_1_optimized_free_model,
+  isGrok4Model,
+} from '@/lib/ai-gateway/providers/xai';
+import { isAnthropicModel } from '@/lib/ai-gateway/providers/anthropic.constants';
+import { isOpenAiModel } from '@/lib/ai-gateway/providers/openai';
 
 export const PRIMARY_DEFAULT_MODEL = CLAUDE_SONNET_CURRENT_MODEL_ID;
 
@@ -63,6 +68,10 @@ export function isFreeModel(model: string): boolean {
   );
 }
 
+export function isPdfSupportingModel(model: string): boolean {
+  return isAnthropicModel(model) || isOpenAiModel(model) || isGrok4Model(model);
+}
+
 export function isKiloExclusiveFreeModel(model: string): boolean {
   return kiloExclusiveModels.some(
     m => m.public_id === model && m.status !== 'disabled' && !m.pricing
diff --git a/apps/web/src/lib/ai-gateway/processUsage.ts b/apps/web/src/lib/ai-gateway/processUsage.ts
@@ -54,7 +54,7 @@ import {
 } from '@/lib/ai-gateway/processUsage.messages';
 import { OPENROUTER_BYOK_COST_MULTIPLIER } from '@/lib/ai-gateway/processUsage.constants';
 import { computeOpenRouterCostFields, drainSseStream } from '@/lib/ai-gateway/processUsage.shared';
-import { isAnthropicModel } from '@/lib/ai-gateway/providers/anthropic';
+import { isAnthropicModel } from '@/lib/ai-gateway/providers/anthropic.constants';
 import { isMinimaxModel } from '@/lib/ai-gateway/providers/minimax';
 import type { KiloExclusiveModel } from '@/lib/ai-gateway/providers/kilo-exclusive-model';
 
diff --git a/apps/web/src/lib/ai-gateway/providers/anthropic.constants.ts b/apps/web/src/lib/ai-gateway/providers/anthropic.constants.ts
@@ -1,4 +1,5 @@
 import type { KiloExclusiveModel } from '@/lib/ai-gateway/providers/kilo-exclusive-model';
+import { modelStartsWith } from '@/lib/ai-gateway/providers/model-prefix';
 
 export const CLAUDE_SONNET_CURRENT_MODEL_ID = 'anthropic/claude-sonnet-4.6';
 
@@ -22,3 +23,11 @@ export const claude_sonnet_clawsetup_model: KiloExclusiveModel = {
   pricing: null,
   exclusive_to: [],
 };
+
+export function isAnthropicModel(requestedModel: string) {
+  return modelStartsWith(requestedModel, 'anthropic/');
+}
+
+export function isHaikuModel(requestedModel: string) {
+  return modelStartsWith(requestedModel, 'anthropic/claude-haiku');
+}
diff --git a/apps/web/src/lib/ai-gateway/providers/anthropic.ts b/apps/web/src/lib/ai-gateway/providers/anthropic.ts
@@ -1,16 +1,7 @@
-import { modelStartsWith } from '@/lib/ai-gateway/providers/model-prefix';
 import { addCacheBreakpoints } from '@/lib/ai-gateway/providers/openrouter/request-helpers';
 import type { GatewayRequest } from '@/lib/ai-gateway/providers/openrouter/types';
 import { normalizeToolCallIds } from '@/lib/ai-gateway/tool-calling';
 
-export function isAnthropicModel(requestedModel: string) {
-  return modelStartsWith(requestedModel, 'anthropic/');
-}
-
-export function isHaikuModel(requestedModel: string) {
-  return modelStartsWith(requestedModel, 'anthropic/claude-haiku');
-}
-
 function appendAnthropicBetaHeader(extraHeaders: Record<string, string>, betaFlag: string) {
   for (const header of ['anthropic-beta', 'x-anthropic-beta']) {
     extraHeaders[header] = [extraHeaders[header], betaFlag].filter(Boolean).join(',');
diff --git a/apps/web/src/lib/ai-gateway/providers/fixOpenCodeDuplicateReasoning.ts b/apps/web/src/lib/ai-gateway/providers/fixOpenCodeDuplicateReasoning.ts
@@ -1,5 +1,5 @@
 import { ReasoningDetailType } from '@/lib/ai-gateway/custom-llm/reasoning-details';
-import { isAnthropicModel } from '@/lib/ai-gateway/providers/anthropic';
+import { isAnthropicModel } from '@/lib/ai-gateway/providers/anthropic.constants';
 import type {
   MessageWithReasoning,
   OpenRouterChatCompletionRequest,
diff --git a/apps/web/src/lib/ai-gateway/providers/index.ts b/apps/web/src/lib/ai-gateway/providers/index.ts
@@ -12,11 +12,8 @@ import { applyMistralModelSettings, isMistralModel } from '@/lib/ai-gateway/prov
 import { applyXaiModelSettings, isXaiModel } from '@/lib/ai-gateway/providers/xai';
 import { shouldRouteToVercel } from '@/lib/ai-gateway/providers/vercel';
 import { kiloExclusiveModels } from '@/lib/ai-gateway/models';
-import {
-  applyAnthropicModelSettings,
-  isAnthropicModel,
-  isHaikuModel,
-} from '@/lib/ai-gateway/providers/anthropic';
+import { applyAnthropicModelSettings } from '@/lib/ai-gateway/providers/anthropic';
+import { isAnthropicModel, isHaikuModel } from '@/lib/ai-gateway/providers/anthropic.constants';
 import {
   getBYOKforOrganization,
   getBYOKforUser,
diff --git a/apps/web/src/lib/ai-gateway/providers/model-prefix.test.ts b/apps/web/src/lib/ai-gateway/providers/model-prefix.test.ts
@@ -1,5 +1,5 @@
 import { modelStartsWith, stripModelTilde } from './model-prefix';
-import { isAnthropicModel, isHaikuModel } from './anthropic';
+import { isAnthropicModel, isHaikuModel } from './anthropic.constants';
 import { isOpenAiModel, isOpenAiOssModel } from './openai';
 import { isGeminiModel, isGemmaModel, isGemini3Model } from './google';
 import { isMoonshotModel } from './moonshotai';
diff --git a/apps/web/src/lib/ai-gateway/providers/model-settings.ts b/apps/web/src/lib/ai-gateway/providers/model-settings.ts
@@ -1,10 +1,11 @@
+import { isAnthropicModel } from '@/lib/ai-gateway/providers/anthropic.constants';
 import { seed_20_pro_free_model } from '@/lib/ai-gateway/providers/bytedance';
 import { isGemini3Model, isGemmaModel } from '@/lib/ai-gateway/providers/google';
 import { modelStartsWith } from '@/lib/ai-gateway/providers/model-prefix';
 import { isMoonshotModel } from '@/lib/ai-gateway/providers/moonshotai';
 import { isOpenAiModel } from '@/lib/ai-gateway/providers/openai';
 import { qwen36_plus_model } from '@/lib/ai-gateway/providers/qwen';
-import { isXaiModel } from '@/lib/ai-gateway/providers/xai';
+import { isGrok4Model, isXaiModel } from '@/lib/ai-gateway/providers/xai';
 import { isZaiModel } from '@/lib/ai-gateway/providers/zai';
 import type {
   CustomLlmProvider,
@@ -36,7 +37,7 @@ export function getModelVariants(model: string): OpenCodeSettings['variants'] {
       max: { reasoning: { enabled: true, effort: 'xhigh' }, verbosity: 'max' },
     };
   }
-  if (modelStartsWith(model, 'anthropic/')) {
+  if (isAnthropicModel(model)) {
     return {
       none: { reasoning: { enabled: false, effort: 'none' } },
       low: { reasoning: { enabled: true, effort: 'low' }, verbosity: 'low' },
@@ -83,7 +84,7 @@ export function getModelVariants(model: string): OpenCodeSettings['variants'] {
       high: { reasoning: { enabled: true, effort: 'high' } },
     };
   }
-  if (model.startsWith('x-ai/grok-4')) {
+  if (isGrok4Model(model)) {
     return {
       'non-reasoning': { reasoning: { enabled: false, effort: 'none' } },
       reasoning: { reasoning: { enabled: true, effort: 'medium' } },
@@ -101,6 +102,10 @@ function getAiSdkProvider(model: string): CustomLlmProvider | undefined {
     // with 'openai' a bunch of bugs in vercel ai sdk v5 get triggered
     return 'openai-compatible';
   }
+  if (isAnthropicModel(model)) {
+    // on Vercel AI Gateway, this is necessary to support document attachments
+    return 'anthropic';
+  }
   if (isOpenAiModel(model) || isXaiModel(model)) {
     // OpenAI: "While Chat Completions remains supported, Responses is recommended for all new projects.""
     // xAI: "The Responses API is the recommended way to interact with xAI models."
@@ -116,6 +121,10 @@ export function getOpenCodeSettings(model: string): OpenCodeSettings | undefined
 }
 
 export function getOpenClawSettings(model: string): OpenClawModelSettings | undefined {
+  // 2026-04-28: this is aspirational, the OpenClaw Kilo provider does not respect this
+  if (isAnthropicModel(model)) {
+    return { api_adapter: 'anthropic-messages' };
+  }
   if (isOpenAiModel(model) || isXaiModel(model)) {
     return { api_adapter: 'openai-responses' };
   }
diff --git a/apps/web/src/lib/ai-gateway/providers/openrouter/index.ts b/apps/web/src/lib/ai-gateway/providers/openrouter/index.ts
@@ -1,4 +1,9 @@
-import { isFreeModel, kiloExclusiveModels, preferredModels } from '@/lib/ai-gateway/models';
+import {
+  isFreeModel,
+  isPdfSupportingModel,
+  kiloExclusiveModels,
+  preferredModels,
+} from '@/lib/ai-gateway/models';
 import PROVIDERS from '@/lib/ai-gateway/providers/provider-definitions';
 import type { OpenRouterModel } from '@/lib/organizations/organization-types';
 import {
@@ -19,35 +24,50 @@ import { AUTO_MODELS } from '@/lib/ai-gateway/kilo-auto';
 export { normalizeModelId } from '@/lib/ai-gateway/model-utils';
 
 function buildAutoModels(): OpenRouterModel[] {
-  return AUTO_MODELS.map(m => ({
-    id: m.id,
-    name: m.name,
-    created: 0,
-    description: m.description,
-    architecture: {
-      input_modalities: m.supports_images ? ['text', 'image'] : ['text'],
-      output_modalities: ['text'],
-      tokenizer: 'Other',
-    },
-    top_provider: {
-      is_moderated: false,
+  return AUTO_MODELS.map(m => {
+    const input_modalities = ['text'];
+    if (m.supports_images) {
+      input_modalities.push('image');
+    }
+    if (m.supports_pdf) {
+      input_modalities.push('pdf');
+    }
+    return {
+      id: m.id,
+      name: m.name,
+      created: 0,
+      description: m.description,
+      architecture: {
+        input_modalities: input_modalities,
+        output_modalities: ['text'],
+        tokenizer: 'Other',
+      },
+      top_provider: {
+        is_moderated: false,
+        context_length: m.context_length,
+        max_completion_tokens: m.max_completion_tokens,
+      },
+      pricing: {
+        prompt: m.prompt_price,
+        completion: m.completion_price,
+        input_cache_read: m.input_cache_read_price,
+        input_cache_write: m.input_cache_write_price,
+        request: '0',
+        image: '0',
+        web_search: '0',
+        internal_reasoning: '0',
+      },
       context_length: m.context_length,
-      max_completion_tokens: m.max_completion_tokens,
-    },
-    pricing: {
-      prompt: m.prompt_price,
-      completion: m.completion_price,
-      input_cache_read: m.input_cache_read_price,
-      input_cache_write: m.input_cache_write_price,
-      request: '0',
-      image: '0',
-      web_search: '0',
-      internal_reasoning: '0',
-    },
-    context_length: m.context_length,
-    supported_parameters: ['max_tokens', 'temperature', 'tools', 'reasoning', 'include_reasoning'],
-    opencode: m.opencode_settings,
-  }));
+      supported_parameters: [
+        'max_tokens',
+        'temperature',
+        'tools',
+        'reasoning',
+        'include_reasoning',
+      ],
+      opencode: m.opencode_settings,
+    };
+  });
 }
 
 function enhancedModelList(models: OpenRouterModel[]) {
@@ -68,13 +88,21 @@ function enhancedModelList(models: OpenRouterModel[]) {
       const ageDays = (Date.now() / 1_000 - model.created) / (24 * 3600);
       const isNew = preferredIndex >= 0 && ageDays >= 0 && ageDays < 7;
       const skipSuffix = model.name.endsWith(')');
+      const addPdf =
+        isPdfSupportingModel(model.id) && !model.architecture.input_modalities.includes('pdf');
       return {
         ...model,
         name: skipSuffix ? model.name : isNew ? model.name + ' (new)' : model.name,
         preferredIndex: preferredIndex >= 0 ? preferredIndex : undefined,
         isFree: isFreeModel(model.id),
         opencode: model.opencode ?? getOpenCodeSettings(model.id),
         openclaw: model.openclaw ?? getOpenClawSettings(model.id),
+        architecture: addPdf
+          ? {
+              ...model.architecture,
+              input_modalities: model.architecture.input_modalities.concat(['pdf']),
+            }
+          : model.architecture,
       };
     });
   const sortedModels = enhancedModels.sort((a, b) => {
diff --git a/apps/web/src/lib/ai-gateway/providers/xai.ts b/apps/web/src/lib/ai-gateway/providers/xai.ts
@@ -21,6 +21,10 @@ export function isXaiModel(requestedModel: string) {
   return requestedModel.startsWith('x-ai/');
 }
 
+export function isGrok4Model(model: string) {
+  return model.startsWith('x-ai/grok-4');
+}
+
 export function applyXaiModelSettings(
   requestToMutate: GatewayRequest,
   extraHeaders: Record<string, string>
