feat(model-experiments): reject reserved public_model_id prefixes (#3618)

* feat(model-experiments): reject reserved public_model_id prefixes

Creating or updating a model experiment with a public_model_id under the kilo/, kilocode/, or kilo-internal/ namespaces now fails validation, since those are reserved for Kilo-owned models and must not be claimed by partner experiment public ids.

* refactor(model-experiments): centralize Kilo provider prefix constants

Introduce KILOCODE_KILO_PROVIDER_PREFIX, KILOCLAW_KILO_PROVIDER_PREFIX, and CUSTOM_LLM_PREFIX in the shared model-utils module and use them across apps/web instead of repeated string literals. Also rename pick-variant.test.ts public ids off the now-reserved kilo/ namespace so the suite keeps passing, and fix the admin create-experiment placeholder which suggested a reserved id.

* style: oxfmt wrap long lines after partner/ id rename

---------

Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com>
Co-authored-by: Christiaan Arnoldus <christiaan@kilocode.ai>

Author: kilo-code-bot[bot]

apps/web/src/app/(app)/claw/hooks/useDefaultModelSelection.ts

@@ -1,14 +1,15 @@
 import { useEffect, useState } from 'react';
 import type { ModelOption } from '@/components/shared/ModelCombobox';
+import { KILOCLAW_KILO_PROVIDER_PREFIX } from '@/lib/ai-gateway/model-utils';
 
 export function getDefaultSelectedModel(
   kilocodeDefaultModel: string | null | undefined,
   modelOptions: ModelOption[]
 ) {
   if (modelOptions.length === 0) return '';
-  if (!kilocodeDefaultModel?.startsWith('kilocode/')) return '';
+  if (!kilocodeDefaultModel?.startsWith(KILOCLAW_KILO_PROVIDER_PREFIX)) return '';
 
-  const defaultModel = kilocodeDefaultModel.replace(/^kilocode\//, '');
+  const defaultModel = kilocodeDefaultModel.slice(KILOCLAW_KILO_PROVIDER_PREFIX.length);
   if (modelOptions.some(model => model.id === defaultModel)) return defaultModel;
   return '';
 }

apps/web/src/app/admin/custom-llms/CustomLlmsContent.tsx

@@ -28,6 +28,7 @@ import {
 import { CustomLlmDefinitionSchema } from '@kilocode/db/schema-types';
 import type { CustomLlmDefinition } from '@kilocode/db/schema-types';
 import { deepStrict } from '@/lib/zod/deep-strict';
+import { CUSTOM_LLM_PREFIX } from '@/lib/ai-gateway/model-utils';
 import { toast } from 'sonner';
 import { Plus, Pencil } from 'lucide-react';
 import Editor from '@monaco-editor/react';
@@ -226,7 +227,7 @@ export function CustomLlmsContent() {
                   }))
                 }
                 disabled={editor.mode === 'edit'}
-                placeholder="e.g. kilo-internal/my-custom-model"
+                placeholder={`e.g. ${CUSTOM_LLM_PREFIX}my-custom-model`}
                 className="font-mono"
               />
             </div>

apps/web/src/app/admin/model-experiments/ModelExperimentsContent.tsx

@@ -305,7 +305,7 @@ function CreateExperimentDialog({
               id="exp-public-id"
               value={publicModelId}
               onChange={e => setPublicModelId(e.target.value)}
-              placeholder="e.g. kilo/preview-experiment-foo"
+              placeholder="e.g. partner/preview-experiment-foo"
               className="font-mono"
             />
           </div>

apps/web/src/lib/ai-gateway/experiments/pick-variant.test.ts

@@ -82,29 +82,31 @@ async function makeActiveExperiment(opts: {
 describe('isPublicIdExperimented', () => {
   it('returns false for an unknown public id', async () => {
     await clearRoutingCaches();
-    expect(await isPublicIdExperimented('kilo/preview-not-experimented')).toBe(false);
+    expect(await isPublicIdExperimented('partner/preview-not-experimented')).toBe(false);
   });
 
   redisIt('returns true when the public id has an active experiment', async () => {
-    await makeActiveExperiment({ publicId: 'kilo/preview-iset-active' });
-    expect(await seedExperimentedPublicIds(['kilo/preview-iset-active'])).toBe(true);
-    expect(await isPublicIdExperimented('kilo/preview-iset-active')).toBe(true);
+    await makeActiveExperiment({ publicId: 'partner/preview-iset-active' });
+    expect(await seedExperimentedPublicIds(['partner/preview-iset-active'])).toBe(true);
+    expect(await isPublicIdExperimented('partner/preview-iset-active')).toBe(true);
   });
 
   redisIt('returns true when the public id has only a paused experiment', async () => {
-    const { experimentId } = await makeActiveExperiment({ publicId: 'kilo/preview-iset-paused' });
+    const { experimentId } = await makeActiveExperiment({
+      publicId: 'partner/preview-iset-paused',
+    });
     const caller = await createCallerForUser(admin.id);
     await caller.admin.modelExperiments.pause({ id: experimentId });
-    expect(await seedExperimentedPublicIds(['kilo/preview-iset-paused'])).toBe(true);
-    expect(await isPublicIdExperimented('kilo/preview-iset-paused')).toBe(true);
+    expect(await seedExperimentedPublicIds(['partner/preview-iset-paused'])).toBe(true);
+    expect(await isPublicIdExperimented('partner/preview-iset-paused')).toBe(true);
   });
 });
 
 describe('pickModelExperimentVariant', () => {
   it('returns null for a public id with no routing-relevant experiment', async () => {
     await clearRoutingCaches();
     const result = await pickModelExperimentVariant({
-      publicModelId: 'kilo/preview-pick-none',
+      publicModelId: 'partner/preview-pick-none',
       userId: 'user-1',
       machineId: null,
       clientIp: null,
@@ -113,15 +115,15 @@ describe('pickModelExperimentVariant', () => {
   });
 
   it('produces stable assignments for the same userId', async () => {
-    await makeActiveExperiment({ publicId: 'kilo/preview-stable' });
+    await makeActiveExperiment({ publicId: 'partner/preview-stable' });
     const first = await pickModelExperimentVariant({
-      publicModelId: 'kilo/preview-stable',
+      publicModelId: 'partner/preview-stable',
       userId: 'user-1',
       machineId: null,
       clientIp: null,
     });
     const second = await pickModelExperimentVariant({
-      publicModelId: 'kilo/preview-stable',
+      publicModelId: 'partner/preview-stable',
       userId: 'user-1',
       machineId: null,
       clientIp: null,
@@ -135,11 +137,11 @@ describe('pickModelExperimentVariant', () => {
 
   it('decrypts and returns the partner-issued api key for the chosen variant', async () => {
     const { variantA, variantB } = await makeActiveExperiment({
-      publicId: 'kilo/preview-key',
+      publicId: 'partner/preview-key',
       apiKeys: ['sk-control-secret', 'sk-treatment-secret'],
     });
     const result = await pickModelExperimentVariant({
-      publicModelId: 'kilo/preview-key',
+      publicModelId: 'partner/preview-key',
       userId: 'user-key',
       machineId: null,
       clientIp: null,
@@ -151,21 +153,21 @@ describe('pickModelExperimentVariant', () => {
   });
 
   it('respects allocation-subject precedence: user > machine > ip', async () => {
-    await makeActiveExperiment({ publicId: 'kilo/preview-alloc' });
+    await makeActiveExperiment({ publicId: 'partner/preview-alloc' });
     const userPick = await pickModelExperimentVariant({
-      publicModelId: 'kilo/preview-alloc',
+      publicModelId: 'partner/preview-alloc',
       userId: 'user-z',
       machineId: 'machine-z',
       clientIp: '1.2.3.4',
     });
     const machinePick = await pickModelExperimentVariant({
-      publicModelId: 'kilo/preview-alloc',
+      publicModelId: 'partner/preview-alloc',
       userId: null,
       machineId: 'machine-z',
       clientIp: '1.2.3.4',
     });
     const ipPick = await pickModelExperimentVariant({
-      publicModelId: 'kilo/preview-alloc',
+      publicModelId: 'partner/preview-alloc',
       userId: null,
       machineId: null,
       clientIp: '1.2.3.4',
@@ -182,9 +184,9 @@ describe('pickModelExperimentVariant', () => {
   });
 
   it('returns unavailable when no allocation subject is available', async () => {
-    await makeActiveExperiment({ publicId: 'kilo/preview-noalloc' });
+    await makeActiveExperiment({ publicId: 'partner/preview-noalloc' });
     const result = await pickModelExperimentVariant({
-      publicModelId: 'kilo/preview-noalloc',
+      publicModelId: 'partner/preview-noalloc',
       userId: null,
       machineId: null,
       clientIp: null,
@@ -193,12 +195,12 @@ describe('pickModelExperimentVariant', () => {
   });
 
   it('returns not-found for a paused experiment so traffic does not silently fall through', async () => {
-    const { experimentId } = await makeActiveExperiment({ publicId: 'kilo/preview-paused' });
+    const { experimentId } = await makeActiveExperiment({ publicId: 'partner/preview-paused' });
     const caller = await createCallerForUser(admin.id);
     await caller.admin.modelExperiments.pause({ id: experimentId });
     await clearRoutingCaches();
     const result = await pickModelExperimentVariant({
-      publicModelId: 'kilo/preview-paused',
+      publicModelId: 'partner/preview-paused',
       userId: 'user-q',
       machineId: null,
       clientIp: null,
@@ -208,11 +210,11 @@ describe('pickModelExperimentVariant', () => {
 
   it('hot-swap: serves the new variant_version_id but keeps the same bucket', async () => {
     const { experimentId, variantA, variantB } = await makeActiveExperiment({
-      publicId: 'kilo/preview-hotswap',
+      publicId: 'partner/preview-hotswap',
     });
     const caller = await createCallerForUser(admin.id);
     const before = await pickModelExperimentVariant({
-      publicModelId: 'kilo/preview-hotswap',
+      publicModelId: 'partner/preview-hotswap',
       userId: 'user-pinned',
       machineId: null,
       clientIp: null,
@@ -229,7 +231,7 @@ describe('pickModelExperimentVariant', () => {
     await clearRoutingCaches();
 
     const after = await pickModelExperimentVariant({
-      publicModelId: 'kilo/preview-hotswap',
+      publicModelId: 'partner/preview-hotswap',
       userId: 'user-pinned',
       machineId: null,
       clientIp: null,
@@ -249,13 +251,13 @@ describe('pickModelExperimentVariant', () => {
   it('weighted distribution lands roughly on configured weights', async () => {
     // 1:3 split. With 200 distinct seeds, control should be near 25%.
     await makeActiveExperiment({
-      publicId: 'kilo/preview-weighted',
+      publicId: 'partner/preview-weighted',
       weights: [1, 3],
     });
     const counts = { control: 0, treatment: 0 };
     for (let i = 0; i < 200; i++) {
       const r = await pickModelExperimentVariant({
-        publicModelId: 'kilo/preview-weighted',
+        publicModelId: 'partner/preview-weighted',
         userId: `user-${i}`,
         machineId: null,
         clientIp: null,
@@ -272,10 +274,10 @@ describe('pickModelExperimentVariant', () => {
 
   it('historical attribution survives hot-swap: old variant_version_id still resolves to old upstream via DB', async () => {
     const { experimentId } = await makeActiveExperiment({
-      publicId: 'kilo/preview-attr',
+      publicId: 'partner/preview-attr',
     });
     const before = await pickModelExperimentVariant({
-      publicModelId: 'kilo/preview-attr',
+      publicModelId: 'partner/preview-attr',
       userId: 'user-attr',
       machineId: null,
       clientIp: null,
@@ -308,13 +310,13 @@ describe('pickModelExperimentVariant', () => {
 
   it('completed experiments are not returned by the picker (status none after completion)', async () => {
     const { experimentId } = await makeActiveExperiment({
-      publicId: 'kilo/preview-completed',
+      publicId: 'partner/preview-completed',
     });
     const caller = await createCallerForUser(admin.id);
     await caller.admin.modelExperiments.complete({ id: experimentId });
     await clearRoutingCaches();
     const result = await pickModelExperimentVariant({
-      publicModelId: 'kilo/preview-completed',
+      publicModelId: 'partner/preview-completed',
       userId: 'user-c',
       machineId: null,
       clientIp: null,

apps/web/src/lib/ai-gateway/model-utils.ts

@@ -3,6 +3,18 @@
  * Keep this file free of server-only dependencies.
  */
 
+/**
+ * Public-id namespace prefixes for Kilo-owned models. These are reserved and
+ * must not be claimed by partner experiment public ids or custom upstreams.
+ *
+ * The names look swapped but are intentional: Kilo Code (the extension) selects
+ * Kilo-hosted models under `kilo/`, while KiloClaw selects them under
+ * `kilocode/`. `kilo-internal/` is the custom LLM (`custom_llm2`) namespace.
+ */
+export const KILOCODE_KILO_PROVIDER_PREFIX = 'kilo/';
+export const KILOCLAW_KILO_PROVIDER_PREFIX = 'kilocode/';
+export const CUSTOM_LLM_PREFIX = 'kilo-internal/';
+
 /**
  * Normalize a model ID by removing the `:free`, `:exacto`, etc. suffixes if present.
  */

apps/web/src/lib/ai-gateway/providers/get-provider.ts

@@ -1,6 +1,7 @@
 import type { GatewayRequest } from '@/lib/ai-gateway/providers/openrouter/types';
 import { shouldRouteToVercel } from '@/lib/ai-gateway/providers/vercel';
 import { findKiloExclusiveModel, isKiloExclusiveModel } from '@/lib/ai-gateway/models';
+import { CUSTOM_LLM_PREFIX } from '@/lib/ai-gateway/model-utils';
 import {
   getBYOKforOrganization,
   getBYOKforUser,
@@ -183,8 +184,8 @@ export async function getProvider(input: GetProviderInput): Promise<GetProviderR
   const kiloExclusiveModel = findKiloExclusiveModel(requestedModel);
 
   // Model experiment routing for dedicated preview public ids. Runs before
-  // `kilo-internal/...` and the `kiloExclusiveModels` lookup so an
-  // experimented public id never falls through to OpenRouter/Vercel.
+  // the custom-LLM (`kilo-internal/...`) and the `kiloExclusiveModels` lookup
+  // so an experimented public id never falls through to OpenRouter/Vercel.
   const experimented = await isPublicIdExperimented(requestedModel);
   if (experimented === true) {
     if (kiloExclusiveModel) {
@@ -223,7 +224,7 @@ export async function getProvider(input: GetProviderInput): Promise<GetProviderR
     // this id. Fall through to non-experiment routing.
   }
 
-  if (requestedModel.startsWith('kilo-internal/') && organizationId) {
+  if (requestedModel.startsWith(CUSTOM_LLM_PREFIX) && organizationId) {
     const customLlmResult = await checkCustomLlm(requestedModel, organizationId);
     if (customLlmResult) {
       return customLlmResult;

apps/web/src/lib/format-model-name.ts

@@ -1,3 +1,5 @@
+import { KILOCODE_KILO_PROVIDER_PREFIX } from '@/lib/ai-gateway/model-utils';
+
 /**
  * Strips provider prefixes from a model slug to produce a short display name.
  *
@@ -9,7 +11,9 @@
 export function formatShortModelName(slug: string): string {
   if (!slug) return slug;
   // Strip kilo/ prefix first
-  const withoutKilo = slug.startsWith('kilo/') ? slug.slice(5) : slug;
+  const withoutKilo = slug.startsWith(KILOCODE_KILO_PROVIDER_PREFIX)
+    ? slug.slice(KILOCODE_KILO_PROVIDER_PREFIX.length)
+    : slug;
   // Strip provider prefix (everything before and including the first /)
   const slashIndex = withoutKilo.indexOf('/');
   return slashIndex === -1 ? withoutKilo : withoutKilo.slice(slashIndex + 1);

apps/web/src/routers/admin/custom-llm-router.ts

@@ -5,13 +5,12 @@ import { CustomLlmDefinitionSchema } from '@kilocode/db/schema-types';
 import { asc, eq } from 'drizzle-orm';
 import { TRPCError } from '@trpc/server';
 import * as z from 'zod';
-
-const KILO_INTERNAL_PREFIX = 'kilo-internal/';
+import { CUSTOM_LLM_PREFIX } from '@/lib/ai-gateway/model-utils';
 
 const publicIdSchema = z
   .string()
   .min(1, 'public_id is required')
-  .startsWith(KILO_INTERNAL_PREFIX, `public_id must start with "${KILO_INTERNAL_PREFIX}"`);
+  .startsWith(CUSTOM_LLM_PREFIX, `public_id must start with "${CUSTOM_LLM_PREFIX}"`);
 
 const UpsertCustomLlmSchema = z.object({
   public_id: publicIdSchema,