feat(cloud-agent-next): reject unavailable models before admission

apps/web/src/app/api/openrouter/models/validate/route.test.ts

@@ -0,0 +1,165 @@
+import { beforeEach, describe, expect, test } from '@jest/globals';
+import { NextRequest, NextResponse } from 'next/server';
+import type { OpenRouterModel } from '@/lib/organizations/organization-types';
+import { getEnhancedOpenRouterModels } from '@/lib/ai-gateway/providers/openrouter';
+import { getUserFromAuth } from '@/lib/user/server';
+import { getDirectByokModelsForUser } from '@/lib/ai-gateway/providers/direct-byok';
+import { listAvailableExperimentModels } from '@/lib/ai-gateway/experiments/list-available-experiment-models';
+import { ORGANIZATION_ID_HEADER } from '@/lib/constants';
+import { POST } from './route';
+
+jest.mock('@sentry/nextjs', () => ({ captureException: jest.fn() }));
+jest.mock('@/lib/user/server', () => ({ getUserFromAuth: jest.fn() }));
+jest.mock('@/lib/ai-gateway/providers/openrouter', () => ({
+  getEnhancedOpenRouterModels: jest.fn(),
+}));
+jest.mock('@/lib/ai-gateway/providers/direct-byok', () => ({
+  getDirectByokModelsForUser: jest.fn(),
+}));
+jest.mock('@/lib/ai-gateway/experiments/list-available-experiment-models', () => ({
+  listAvailableExperimentModels: jest.fn(),
+}));
+
+const mockedGetUserFromAuth = jest.mocked(getUserFromAuth);
+const mockedGetEnhancedOpenRouterModels = jest.mocked(getEnhancedOpenRouterModels);
+const mockedGetDirectByokModelsForUser = jest.mocked(getDirectByokModelsForUser);
+const mockedListAvailableExperimentModels = jest.mocked(listAvailableExperimentModels);
+
+function makeModel(id: string): OpenRouterModel {
+  return {
+    id,
+    name: id,
+    created: 0,
+    description: '',
+    architecture: {
+      input_modalities: ['text'],
+      output_modalities: ['text'],
+      tokenizer: 'test',
+    },
+    top_provider: { is_moderated: false },
+    pricing: { prompt: '0', completion: '0' },
+    context_length: 0,
+    supported_parameters: ['tools'],
+  };
+}
+
+function request(modelId: string, headers?: HeadersInit) {
+  return new NextRequest('http://localhost:3000/api/openrouter/models/validate', {
+    method: 'POST',
+    headers,
+    body: JSON.stringify({ modelId }),
+  });
+}
+
+describe('POST /api/openrouter/models/validate', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+    mockedGetUserFromAuth.mockResolvedValue({
+      user: null,
+      organizationId: null,
+      authFailedResponse: null,
+    } as never);
+    mockedGetEnhancedOpenRouterModels.mockResolvedValue({ data: [makeModel('available/model')] });
+    mockedGetDirectByokModelsForUser.mockResolvedValue([]);
+    mockedListAvailableExperimentModels.mockResolvedValue([]);
+  });
+
+  test('confirms a Kilo-eligible catalog model', async () => {
+    const response = await POST(request('available/model'));
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toEqual({ valid: true });
+  });
+
+  test('does not expose details for an unavailable model', async () => {
+    const response = await POST(request('missing/model'));
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toEqual({ valid: false, reason: 'unavailable' });
+  });
+
+  test('uses the public catalog after failed optional authentication', async () => {
+    mockedGetUserFromAuth.mockResolvedValue({
+      user: null,
+      organizationId: null,
+      authFailedResponse: NextResponse.json({ error: 'Unauthorized' }, { status: 401 }),
+    } as never);
+
+    const response = await POST(request('available/model'));
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toEqual({ valid: true });
+  });
+
+  test('rejects organization-scoped validation through the personal endpoint', async () => {
+    const response = await POST(
+      request('available/model', { [ORGANIZATION_ID_HEADER]: 'organization-id' })
+    );
+
+    expect(response.status).toBe(400);
+    await expect(response.json()).resolves.toEqual({
+      error: 'Organization-scoped validation must use /api/organizations/[id]/models/validate',
+    });
+    expect(mockedGetUserFromAuth).not.toHaveBeenCalled();
+    expect(mockedGetEnhancedOpenRouterModels).not.toHaveBeenCalled();
+  });
+
+  test('returns a service failure when catalog construction fails', async () => {
+    mockedGetEnhancedOpenRouterModels.mockRejectedValue(new Error('catalog unavailable'));
+
+    const response = await POST(request('available/model'));
+
+    expect(response.status).toBe(500);
+    await expect(response.json()).resolves.toEqual({
+      error: 'Failed to validate model',
+      message: 'Error from model catalog',
+    });
+  });
+
+  test('loads authenticated auxiliary catalogs concurrently', async () => {
+    mockedGetUserFromAuth.mockResolvedValue({
+      user: { id: 'user-id' },
+      organizationId: null,
+      authFailedResponse: null,
+    } as never);
+    mockedGetEnhancedOpenRouterModels.mockResolvedValue({ data: [] });
+    let markByokStarted: (() => void) | undefined;
+    const byokStarted = new Promise<void>(resolve => {
+      markByokStarted = resolve;
+    });
+    type DirectByokModels = Awaited<ReturnType<typeof getDirectByokModelsForUser>>;
+    let resolveByok: ((models: DirectByokModels) => void) | undefined;
+    const byokPending = new Promise<DirectByokModels>(resolve => {
+      resolveByok = resolve;
+    });
+    mockedGetDirectByokModelsForUser.mockImplementation(() => {
+      if (!markByokStarted) throw new Error('BYOK start signal was not initialized');
+      markByokStarted();
+      return byokPending;
+    });
+    mockedListAvailableExperimentModels.mockResolvedValue([makeModel('experiment/model')]);
+
+    const responsePromise = POST(request('experiment/model'));
+    await byokStarted;
+    const finishByok = resolveByok;
+    if (!finishByok) throw new Error('BYOK lookup did not start');
+    try {
+      expect(mockedListAvailableExperimentModels).toHaveBeenCalledTimes(1);
+    } finally {
+      finishByok([]);
+      await responsePromise;
+    }
+  });
+
+  test('rejects an invalid body without reading a catalog', async () => {
+    const response = await POST(
+      new NextRequest('http://localhost:3000/api/openrouter/models/validate', {
+        method: 'POST',
+        body: JSON.stringify({ modelId: '' }),
+      })
+    );
+
+    expect(response.status).toBe(400);
+    expect(mockedGetEnhancedOpenRouterModels).not.toHaveBeenCalled();
+  });
+});

apps/web/src/app/api/openrouter/models/validate/route.ts

@@ -0,0 +1,77 @@
+import type { NextRequest } from 'next/server';
+import { NextResponse } from 'next/server';
+import { captureException } from '@sentry/nextjs';
+import * as z from 'zod';
+import { getEnhancedOpenRouterModels } from '@/lib/ai-gateway/providers/openrouter';
+import { getUserFromAuth } from '@/lib/user/server';
+import { getDirectByokModelsForUser } from '@/lib/ai-gateway/providers/direct-byok';
+import { FEATURE_HEADER, validateFeatureHeader } from '@/lib/feature-detection';
+import { ORGANIZATION_ID_HEADER } from '@/lib/constants';
+import { filterByFeature } from '@/lib/ai-gateway/models';
+import { listAvailableExperimentModels } from '@/lib/ai-gateway/experiments/list-available-experiment-models';
+
+const BodySchema = z.object({ modelId: z.string().trim().min(1) });
+
+async function tryGetUserFromAuth() {
+  try {
+    return await getUserFromAuth({ adminOnly: false });
+  } catch (error) {
+    console.error('[validateOpenRouterModel] failed to get user from auth', error);
+    return { user: null, organizationId: null };
+  }
+}
+
+export async function POST(request: NextRequest) {
+  let body: unknown;
+  try {
+    body = await request.json();
+  } catch {
+    return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 });
+  }
+
+  const bodyResult = BodySchema.safeParse(body);
+  if (!bodyResult.success) {
+    return NextResponse.json(
+      { error: 'Invalid request body', details: z.treeifyError(bodyResult.error) },
+      { status: 400 }
+    );
+  }
+
+  if (request.headers.get(ORGANIZATION_ID_HEADER)) {
+    return NextResponse.json(
+      { error: 'Organization-scoped validation must use /api/organizations/[id]/models/validate' },
+      { status: 400 }
+    );
+  }
+
+  const feature = validateFeatureHeader(request.headers.get(FEATURE_HEADER));
+  const auth = await tryGetUserFromAuth();
+  try {
+    const models = await getEnhancedOpenRouterModels();
+    if (!Array.isArray(models.data)) {
+      throw new Error('Model catalog returned invalid data');
+    }
+    const [byokModels, experimentModels] = await Promise.all([
+      auth?.user ? getDirectByokModelsForUser(auth.user.id) : [],
+      listAvailableExperimentModels(),
+    ]);
+    const available = filterByFeature(
+      models.data.concat(byokModels, experimentModels),
+      feature
+    ).some(model => model.id === bodyResult.data.modelId);
+    return NextResponse.json(available ? { valid: true } : { valid: false, reason: 'unavailable' });
+  } catch (error) {
+    captureException(error, {
+      tags: { endpoint: 'openrouter/models/validate' },
+      extra: {
+        action: 'validating_model',
+        userId: auth?.user?.id,
+        organizationId: auth?.organizationId,
+      },
+    });
+    return NextResponse.json(
+      { error: 'Failed to validate model', message: 'Error from model catalog' },
+      { status: 500 }
+    );
+  }
+}

apps/web/src/app/api/organizations/[id]/models/validate/route.test.ts

@@ -0,0 +1,78 @@
+import { beforeEach, describe, expect, test } from '@jest/globals';
+import { NextRequest, NextResponse } from 'next/server';
+import type { OpenRouterModel } from '@/lib/organizations/organization-types';
+import { handleTRPCRequest } from '@/lib/trpc-route-handler';
+import { POST } from './route';
+
+jest.mock('@/lib/trpc-route-handler', () => ({ handleTRPCRequest: jest.fn() }));
+
+const mockedHandleTRPCRequest = jest.mocked(handleTRPCRequest);
+const listAvailableModels = jest.fn();
+
+function makeModel(id: string): OpenRouterModel {
+  return {
+    id,
+    name: id,
+    created: 0,
+    description: '',
+    architecture: {
+      input_modalities: ['text'],
+      output_modalities: ['text'],
+      tokenizer: 'test',
+    },
+    top_provider: { is_moderated: false },
+    pricing: { prompt: '0', completion: '0' },
+    context_length: 0,
+    supported_parameters: ['tools'],
+  };
+}
+
+function request(modelId: string) {
+  return new NextRequest('http://localhost:3000/api/organizations/org-1/models/validate', {
+    method: 'POST',
+    body: JSON.stringify({ modelId }),
+  });
+}
+
+describe('POST /api/organizations/[id]/models/validate', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+    listAvailableModels.mockResolvedValue({ data: [makeModel('available/model')] });
+    mockedHandleTRPCRequest.mockImplementation(async (request, handler) => {
+      const result = await handler({
+        organizations: { settings: { listAvailableModels } },
+      } as never);
+      return NextResponse.json(result);
+    });
+  });
+
+  test('validates against the authorized organization catalog', async () => {
+    const response = await POST(request('available/model'), {
+      params: Promise.resolve({ id: 'org-1' }),
+    });
+
+    expect(listAvailableModels).toHaveBeenCalledWith({ organizationId: 'org-1' });
+    await expect(response.json()).resolves.toEqual({ valid: true });
+  });
+
+  test('reports an organization-unavailable model without policy details', async () => {
+    const response = await POST(request('missing/model'), {
+      params: Promise.resolve({ id: 'org-1' }),
+    });
+
+    await expect(response.json()).resolves.toEqual({ valid: false, reason: 'unavailable' });
+  });
+
+  test('rejects an invalid body before invoking organization authorization', async () => {
+    const response = await POST(
+      new NextRequest('http://localhost:3000/api/organizations/org-1/models/validate', {
+        method: 'POST',
+        body: JSON.stringify({ modelId: '' }),
+      }),
+      { params: Promise.resolve({ id: 'org-1' }) }
+    );
+
+    expect(response.status).toBe(400);
+    expect(mockedHandleTRPCRequest).not.toHaveBeenCalled();
+  });
+});

apps/web/src/app/api/organizations/[id]/models/validate/route.ts

@@ -0,0 +1,41 @@
+import type { NextRequest } from 'next/server';
+import { NextResponse } from 'next/server';
+import * as z from 'zod';
+import { handleTRPCRequest } from '@/lib/trpc-route-handler';
+import { FEATURE_HEADER, validateFeatureHeader } from '@/lib/feature-detection';
+import { filterByFeature } from '@/lib/ai-gateway/models';
+
+const BodySchema = z.object({ modelId: z.string().trim().min(1) });
+
+type ValidationResult = { valid: true } | { valid: false; reason: 'unavailable' };
+
+export async function POST(
+  request: NextRequest,
+  { params }: { params: Promise<{ id: string }> }
+): Promise<NextResponse<{ error: string; message?: string } | ValidationResult>> {
+  let body: unknown;
+  try {
+    body = await request.json();
+  } catch {
+    return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 });
+  }
+
+  const bodyResult = BodySchema.safeParse(body);
+  if (!bodyResult.success) {
+    return NextResponse.json(
+      { error: 'Invalid request body', details: z.treeifyError(bodyResult.error) },
+      { status: 400 }
+    );
+  }
+
+  const organizationId = (await params).id;
+  const feature = validateFeatureHeader(request.headers.get(FEATURE_HEADER));
+
+  return handleTRPCRequest<ValidationResult>(request, async caller => {
+    const result = await caller.organizations.settings.listAvailableModels({ organizationId });
+    const available = filterByFeature(result.data, feature).some(
+      model => model.id === bodyResult.data.modelId
+    );
+    return available ? { valid: true } : { valid: false, reason: 'unavailable' };
+  });
+}

services/cloud-agent-next/.dev.vars.example

@@ -24,9 +24,9 @@ INTERNAL_API_SECRET=your-internal-api-secret-here
 # @url nextjs
 KILOCODE_BACKEND_BASE_URL=http://localhost:3000
 # KILO_OPENROUTER_BASE options:
-#   Real LLM path (dev):  http://host.docker.internal:3000/api
+#   Real LLM path (dev):  http://localhost:3000/api
 #   Fake LLM (E2E harness, local dev only):
-#     http://host.docker.internal:8811/api   (add portOffset if set)
+#     http://localhost:8811/api   (add portOffset if set)
 #   Start the `fake-llm` dev service before pointing at the fake URL.
 # @url nextjs/api
 KILO_OPENROUTER_BASE=http://localhost:3000/api