ci(evals): run eval workflow on OpenRouter

.github/workflows/eval.yml

@@ -60,7 +60,8 @@ jobs:
         run: pnpm eval:ci evals
         continue-on-error: true
         env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          EMBEDDED_AGENT_PROVIDER: openrouter
 
       - name: Create eval status check
         uses: actions/github-script@v7

docs/testing/overview.md

@@ -279,7 +279,7 @@ describeEval("tool-name", {
 ### Running Evals
 
 ```bash
-# Requires OPENAI_API_KEY in .env
+# Requires OPENROUTER_API_KEY in .env
 pnpm eval
 
 # Run specific eval

packages/mcp-server-evals/src/bin/start-mock-stdio.ts

@@ -8,7 +8,10 @@ import type { ServerContext } from "@sentry/mcp-core/types";
 
 mswServer.listen({
   onUnhandledRequest: (req, print) => {
-    if (req.url.startsWith("https://api.openai.com/")) {
+    if (
+      req.url.startsWith("https://api.openai.com/") ||
+      req.url.startsWith("https://openrouter.ai/")
+    ) {
       return;
     }
 

packages/mcp-server-evals/src/evals/search-events.eval.ts

@@ -1,7 +1,7 @@
 import { describeEval } from "vitest-evals";
 import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils";
 
-// Note: This eval requires OPENAI_API_KEY to be set in the environment
+// Note: This eval requires OPENROUTER_API_KEY to be set in the environment
 // The search_events tool uses the AI SDK to translate natural language queries
 describeEval("search-events", {
   data: async () => {

packages/mcp-server-evals/src/evals/search-issue-events.eval.ts

@@ -1,7 +1,7 @@
 import { describeEval } from "vitest-evals";
 import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils";
 
-// Note: This eval requires OPENAI_API_KEY to be set in the environment
+// Note: This eval requires OPENROUTER_API_KEY to be set in the environment
 // The search_issue_events tool uses the AI SDK to translate natural language queries
 describeEval("search-issue-events", {
   data: async () => {

packages/mcp-server-evals/src/evals/search-issues.eval.ts

@@ -1,7 +1,7 @@
 import { describeEval } from "vitest-evals";
 import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils";
 
-// Note: This eval requires OPENAI_API_KEY to be set in the environment
+// Note: This eval requires OPENROUTER_API_KEY to be set in the environment
 // The search_issues tool uses the AI SDK to translate natural language queries
 describeEval("search-issues", {
   data: async () => {

packages/mcp-server-evals/src/evals/utils/mcpToolCallRunner.ts

@@ -1,10 +1,8 @@
 import { experimental_createMCPClient } from "@ai-sdk/mcp";
 import { Experimental_StdioMCPTransport } from "@ai-sdk/mcp/mcp-stdio";
-import { openai } from "@ai-sdk/openai";
+import { getOpenRouterModel } from "@sentry/mcp-core/internal/agents/openrouter-provider";
 import { generateText, stepCountIs, type LanguageModel } from "ai";
 
-const defaultModel = openai("gpt-4o");
-
 function toToolCall(call: { toolName: string; input: unknown }) {
   const input =
     call.input && typeof call.input === "object" && !Array.isArray(call.input)
@@ -18,7 +16,7 @@ function toToolCall(call: { toolName: string; input: unknown }) {
 }
 
 export function McpToolCallTaskRunner(
-  model: LanguageModel = defaultModel,
+  model: LanguageModel = getOpenRouterModel(),
   maxSteps = 6,
 ) {
   return async function McpToolCallTaskRunner(input: string) {

packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts

@@ -1,8 +1,8 @@
-import { openai } from "@ai-sdk/openai";
 import { generateObject, type LanguageModel } from "ai";
 import { z } from "zod";
 import { experimental_createMCPClient } from "@ai-sdk/mcp";
 import { Experimental_StdioMCPTransport } from "@ai-sdk/mcp/mcp-stdio";
+import { getOpenRouterModel } from "@sentry/mcp-core/internal/agents/openrouter-provider";
 
 // Cache for available tools to avoid reconnecting for each test
 let cachedTools: string[] | null = null;
@@ -64,8 +64,6 @@ interface ToolPredictionScorerOptions {
   result?: any;
 }
 
-const defaultModel = openai("gpt-4o");
-
 const predictionSchema = z.object({
   score: z.number().min(0).max(1).describe("Score from 0 to 1"),
   rationale: z.string().describe("Explanation of the score"),
@@ -129,7 +127,7 @@ CRITICAL: The expected tools represent the actual realistic behavior for this sp
  * A scorer that uses AI to predict what tools would be called without executing them.
  * This is much faster than actually running the tools and checking what was called.
  *
- * @param model - Optional language model to use for predictions (defaults to gpt-4o)
+ * @param model - Optional language model to use for predictions (defaults to OpenRouter)
  * @returns A scorer function that compares predicted vs expected tool calls
  *
  * @example
@@ -170,7 +168,9 @@ CRITICAL: The expected tools represent the actual realistic behavior for this sp
  * If `expectedTools` is not provided in test data, the scorer is automatically skipped
  * and returns `{ score: null }` to allow other scorers to run without interference.
  */
-export function ToolPredictionScorer(model: LanguageModel = defaultModel) {
+export function ToolPredictionScorer(
+  model: LanguageModel = getOpenRouterModel(),
+) {
   return async function ToolPredictionScorer(
     opts: ToolPredictionScorerOptions,
   ) {
@@ -205,6 +205,12 @@ export function ToolPredictionScorer(model: LanguageModel = defaultModel) {
         expectedDescription,
       ),
       schema: predictionSchema,
+      providerOptions: {
+        openai: {
+          structuredOutputs: false,
+          strictJsonSchema: false,
+        },
+      },
       experimental_telemetry: {
         isEnabled: true,
         functionId: "tool_prediction_scorer",

packages/mcp-server-mocks/src/utils.ts

@@ -17,8 +17,12 @@ export function startMockServer(options?: {
 
   mswServer.listen({
     onUnhandledRequest: (req: any, print: any) => {
-      // Ignore OpenAI requests if specified (default behavior for AI agent tests)
-      if (ignoreOpenAI && req.url.startsWith("https://api.openai.com/")) {
+      // Ignore LLM provider requests if specified (default behavior for AI agent tests)
+      if (
+        ignoreOpenAI &&
+        (req.url.startsWith("https://api.openai.com/") ||
+          req.url.startsWith("https://openrouter.ai/"))
+      ) {
         return;
       }