test(evals): Migrate to vitest-evals harness

README.md

@@ -107,7 +107,7 @@ EMBEDDED_AGENT_PROVIDER=     # Required when multiple provider keys are set: 'op
 OPENAI_API_KEY=              # Required if using OpenAI
 ANTHROPIC_API_KEY=           # Required if using Anthropic
 OPENROUTER_API_KEY=          # Required if using OpenRouter
-OPENROUTER_MODEL=            # Optional OpenRouter model, defaults to 'openai/gpt-5'
+# OPENROUTER_MODEL=openai/gpt-5  # Optional OpenRouter model, defaults to 'openai/gpt-5'
 
 # Optional overrides
 SENTRY_HOST=                 # For self-hosted deployments
@@ -227,8 +227,9 @@ pnpm test
 
 ```shell
 # .env (in project root)
-OPENAI_API_KEY=      # Use OpenAI-backed AI-powered tools
-OPENROUTER_API_KEY=  # Or use OpenRouter-backed AI-powered tools
+EMBEDDED_AGENT_PROVIDER=openrouter
+OPENROUTER_API_KEY=  # Use OpenRouter-backed AI-powered tools and evals
+# OPENROUTER_MODEL=openai/gpt-5  # Optional, defaults to openai/gpt-5
 ```
 
 Note: The root `.env` file provides defaults for all packages. Individual packages can have their own `.env` files to override these defaults during development.

docs/contributing/adding-tools.md

@@ -277,18 +277,20 @@ See [api-patterns.md](api-patterns.md#mock-patterns) for validation examples.
 **⚠️ Each eval costs time and API credits. Only test core functionality!**
 
 ```typescript
-describeEval("your-tool", {
-  data: async () => [
-    {
-      input: `Primary use case in ${FIXTURES.organizationSlug}`,
-      expected: "Expected response"
-    },
-    // Maximum 2-3 scenarios!
-  ],
-  task: TaskRunner(),
-  scorers: [Factuality()],
-  threshold: 0.6,
-});
+import { FIXTURES, defineToolPredictionEval } from "./utils";
+
+defineToolPredictionEval("your-tool", [
+  {
+    input: `Primary use case in ${FIXTURES.organizationSlug}`,
+    expectedTools: [
+      {
+        name: "your_sentry_tool",
+        arguments: { organizationSlug: FIXTURES.organizationSlug },
+      },
+    ],
+  },
+  // Maximum 2-3 scenarios!
+]);
 ```
 
 ## Testing Workflow

docs/contributing/pr-management.md

@@ -184,11 +184,10 @@ datasets: errors, logs, and spans.
 Co-Authored-By: Codex CLI Agent <noreply@openai.com>"
 
 # Bug fix
-git commit -m "fix(evals): update search-events eval to use available exports
+git commit -m "fix(evals): update search-events eval to use suite helpers
 
-Replace missing TaskRunner and Factuality imports with NoOpTaskRunner 
-and ToolPredictionScorer to resolve CI build failures after factuality 
-checker removal.
+Replace stale eval imports with the shared vitest-evals harness helpers
+so the eval compiles against the current judge API.
 
 Co-Authored-By: Codex CLI Agent <noreply@openai.com>"
 

docs/testing/overview.md

@@ -259,27 +259,21 @@ expect(result.timestamp).toMatchInlineSnapshot(); // ❌
 ### Eval Test Structure
 
 ```typescript
-import { describeEval } from "vitest-evals";
-import { TaskRunner, Factuality } from "./utils";
-
-describeEval("tool-name", {
-  data: async () => [
-    {
-      input: "Natural language request",
-      expected: "Expected response content"
-    }
-  ],
-  task: TaskRunner(),      // Uses AI to call tools
-  scorers: [Factuality()], // Validates output
-  threshold: 0.6,
-  timeout: 30000
-});
+import { defineMcpToolCallEval } from "./utils";
+
+defineMcpToolCallEval("tool-name", [
+  {
+    input: "Natural language request",
+    expectedTools: [{ name: "find_organizations", arguments: {} }],
+  },
+]);
 ```
 
 ### Running Evals
 
 ```bash
-# Requires OPENAI_API_KEY in .env
+# Requires an embedded-agent provider in .env, such as
+# EMBEDDED_AGENT_PROVIDER=openrouter and OPENROUTER_API_KEY
 pnpm eval
 
 # Run specific eval
@@ -319,12 +313,12 @@ function createTestIssues(count: number) {
 ### Timeout Configuration
 
 ```typescript
-it("handles large datasets", async () => {
-  const largeDataset = createTestIssues(1000);
-
-  const result = await handler(mockContext, params);
-  expect(result).toBeDefined();
-}, { timeout: 10000 }); // 10 second timeout
+// vitest.config.ts
+export default defineConfig({
+  test: {
+    testTimeout: 10000,
+  },
+});
 ```
 
 ### Memory Testing

docs/testing/stdio.md

@@ -662,12 +662,12 @@ sentry-mcp --access-token=TOKEN
 
 ```bash
 # Using nvm
-nvm install 20
-nvm use 20
+nvm install 22.13
+nvm use 22.13
 pnpm start --access-token=TOKEN
 
-nvm install 22
-nvm use 22
+nvm install 24
+nvm use 24
 pnpm start --access-token=TOKEN
 ```
 

packages/mcp-core/src/mocks-exports.test.ts

@@ -4,6 +4,7 @@ import {
   createFeedbackIssue,
   flamegraphFixture as payloadFlamegraphFixture,
 } from "@sentry/mcp-server-mocks/payloads";
+import { isLLMProviderRequest } from "@sentry/mcp-server-mocks/utils";
 
 describe("@sentry/mcp-server-mocks exports", () => {
   it("re-exports flamegraphFixture from the package entrypoint", () => {
@@ -21,4 +22,33 @@ describe("@sentry/mcp-server-mocks exports", () => {
 
     expect(feedbackIssue.title).toBe("User Feedback: Export regression");
   });
+
+  it("identifies LLM provider requests ignored by MSW", () => {
+    expect(isLLMProviderRequest("https://api.openai.com/v1/responses")).toBe(
+      true,
+    );
+    expect(isLLMProviderRequest("https://api.anthropic.com/v1/messages")).toBe(
+      true,
+    );
+    expect(
+      isLLMProviderRequest("https://openrouter.ai/api/v1/chat/completions"),
+    ).toBe(true);
+    expect(
+      isLLMProviderRequest("https://example.openai.azure.com/openai/v1/"),
+    ).toBe(true);
+    expect(
+      isLLMProviderRequest(
+        "https://example.openai.azure.com/openai/deployments/model/chat/completions",
+      ),
+    ).toBe(true);
+
+    expect(isLLMProviderRequest("https://sentry.io/api/0/projects/")).toBe(
+      false,
+    );
+    expect(isLLMProviderRequest("https://example.azure.com/openai/v1/")).toBe(
+      false,
+    );
+    expect(isLLMProviderRequest("/relative/request")).toBe(false);
+    expect(isLLMProviderRequest("not a valid URL")).toBe(false);
+  });
 });

packages/mcp-core/src/test-setup.ts

@@ -16,7 +16,7 @@ config({ path: path.resolve(__dirname, "../.env") });
 // Load root .env second (for shared defaults - won't override local or shell vars)
 config({ path: path.join(rootDir, ".env") });
 
-startMockServer({ ignoreOpenAI: true });
+startMockServer({ ignoreLLMProviderRequests: true });
 
 /**
  * Creates a ServerContext for testing with default values and optional overrides.

packages/mcp-core/src/tools/catalog/search-events.ts

@@ -300,7 +300,7 @@ function buildSearchRepairPrompt(params: {
     "The query may be natural language or already-valid Sentry search syntax.",
     "Preserve valid explicit parameters, but correct dataset, query syntax, fields, sort, and time range when they conflict or would fail.",
     "If the user query already uses Sentry search syntax, treat its filters as authoritative unless the search validation step proves a field is invalid.",
-    "For spans, logs, and metrics, use datasetAttributes to discover likely fields with substringMatch, query, and attributeTypes before dropping or renaming explicit fields.",
+    "For spans, logs, and metrics, use datasetAttributes to discover likely fields before dropping or renaming explicit fields.",
     "A broad datasetAttributes result may be truncated, so absence from that preview does not prove an explicit field is invalid.",
     "For non-replay datasets, convert environment parameters into query filters. For replays, keep environment in the separate environment parameter.",
     "",

packages/mcp-core/src/tools/support/search-events/agent.ts

@@ -83,16 +83,18 @@ export interface SearchEventsAgentOptions {
   projectId?: string;
 }
 
+type SearchEventsAgentResult = {
+  result: z.output<typeof searchEventsAgentOutputSchema>;
+  toolCalls: any[];
+};
+
 /**
  * Search events agent - single entry point for translating natural language queries to Sentry search syntax
  * This returns both the translated query result AND the tool calls made by the agent
  */
 export async function searchEventsAgent(
   options: SearchEventsAgentOptions,
-): Promise<{
-  result: z.output<typeof searchEventsAgentOutputSchema>;
-  toolCalls: any[];
-}> {
+): Promise<SearchEventsAgentResult> {
   // Provider check happens in callEmbeddedAgent via getAgentProvider()
   // Create tools pre-bound with the provided API service and organization
   const datasetAttributesTool = createDatasetAttributesTool({

packages/mcp-core/src/tools/support/search-events/config.ts

@@ -1,7 +1,7 @@
 // Build a dataset-agnostic system prompt
 export const systemPrompt = `You are a Sentry query translator. You need to:
 1. FIRST determine which dataset (spans, errors, logs, metrics, profiles, or replays) is most appropriate for the query
-2. Query the available attributes for that dataset using the datasetAttributes tool for spans/errors/logs/metrics/profiles, or the replayFields tool for replays
+2. Use built-in and common fields from this prompt directly; query available attributes only for custom, uncommon, user-supplied, or ambiguous fields
 3. Use the otelSemantics tool if you need OpenTelemetry semantic conventions
 4. Convert the natural language query to Sentry's search syntax (NOT SQL syntax)
 5. Decide which fields to return in the results
@@ -21,25 +21,30 @@ For queries that explicitly ask about a metric name, metric type, counter/gauge/
 For queries about captured profiles, profile IDs, flamegraphs, or profiled transactions, prefer profiles.
 For queries about session replays, clicked UI elements, rage clicks, dead clicks, visited URLs/screens, or replay users/sessions, prefer replays.
 If the user says logs, log messages, error logs, or warning logs, choose logs instead of errors.
-
-CRITICAL - FIELD VERIFICATION REQUIREMENT:
-Before constructing ANY query, you MUST verify field availability:
-1. You CANNOT assume ANY field exists without checking - not even common ones
-2. This includes ALL fields: custom attributes, database fields, HTTP fields, AI fields, user fields, etc.
-3. Fields vary by project based on what data is being sent to Sentry
-4. Using an unverified field WILL cause your query to fail with "field not found" errors
-5. For spans, logs, and metrics, datasetAttributes can list likely fields using substringMatch/query/attributeTypes
-6. A broad datasetAttributes listing is a discovery preview and may be truncated; do not treat absence from the preview as proof that a user-supplied field is invalid
-7. Replay fields vary by project too, so use replayFields before constructing replay queries
+For HTTP/API span queries, use common fields like http.method, http.url, http.status_code, span.duration, and transaction directly.
+
+FIELD VERIFICATION REQUIREMENT:
+Use built-in fields and documented common fields from this prompt directly.
+When the user provides complete Sentry search syntax, requested return fields, and sort/grouping intent, preserve those explicit filters and fields directly instead of calling discovery tools only to validate them.
+Use discovery tools for custom, uncommon, user-supplied, or ambiguous fields:
+1. Custom fields and tags vary by project based on what data is being sent
+2. Using a non-existent custom field will cause query failures
+3. For spans, logs, and metrics, datasetAttributes can list likely fields; use substringMatch for specific custom or ambiguous field names
+4. A broad datasetAttributes listing is a discovery preview and may be truncated; do not treat absence from the preview as proof that a user-supplied field is invalid
+5. Replay fields vary by project too, so use replayFields before constructing replay queries
 
 TOOL USAGE GUIDELINES:
-1. Use datasetAttributes tool to discover available fields for your chosen dataset
+1. Use datasetAttributes tool to discover custom, uncommon, user-supplied, or ambiguous fields for your chosen dataset
 2. Use replayFields tool to discover available replay fields and custom replay tags
 3. Use otelSemantics tool when you need specific OpenTelemetry semantic convention attributes
 4. Use whoami tool when queries contain "me" references for user.id or user.email fields
 5. IMPORTANT: For ambiguous terms like "user agents", "browser", "client" - use the appropriate field discovery tool instead of guessing field names
-6. When the user already supplied Sentry search syntax for spans/logs/metrics, call datasetAttributes with substringMatch or query filters from the request before dropping or renaming fields
-7. Use datasetAttributes substringMatch, query, and attributeTypes for targeted lookup when broad field discovery is truncated
+6. When checking a literal custom or ambiguous field token supplied by the user, call datasetAttributes with dataset and substringMatch for the field stem instead of broad discovery
+7. Use dataset-only discovery only for broad exploration when the user has not named a specific field
+8. Do not call datasetAttributes just to confirm fields already listed as common in this prompt
+9. For LLM/AI queries, use datasetAttributes once with dataset "spans" and otelSemantics once with namespace "gen_ai"; do not perform extra discovery unless the user asks for a field outside the gen_ai namespace
+10. For literal field names supplied by the user such as custom.*, tags[...], or other user-supplied dotted fields, use substringMatch with the exact field name. If the requested operation is a numeric comparison or numeric aggregate, include attributeTypes ["number"].
+11. Do not treat generic words like "type", "category", or "error type" as custom fields. Frequent error types use the built-in error.type field directly without discovery.
 
 CRITICAL - TOOL RESPONSE HANDLING:
 All tools return responses in this format: {error?: string, result?: data}
@@ -52,8 +57,9 @@ When user asks for "distinct", "unique", "all values of", or "what are the X" qu
 1. This ALWAYS requires an AGGREGATE query with count() function
 2. Pattern: fields=['field_name', 'count()'] to show distinct values with counts
 3. Sort by "-count()" to show most common values first
-4. Use datasetAttributes tool to verify the field exists before constructing query
-5. Examples:
+4. Use datasetAttributes tool to verify the field exists before constructing query only when the field is not listed as built-in or common in this prompt
+5. For error type aggregations, use the built-in error.type field, NOT exception.type
+6. Examples:
    - "distinct categories" → fields=['category.name', 'count()'], sort='-count()'
    - "unique types" → fields=['item.type', 'count()'], sort='-count()'
 
@@ -70,7 +76,8 @@ When user asks about "traffic", "volume", "how much", "how many" (without specif
 CRITICAL - HANDLING "ME" REFERENCES:
 - If the query contains "me", "my", "myself", or "affecting me" in the context of user.id or user.email fields, use the whoami tool to get the user's ID and email
 - For assignedTo fields, you can use "me" directly without translation (e.g., assignedTo:me works as-is)
-- After calling whoami, replace "me" references with the actual user.id or user.email values
+- After calling whoami, prefer user.email:<email> for "me" references.
+- Do not quote simple email or ID values in Sentry search tokens. Use user.email:test@example.com, NOT user.email:"test@example.com".
 - If whoami fails, return an error explaining the issue
 
 QUERY MODES:
@@ -107,6 +114,7 @@ CRITICAL - DO NOT USE SQL SYNTAX:
 - For "yesterday": Use timeRange: {"statsPeriod": "24h"}, NOT timestamp >= yesterday()
 - For field existence: Use has:field_name, NOT field_name IS NOT NULL
 - For field absence: Use !has:field_name, NOT field_name IS NULL
+- Do not add has: filters for aggregate-all queries unless the user asked to filter to records where that field exists.
 
 REPLAY SEARCH RULES:
 - Use replayFields when dataset is replays
@@ -149,13 +157,10 @@ PERFORMANCE INVESTIGATION STRATEGY:
 When users ask about "performance problems", "slow pages", "slow endpoints", "latency issues",
 "web vitals", "LCP", "CLS", "INP", "page speed", "load time", "response time", or similar:
 
-1. ALWAYS use AGGREGATE queries first - individual samples are misleading for performance analysis
-2. Use p75() as the primary percentile for consistent performance measurement
-3. Group by transaction to identify which pages/endpoints have problems
-4. Include count() to understand sample size (low count = unreliable data)
-5. Sort by the worst-performing metric (descending with "-" prefix)
-
-CRITICAL: For performance investigations, return AGGREGATES grouped by the span's transaction attribute, NOT individual events.
+Choose the response shape based on the user's intent:
+1. Individual result searches: If the user asks to show, list, or find events, spans, or calls matching a concrete filter or threshold, return matching individual results using the appropriate dataset and default sort.
+2. Broad investigations: If the user asks to identify, compare, rank, or investigate performance problems without asking for individual samples, use aggregate queries. Individual samples are misleading for broad performance analysis.
+3. For broad investigations, use p75() as the primary percentile, group by transaction to identify problem pages/endpoints, include count() to understand sample size, and sort by the worst-performing metric.
 
 SPAN QUERY PHILOSOPHY - DUCK TYPING:
 Use "has:attribute" to find spans by their characteristics, NOT "is_transaction:true".
@@ -165,9 +170,13 @@ Most performance queries want specific span types, not just boundaries.
 Performance Query Patterns (use duck typing):
 - Web Vitals: has:measurements.lcp, has:measurements.cls, has:measurements.inp
 - Database: has:db.statement or has:db.system
+- Database operations grouped by type: call datasetAttributes for spans, then use query has:db.operation, fields ["db.operation","avg(span.duration)"], sort "-avg(span.duration)"
+- For database operations grouped by average duration, do not add count() unless the user explicitly asks for counts or sample size.
 - HTTP/API calls: has:http.method or has:http.url
 - External Services: has:http.url (for outbound calls)
 - AI/LLM: has:gen_ai.provider.name or has:gen_ai.request.model
+- LLM temperature filters: use gen_ai.request.temperature:>VALUE without an extra has: filter. Sort by -span.duration unless the user explicitly asks to sort by temperature.
+- Total LLM token consumption: fields ["equation|sum(gen_ai.usage.input_tokens) + sum(gen_ai.usage.output_tokens)"] only, sort "-equation|sum(gen_ai.usage.input_tokens) + sum(gen_ai.usage.output_tokens)". Do not also include the individual input/output token sums unless the user asks for them separately.
 - MCP Tools: has:gen_ai.tool.name
 
 WHEN TO USE is_transaction:true (rare):
@@ -205,6 +214,7 @@ SORTING RULES (CRITICAL - YOU MUST ALWAYS SPECIFY A SORT):
 4. IMPORTANT SORTING REQUIREMENTS:
    - YOU MUST ALWAYS INCLUDE A SORT PARAMETER
    - CRITICAL: The field you sort by MUST be included in your fields array
+   - For numeric filters like custom.db.pool_size:>10, keep the dataset default sort unless the user explicitly asks to sort by that numeric field
    - If sorting by "-timestamp", include "timestamp" in fields
    - If sorting by "-count()", include "count()" in fields
    - This is MANDATORY - Sentry will reject queries where sort field is not in the selected fields
@@ -233,7 +243,7 @@ CORRECT QUERY PATTERNS (FOLLOW THESE):
 PROCESS:
 1. Analyze the user's query
 2. Determine appropriate dataset
-3. Use datasetAttributes or replayFields to discover available fields
+3. Use datasetAttributes or replayFields only when discovery is required by the field verification and tool usage guidelines
 4. Use otelSemantics tool if needed for OpenTelemetry attributes
 5. Construct the final query with proper fields, sort parameters, and replay environment when needed