chore: add more accuracy models via Grove MCP-491

.github/workflows/accuracy-tests.yml

@@ -18,8 +18,7 @@ jobs:
       pull-requests: write
     if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'accuracy-tests')
     env:
-      MDB_OPEN_AI_API_KEY: ${{ secrets.ACCURACY_OPEN_AI_API_KEY }}
-      MDB_GEMINI_API_KEY: ${{ secrets.ACCURACY_GEMINI_API_KEY }}
+      MDB_GROVE_API_KEY: ${{ secrets.ACCURACY_GROVE_API_KEY }}
       MDB_AZURE_OPEN_AI_API_KEY: ${{ secrets.ACCURACY_AZURE_OPEN_AI_API_KEY }}
       MDB_AZURE_OPEN_AI_API_URL: ${{ vars.ACCURACY_AZURE_OPEN_AI_API_URL_V2 }}
       MDB_ACCURACY_MDB_URL: ${{ secrets.ACCURACY_MDB_CONNECTION_STRING }}

.vscode/launch.json

@@ -44,6 +44,29 @@
         "scripts/generate/index.ts"
       ],
       "cwd": "${workspaceFolder}"
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug Accuracy Tests",
+      "autoAttachChildProcesses": true,
+      "skipFiles": ["<node_internals>/**"],
+      "smartStep": true,
+      "runtimeExecutable": "pnpm",
+      "runtimeArgs": ["exec", "vitest"],
+      "args": [
+        "--config", "vitest.config.ts",
+        "--project", "accuracy",
+        "--coverage=false",
+        "--max-workers=1",
+        "--run"
+      ],
+      "cwd": "${workspaceFolder}",
+      "env": {
+        "MDB_ACCURACY_RUN_ID": "debug-run",
+        "MDB_API_CLIENT_ID": "test-atlas-client-id",
+        "MDB_API_CLIENT_SECRET": "test-atlas-client-secret"
+      }
     }
   ]
 }

package.json

@@ -101,6 +101,7 @@
   },
   "license": "Apache-2.0",
   "devDependencies": {
+    "@ai-sdk/anthropic": "^3.0.57",
     "@ai-sdk/azure": "^3.0.54",
     "@ai-sdk/google": "^3.0.64",
     "@ai-sdk/mcp": "^1.0.36",

scripts/accuracy/runAccuracyTests.sh

@@ -3,15 +3,16 @@
 export MDB_ACCURACY_RUN_ID=$(pnpm dlx uuid v4)
 
 # For providing access tokens for different LLM providers
-# export MDB_OPEN_AI_API_KEY=""
-# export MDB_GEMINI_API_KEY=""
 # export MDB_AZURE_OPEN_AI_API_KEY=""
 # export MDB_AZURE_OPEN_AI_API_URL=""
+# export MDB_GROVE_API_KEY=""
+# Optional comma-separated model allow list, for example:
+# export MDB_ACCURACY_MODEL_ALLOWLIST="gpt-5.5,claude-sonnet-4-6,Kimi-K2.6,grok-4-20-reasoning,deepseek-r1-0528"
 
 # For providing Atlas API credentials (required for Atlas tools)
 # Set dummy values for testing (allows Atlas tools to be registered for mocking)
-export MDB_MCP_API_CLIENT_ID=${MDB_MCP_API_CLIENT_ID:-"test-atlas-client-id"}
-export MDB_MCP_API_CLIENT_SECRET=${MDB_MCP_API_CLIENT_SECRET:-"test-atlas-client-secret"}
+export MDB_API_CLIENT_ID=${MDB_API_CLIENT_ID:-"test-atlas-client-id"}
+export MDB_API_CLIENT_SECRET=${MDB_API_CLIENT_SECRET:-"test-atlas-client-secret"}
 
 # For providing a mongodb based storage to store accuracy result
 # export MDB_ACCURACY_MDB_URL=""
@@ -22,7 +23,7 @@ export MDB_MCP_API_CLIENT_SECRET=${MDB_MCP_API_CLIENT_SECRET:-"test-atlas-client
 # specified in the command line. Such as:
 # pnpm run test:accuracy -- tests/accuracy/some-test.test.ts
 echo "Running accuracy tests with MDB_ACCURACY_RUN_ID '$MDB_ACCURACY_RUN_ID'"
-vitest --config vitest.config.ts --project=accuracy --coverage=false --max-workers=2 --run "$@"
+vitest --config vitest.config.ts --project=accuracy --coverage=false --max-workers="${MDB_ACCURACY_MAX_WORKERS:-8}" --run "$@"
 
 # Preserving the exit code from test run to correctly notify in the CI
 # environments when the tests fail.

tests/accuracy/createDeployment.test.ts

@@ -45,7 +45,7 @@ describeAccuracyTests([
                 parameters: {
                     deploymentName: "local-cluster",
                     loadSampleData: Matcher.anyOf(Matcher.undefined, Matcher.boolean()),
-                    imageTag: "preview",
+                    imageTag: Matcher.anyOf(Matcher.undefined, Matcher.string()),
                 },
             },
         ],
@@ -63,7 +63,7 @@ describeAccuracyTests([
                 parameters: {
                     deploymentName: "local-cluster",
                     loadSampleData: Matcher.anyOf(Matcher.undefined, Matcher.boolean()),
-                    imageTag: "preview",
+                    imageTag: Matcher.anyOf(Matcher.undefined, Matcher.string()),
                 },
             },
         ],
@@ -80,7 +80,7 @@ describeAccuracyTests([
                 toolName: "atlas-local-create-deployment",
                 parameters: {
                     loadSampleData: Matcher.anyOf(Matcher.undefined, Matcher.boolean()),
-                    imageTag: "preview",
+                    imageTag: Matcher.anyOf(Matcher.undefined, Matcher.string()),
                 },
             },
         ],
@@ -102,7 +102,7 @@ describeAccuracyTests([
                 parameters: {
                     deploymentName: "new-database",
                     loadSampleData: Matcher.anyOf(Matcher.undefined, Matcher.boolean()),
-                    imageTag: "preview",
+                    imageTag: Matcher.anyOf(Matcher.undefined, Matcher.string()),
                 },
             },
         ],
@@ -142,7 +142,7 @@ describeAccuracyTests([
                 parameters: {
                     deploymentName: "sample-cluster",
                     loadSampleData: true,
-                    imageTag: "preview",
+                    imageTag: Matcher.anyOf(Matcher.undefined, Matcher.string()),
                 },
             },
         ],
@@ -160,7 +160,7 @@ describeAccuracyTests([
                 parameters: {
                     deploymentName: "empty-cluster",
                     loadSampleData: false,
-                    imageTag: "preview",
+                    imageTag: Matcher.anyOf(Matcher.undefined, Matcher.string()),
                 },
             },
         ],

tests/accuracy/sdk/accuracyResultStorage/diskStorage.ts

@@ -28,25 +28,29 @@ export class DiskBasedResultStorage implements AccuracyResultStorage {
 
     async updateRunStatus(commitSHA: string, runId: string, status: AccuracyRunStatuses): Promise<void> {
         const resultFilePath = this.getAccuracyResultFilePath(commitSHA, runId);
-        await this.withFileLock(resultFilePath, async () => {
-            const accuracyResult = await this.getAccuracyResultWithoutLock(resultFilePath);
-            if (!accuracyResult) {
-                throw new Error("Results not found!");
-            }
-
-            await fs.writeFile(
-                resultFilePath,
-                JSON.stringify(
-                    {
-                        ...accuracyResult,
-                        runStatus: status,
-                    },
-                    null,
-                    2
-                ),
-                { encoding: "utf8" }
-            );
-        });
+        await this.withFileLock(
+            resultFilePath,
+            async () => {
+                const accuracyResult = await this.getAccuracyResultWithoutLock(resultFilePath);
+                if (!accuracyResult) {
+                    throw new Error("Results not found!");
+                }
+
+                await fs.writeFile(
+                    resultFilePath,
+                    JSON.stringify(
+                        {
+                            ...accuracyResult,
+                            runStatus: status,
+                        },
+                        null,
+                        2
+                    ),
+                    { encoding: "utf8" }
+                );
+            },
+            { ignoreNotFound: true }
+        );
 
         // This bit is important to mark the current run as the latest run for a
         // commit so that we can use that during baseline comparison.
@@ -166,12 +170,25 @@ export class DiskBasedResultStorage implements AccuracyResultStorage {
         }
     }
 
-    private async withFileLock<R>(filePath: string, callback: () => Promise<R>): Promise<R> {
+    private async withFileLock<R>(filePath: string, callback: () => Promise<R>): Promise<R>;
+    private async withFileLock<R>(
+        filePath: string,
+        callback: () => Promise<R>,
+        options: { ignoreNotFound: true }
+    ): Promise<R | undefined>;
+    private async withFileLock<R>(
+        filePath: string,
+        callback: () => Promise<R>,
+        { ignoreNotFound = false }: { ignoreNotFound?: boolean } = {}
+    ): Promise<R | undefined> {
         let releaseLock: (() => Promise<void>) | undefined;
         try {
             releaseLock = await lock(filePath, { retries: 10 });
             return await callback();
         } catch (error) {
+            if (ignoreNotFound && (error as NodeJS.ErrnoException).code === "ENOENT") {
+                return undefined;
+            }
             console.warn(`Could not acquire lock for file - ${filePath}.`, error);
             throw error;
         } finally {

tests/accuracy/sdk/accuracyTestingClient.ts

@@ -94,13 +94,17 @@ export class AccuracyTestingClient {
 
         const args = [MCP_SERVER_CLI_SCRIPT, mdbConnectionString, ...additionalArgs];
 
+        // Strip MDB_MCP_* env vars so the local MCP config (e.g.
+        // readOnly, disabledTools) doesn't bleed into the test server process.
+        const env = Object.fromEntries(
+            Object.entries(process.env).filter(([key]) => !key.startsWith("MDB_MCP_"))
+        ) as Record<string, string>;
+        env.DO_NOT_TRACK = "1";
+
         const clientTransport = new StdioClientTransport({
             command: process.execPath,
             args,
-            env: {
-                ...process.env,
-                DO_NOT_TRACK: "1",
-            },
+            env,
         });
 
         const client = await createMCPClient({