fix: default parameters

Author: nima-taheri-mongodb

CONTRIBUTING.md

@@ -121,7 +121,7 @@ The Braintrust eval suite (found in `tests/eval/`) evaluates how well an LLM, wh
 #### Notable Environment Variables
 
 - `BRAINTRUST_API_KEY`: Required for all eval runs.
-- `EVAL_CONNECTION_STRING`: Overrides the MongoDB connection string (defaults to `mongodb://localhost:27017/?directConnection=true`).
+- `BT_EVAL_PARAMS_JSON`: JSON string of parameters to override the default parameters for the eval.
 - `EVAL_BASE_EXPERIMENT_NAME`: Lets you compare the current run against a specific baseline experiment. In CI, this is set automatically from the latest `main-<number>` run.
 
 ### Running in CI

tests/eval/mongodb.eval.ts

@@ -15,72 +15,61 @@ import { GetReferenceAnswerTool } from "./lib/tool/getReferenceAnswer.js";
 const PROJECT_NAME = "mongodb-mcp-server-evals";
 const DATASET_NAME = "Search";
 const AGENT_STEP_LIMIT = 10;
-const DEFAULT_MODEL = "gpt-5";
-const DEFAULT_JUDGE_MODEL = "us.anthropic.claude-sonnet-4-6";
-const DEFAULT_CONNECTION_STRING = "mongodb://localhost:27017/?directConnection=true";
 
-const DEFAULT_SYSTEM_CONTEXT =
-    'You are a MongoDB assistant operating autonomously in a single turn; the user cannot answer follow-up questions. Use the available MongoDB MCP tools to fulfill the request end-to-end. Never ask for clarification; make a reasonable decision and finish the task. If the request refers to "the collection" without naming it, discover collections with the list tools and act on the appropriate one (if there is exactly one user collection, use it). Prefer tools over guessing, and briefly confirm what you did when done.';
+const staticDefaultParameters = {
+    connectionString: "mongodb://localhost:27017/?directConnection=true",
+    model: "gpt-5",
+    judgeModel: "us.anthropic.claude-sonnet-4-6",
+    systemContext: `You are a MongoDB assistant operating autonomously in a single turn;
+the user cannot answer follow-up questions.
+Use the available MongoDB MCP tools to fulfill the request end-to-end.
+Never ask for clarification; make a reasonable decision and finish the task.
+If the request refers to "the collection" without naming it,
+discover collections with the list tools and act on the appropriate one
+(if there is exactly one user collection, use it).
+Prefer tools over guessing, and briefly confirm what you did when done.`,
+    validateReferenceAnswer: false,
+};
+
+const defaultParameters = {
+    ...staticDefaultParameters,
+    ...(process.env.BT_EVAL_PARAMS_JSON
+        ? (JSON.parse(process.env.BT_EVAL_PARAMS_JSON) as typeof staticDefaultParameters)
+        : {}),
+};
 
 const parameters = {
-    connectionString: z.string().default(DEFAULT_CONNECTION_STRING).describe("MongoDB connection string."),
+    connectionString: z.string().default(defaultParameters.connectionString).describe("MongoDB connection string."),
     model: {
         type: "model" as const,
-        default: DEFAULT_MODEL,
+        default: defaultParameters.model,
         description: "Model used by the agent under test.",
     },
     judgeModel: {
         type: "model" as const,
-        default: DEFAULT_JUDGE_MODEL,
+        default: defaultParameters.judgeModel,
         description: "Model used by the judge.",
     },
     systemContext: z
         .string()
-        .default(DEFAULT_SYSTEM_CONTEXT)
+        .default(defaultParameters.systemContext)
         .describe("System prompt prepended for the agent under test."),
+    validateReferenceAnswer: z
+        .boolean()
+        .default(defaultParameters.validateReferenceAnswer)
+        .describe(
+            "Uses `input.reference_answer` as the prompt instead of `input.prompt`; helpful for validating judge criteria against the reference answer."
+        ),
 } as unknown as EvalParameters;
 
 type ResolvedParameters = {
     connectionString: string;
     model: string;
     judgeModel: string;
     systemContext: string;
+    validateReferenceAnswer: boolean;
 };
 
-function parseEnvParams(): Record<string, unknown> {
-    const raw = process.env.BT_EVAL_PARAMS_JSON;
-    if (!raw) return {};
-    try {
-        return JSON.parse(raw) as Record<string, unknown>;
-    } catch (error) {
-        console.warn("Failed to parse BT_EVAL_PARAMS_JSON:", error);
-        return {};
-    }
-}
-
-function stringParam(value: unknown, fallback: string): string {
-    return typeof value === "string" && value.trim() ? value.trim() : fallback;
-}
-
-/**
- * Ensures parameter resolution works across all eval modes:
- * - Braintrust CLI ('bt'): supports 'BT_EVAL_PARAMS_JSON'
- * - Braintrust script ('braintrust') and tsx: do not support 'BT_EVAL_PARAMS_JSON'
- * Always resolve parameters so eval runs correctly regardless of entry point.
- */
-function resolveParameters(hooksParameters: Record<string, unknown>): ResolvedParameters {
-    const envParams = parseEnvParams();
-    const connectionString = stringParam(
-        hooksParameters.connectionString ?? envParams.connectionString ?? process.env.EVAL_CONNECTION_STRING,
-        DEFAULT_CONNECTION_STRING
-    );
-    const model = stringParam(hooksParameters.model ?? envParams.model, DEFAULT_MODEL);
-    const judgeModel = stringParam(hooksParameters.judgeModel ?? envParams.judgeModel, DEFAULT_JUDGE_MODEL);
-    const systemContext = stringParam(hooksParameters.systemContext ?? envParams.systemContext, DEFAULT_SYSTEM_CONTEXT);
-
-    return { connectionString, model, judgeModel, systemContext };
-}
-
 /**
  * Generates a unique but shorter name for the transient database.
  *
@@ -121,7 +110,7 @@ void Eval<RunEvalInput, RunEvalOutput, RunEvalExpected, void, boolean, EvalParam
         }),
         task: async (input, hooks) => {
             const aiProvider = await shared.getAiProvider();
-            const resolved = resolveParameters(hooks.parameters as Record<string, unknown>);
+            const resolved = hooks.parameters as ResolvedParameters;
             shared.registerConnectionString(resolved.connectionString);
 
             const model = aiProvider.chat(resolved.model);
@@ -139,11 +128,21 @@ void Eval<RunEvalInput, RunEvalOutput, RunEvalExpected, void, boolean, EvalParam
 
                 const tools = await mcpClient.tools();
 
+                let prompt: string;
+                if (resolved.validateReferenceAnswer) {
+                    if (!hooks.expected?.reference_answer) {
+                        throw new Error("No reference answer provided in the eval case");
+                    }
+                    prompt = `Run the reference answer: ${hooks.expected.reference_answer}`;
+                } else {
+                    prompt = input.prompt;
+                }
+
                 const { response, messages } = await runTask({
                     model,
                     systemContext: resolved.systemContext,
                     tools,
-                    prompt: input.prompt,
+                    prompt,
                     tempDbName: dbName,
                     stepLimit: AGENT_STEP_LIMIT,
                 });