Skip to content

Commit 765dbf5

Browse files
nima-taheri-mongodbnima-taheri-mongodb
committed
feat: some small changes and improvements
1 parent 896a7d0 commit 765dbf5

11 files changed

Lines changed: 41932 additions & 8427 deletions

‎tests/eval/dbSeed/mflix.movies-with-plot-embedding.json‎

Lines changed: 41375 additions & 8304 deletions
Large diffs are not rendered by default.

‎tests/eval/dbSeed/mflix.movies.json‎

Lines changed: 414 additions & 103 deletions
Large diffs are not rendered by default.

‎tests/eval/dbSeed/mflix.synonyms.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
[
2+
{
3+
"mappingType": "equivalent",
4+
"synonyms": [
5+
"doctor",
6+
"physician",
7+
"scientist"
8+
]
9+
},
10+
{
11+
"mappingType": "equivalent",
12+
"synonyms": [
13+
"war",
14+
"battle",
15+
"conflict"
16+
]
17+
},
18+
{
19+
"mappingType": "equivalent",
20+
"synonyms": [
21+
"experiment",
22+
"trial",
23+
"test"
24+
]
25+
}
26+
]

‎tests/eval/lib/datasetHelpers.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,17 @@ import type { Document } from "mongodb";
22
import type { DbSeedEntry, SeedIndexSpec } from "./datasetTypes.js";
33
import movies from "../dbSeed/mflix.movies.json" with { type: "json" };
44
import moviesWithPlotEmbedding from "../dbSeed/mflix.movies-with-plot-embedding.json" with { type: "json" };
5+
import synonyms from "../dbSeed/mflix.synonyms.json" with { type: "json" };
6+
import { EJSON } from "bson";
57

68
// ╭──────────────────────────────────────────────╮
79
// │ ↘️ Seeding Documents │
810
// ╰──────────────────────────────────────────────╯
911

1012
const SEED_DOCUMENTS: Record<string, Document[]> = {
11-
movies: movies as Document[],
12-
"movies-with-plot-embedding": moviesWithPlotEmbedding as Document[],
13+
movies: EJSON.deserialize(movies) as Document[],
14+
"movies-with-plot-embedding": EJSON.deserialize(moviesWithPlotEmbedding) as Document[],
15+
synonyms: EJSON.deserialize(synonyms) as Document[],
1316
};
1417

1518
/**

‎tests/eval/lib/datasetTypes.ts‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import { z } from "zod";
22
import type { EvalScorerArgs } from "braintrust";
3+
import { GetResponseTool } from "./tool/getResponse.js";
4+
import { GetConversationTool } from "./tool/getConversation.js";
5+
import { GetReferenceAnswerTool } from "./tool/getReferenceAnswer.js";
36

47
// ╭──────────────────────────────────────────────╮
58
// │ ↘️ "Input" Types in Eval │
@@ -80,13 +83,38 @@ export const RunEvalInputSchema = z
8083
/** Eval `expected`: the criteria the LLM judge grades the answer against. */
8184
export const RunEvalExpectedSchema = z
8285
.object({
83-
llm_judge: z
84-
.union([z.string(), z.array(z.string())])
86+
llm_judge: z.union([z.string()]).optional()
87+
.describe(`Provide a prompt for the LLM judge to evaluate and make assertions about:
88+
- the state of the database after the assistant completes the prompt. The judge may use any available read-only MCP tools to check and validate these assertions.
89+
- if prompt references ${GetResponseTool.keyword}, the assistant’s response for this eval case will be made available for evaluation.
90+
- if prompt references ${GetConversationTool.keyword}, the full conversation history, including tool calls and tool results for this eval case, will be accessible for evaluation.
91+
- if prompt references ${GetReferenceAnswerTool.keyword}, the reference answer as specified in the "expected.reference_answer" field will be made available for evaluation.
92+
`),
93+
reference_answer: z
94+
.string()
8595
.optional()
86-
.describe("Grading criteria (one string or several) the LLM judge checks the output against."),
96+
.describe(
97+
`The reference answer for the eval case. This provides human reviewers with a clear example of the expected answer.
98+
If the LLM judge prompt references ${GetReferenceAnswerTool.keyword}, this value will be made available to the judge for automated evaluation.`
99+
),
87100
})
88101
.describe("Expected outcome for a case, expressed as LLM-judge criteria.");
89102

103+
// ╭──────────────────────────────────────────────╮
104+
// │ ↘️ "Metadata" Types in Eval │
105+
// ╰──────────────────────────────────────────────╯
106+
107+
export const RunEvalMetadataSchema = z
108+
.object({
109+
name: z.string().describe("A short, descriptive name for this eval case."),
110+
description: z.string().describe("A brief summary explaining what this eval case tests."),
111+
category: z.string().optional().describe("The primary (level 1) taxonomy category for this eval case."),
112+
subcategory: z.string().optional().describe("The secondary (level 2) taxonomy subcategory for this eval case."),
113+
group: z.string().optional().describe("Level 3 taxonomy group for this eval case."),
114+
subGroup: z.string().optional().describe("Level 4 taxonomy subgroup for this eval case."),
115+
})
116+
.describe("Metadata for a single MongoDB agent eval case.");
117+
90118
// ╭──────────────────────────────────────────────╮
91119
// │ ↘️ "Output" Types in Eval │
92120
// ╰──────────────────────────────────────────────╯

‎tests/eval/lib/judge.ts‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export async function judgeUsingLLM(params: {
2929
model: LanguageModel;
3030
tools: ToolSet;
3131
tempDbName: string;
32-
criteria: string | string[];
32+
criteria: string;
3333
}): Promise<Verdict> {
3434
const { model, tools, criteria, tempDbName } = params;
3535
const submitScoreTool = new SubmitScoreTool();
@@ -38,11 +38,11 @@ export async function judgeUsingLLM(params: {
3838
...tools,
3939
[SubmitScoreTool.toolName]: submitScoreTool.getTool(),
4040
};
41-
const system = composeJudgeSystemPrompt(criteria, tempDbName);
41+
const system = composeJudgeSystemPrompt(tempDbName);
4242
const messages: untracedAi.ModelMessage[] = [
4343
{
4444
role: "user",
45-
content: `Verify the criteria, then call ${SubmitScoreTool.toolName} exactly once.`,
45+
content: `Verify the following criteria:\n${criteria}`,
4646
},
4747
];
4848

@@ -86,24 +86,22 @@ export async function judgeUsingLLM(params: {
8686
return submitScoreTool.getCapturedVerdict() ?? FALLBACK;
8787
}
8888

89-
function composeJudgeSystemPrompt(criteria: string | string[], tempDbName: string): string {
90-
const list = Array.isArray(criteria) ? criteria : [criteria];
89+
function composeJudgeSystemPrompt(tempDbName: string): string {
9190
return [
9291
"You are evaluating a MongoDB AI assistant on behalf of a human tester.",
93-
"Decide whether the criteria below are satisfied and produce a score from 0 to 1.",
92+
"Decide whether the provided criteria are satisfied and produce a score from 0 to 1.",
9493
"",
9594
"### Rules of Engagement",
9695
`- You MUST conclude the iteration by passing your results to the ${SubmitScoreTool.toolName} tool exactly once.`,
9796
"",
98-
"### Criteria",
99-
...list.map((c, i) => `${i + 1}. ${c}`),
100-
"",
10197
"### Tools",
10298
`- Use the MCP tools to inspect the current database state. Operate ONLY on the database named '${tempDbName}'.`,
10399
`- If a criterion references ${GetConversationTool.keyword}, call ${GetConversationTool.toolName} to read the assistant's transcript.`,
104100
`- If a criterion references ${GetResponseTool.keyword}, call ${GetResponseTool.toolName} to read the assistant's final response.`,
105101
"",
106102
"### Scoring",
107-
"- 1.0 = every criterion fully satisfied; partial credit proportional to how many are satisfied; 0.0 = none.",
103+
"If no scoring criteria are provided, score the assistant's response based on the following criteria:",
104+
"- 1.0 = every criterion fully satisfied; partial credit proportional to how many are satisfied.",
105+
"- 0.0 = none.",
108106
].join("\n");
109107
}

‎tests/eval/lib/tool/getReferenceAnswer.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import { tool, type Tool } from "ai";
2+
import { z } from "zod";
3+
4+
/**
5+
* Synthetic judge tool exposing the assistant's transcript.
6+
* Returned to the judge when a criterion references `$conversation`.
7+
*
8+
* Accepts a raw ModelMessage[] conversation; serializes it for judge inspection.
9+
*/
10+
export class GetReferenceAnswerTool {
11+
public static readonly toolName: string = "get_reference_answer";
12+
public static readonly keyword: string = "$reference_answer";
13+
#referenceAnswer: string;
14+
15+
/**
16+
* @param referenceAnswer - The reference answer for the eval case.
17+
*/
18+
constructor(referenceAnswer: string) {
19+
this.#referenceAnswer = referenceAnswer;
20+
}
21+
22+
/**
23+
* Get the synthetic judge tool.
24+
* @returns The synthetic judge tool.
25+
*/
26+
getTool(): Tool {
27+
return tool({
28+
description: `Returns the reference_answer for the eval case.
29+
Call this when the criteria references ${GetReferenceAnswerTool.keyword}.`,
30+
inputSchema: z.object({}),
31+
execute: () => ({ referenceAnswer: this.#referenceAnswer }),
32+
});
33+
}
34+
}

‎tests/eval/lib/user.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,17 @@ export async function runTask(params: {
4141
tools,
4242
stopWhen: untracedAi.stepCountIs(stepLimit),
4343
});
44+
const messages = [userMessage, ...(response.response.messages as ModelMessage[])];
45+
46+
if (response.finishReason !== "stop") {
47+
return {
48+
response: `The LLM model was stopped unexpectedly because of [${response.finishReason}] ${response.rawFinishReason}.\n${response.text}`,
49+
messages,
50+
};
51+
}
4452

4553
return {
4654
response: response.text,
47-
messages: [userMessage, ...(response.response.messages as ModelMessage[])],
55+
messages,
4856
};
4957
}

‎tests/eval/mongodb.eval.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import type { RunEvalExpected, RunEvalInput, RunEvalOutput } from "./lib/dataset
1010
import { GetConversationTool } from "./lib/tool/getConversation.js";
1111
import { GetResponseTool } from "./lib/tool/getResponse.js";
1212
import { initDataset } from "braintrust";
13+
import { GetReferenceAnswerTool } from "./lib/tool/getReferenceAnswer.js";
1314

1415
const PROJECT_NAME = "mongodb-mcp-server-evals";
1516
const DATASET_NAME = "Search";
@@ -161,6 +162,9 @@ void Eval<RunEvalInput, RunEvalOutput, RunEvalExpected, void, boolean, EvalParam
161162
...readOnlyTools,
162163
[GetConversationTool.toolName]: new GetConversationTool(messages).getTool(),
163164
[GetResponseTool.toolName]: new GetResponseTool(response).getTool(),
165+
[GetReferenceAnswerTool.toolName]: new GetReferenceAnswerTool(
166+
hooks.expected?.reference_answer ?? "<no reference answer provided>"
167+
).getTool(),
164168
},
165169
criteria,
166170
tempDbName: dbName,

‎tests/eval/scripts/evalDb.sh‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,26 @@
22
set -e
33

44
CONTAINER_NAME=mcp-eval-db
5-
IMAGE=mongodb/mongodb-atlas-local:latest
5+
IMAGE=mongodb/mongodb-atlas-local
6+
IMAGE_TAG=preview
67

78
start() {
8-
docker run -d --rm --name="$CONTAINER_NAME" --publish=27017:27017 "$IMAGE"
9+
10+
if [ -z "$VOYAGE_API_KEY" ]; then
11+
echo "⚠️ VOYAGE_API_KEY environment variable is not set"
12+
echo " without it, the local MongoDB instance will not be able to use auto-embed vector search."
13+
echo " You can get it from the Voyage AI dashboard"
14+
echo " https://www.mongodb.com/docs/voyageai/management/api-keys/"
15+
echo " "
16+
echo " @see https://hub.docker.com/r/mongodb/mongodb-atlas-local#current-experimental-features"
17+
fi
18+
19+
docker run -d \
20+
--rm \
21+
--name="$CONTAINER_NAME" \
22+
--publish=27017:27017 \
23+
--env VOYAGE_API_KEY="$VOYAGE_API_KEY" \
24+
"$IMAGE:$IMAGE_TAG"
925
docker logs -f "$CONTAINER_NAME" &
1026
LOG_PID=$!
1127
until [ "$(docker inspect -f '{{.State.Health.Status}}' "$CONTAINER_NAME")" = "healthy" ]; do

0 commit comments

Comments
 (0)