|
| 1 | +--- |
| 2 | +date: 2026-06-10 |
| 3 | +title: Manage evaluators via MCP |
| 4 | +description: Set up evaluators and evaluation rules from AI agents through the Langfuse MCP server, and create code evaluators through the unstable public API. |
| 5 | +author: Tobias Wochinger |
| 6 | +--- |
| 7 | + |
| 8 | +import { ChangelogHeader } from "@/components/changelog/ChangelogHeader"; |
| 9 | +import { Book, Code, FileCode } from "lucide-react"; |
| 10 | + |
| 11 | +<ChangelogHeader /> |
| 12 | + |
| 13 | +You can now set up and manage evaluation directly from AI agents: the Langfuse MCP server exposes evaluators and evaluation rules as tools, and the unstable public API now supports [code evaluators](/docs/evaluation/evaluation-methods/code-evaluators) in addition to LLM-as-a-Judge. |
| 14 | + |
| 15 | +This lets agents own more of the evaluation loop. For example, an agent can inspect failing traces, write a code evaluator that catches the failure pattern, and wire up an evaluation rule that runs it on live observations — all without leaving the chat. |
| 16 | + |
| 17 | +## New MCP tools |
| 18 | + |
| 19 | +<table className="[&_code]:rounded-full [&_code]:border [&_code]:border-border [&_code]:bg-muted [&_code]:px-2 [&_code]:py-1 [&_code]:text-muted-foreground"> |
| 20 | + <tbody> |
| 21 | + <tr> |
| 22 | + <td className="px-3"> |
| 23 | + <strong>Evaluators</strong> |
| 24 | + <div className="mt-2 flex flex-wrap gap-x-3 gap-y-2 leading-3"> |
| 25 | + <code>listEvaluators</code> |
| 26 | + <code>getEvaluator</code> |
| 27 | + <code>createEvaluator</code> |
| 28 | + </div> |
| 29 | + </td> |
| 30 | + </tr> |
| 31 | + <tr> |
| 32 | + <td className="px-3"> |
| 33 | + <strong>Evaluation rules</strong> |
| 34 | + <div className="mt-2 flex flex-wrap gap-x-3 gap-y-2 leading-3"> |
| 35 | + <code>listEvaluationRules</code> |
| 36 | + <code>getEvaluationRule</code> |
| 37 | + <code>createEvaluationRule</code> |
| 38 | + <code>updateEvaluationRule</code> |
| 39 | + <code>deleteEvaluationRule</code> |
| 40 | + </div> |
| 41 | + </td> |
| 42 | + </tr> |
| 43 | + </tbody> |
| 44 | +</table> |
| 45 | + |
| 46 | +## Code evaluators in the API |
| 47 | + |
| 48 | +The unstable evaluator endpoints now accept `type: "code"` to create deterministic Python or TypeScript evaluators programmatically, alongside the existing `llm_as_judge` type. Evaluation rules can reference code evaluators, and active rules are test-run before creation so broken evaluator code is rejected upfront. |
| 49 | + |
| 50 | +<Callout type="info"> |
| 51 | + These endpoints and MCP tools are **unstable** and may change while the |
| 52 | + underlying evaluation data model is being redesigned. The UI workflow remains |
| 53 | + fully supported. |
| 54 | +</Callout> |
| 55 | + |
| 56 | +## Get started |
| 57 | + |
| 58 | +<Cards num={3}> |
| 59 | + <Card |
| 60 | + title="MCP server documentation" |
| 61 | + href="/docs/api-and-data-platform/features/mcp-server" |
| 62 | + icon={<Book />} |
| 63 | + arrow |
| 64 | + /> |
| 65 | + <Card |
| 66 | + title="Code evaluators" |
| 67 | + href="/docs/evaluation/evaluation-methods/code-evaluators" |
| 68 | + icon={<FileCode />} |
| 69 | + arrow |
| 70 | + /> |
| 71 | + <Card |
| 72 | + title="Evaluators API reference" |
| 73 | + href="https://api.reference.langfuse.com/#tag/unstableevaluators" |
| 74 | + icon={<Code />} |
| 75 | + arrow |
| 76 | + /> |
| 77 | +</Cards> |
0 commit comments