Skip to content

feat(evaluator): add create-llm and rework upload replacement #188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Ramon Nogueira (ramon-langchain) wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(evaluator): add create-llm and rework upload replacement #188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,18 @@ langsmith evaluator upload evals.py \

# Delete an evaluator
langsmith evaluator delete accuracy --yes

# Create an LLM-as-judge evaluator (--model-config is always required)
# model.json: copy the structured.model block from an existing evaluator or the UI.
langsmith evaluator create-llm \
--name relevance --project my-app \
--prompt prompt.json --schema schema.json --model-config model.json \
--variable-mapping '{"input":"input.question","output":"output.answer"}'

# Or reference an existing Prompt Hub commit (--hub-ref replaces --prompt and --schema)
langsmith evaluator create-llm \
--name relevance --project my-app \
--hub-ref my-org/relevance:latest --model-config model.json
```

### `experiment` — Query experiment results
Expand Down
188 changes: 144 additions & 44 deletions internal/cmd/evaluator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@ func newEvaluatorCmd() *cobra.Command {
Long: `Manage online and offline evaluator rules.

Evaluators automatically score runs against a dataset (offline) or a project
(online). Two evaluator types are supported:
(online). Supported evaluator types:

upload Code evaluator from a Python or JavaScript/TypeScript file
upload Code evaluator from a Python or JavaScript/TypeScript file
create-llm LLM-as-judge evaluator (--model-config required; prompt via files or --hub-ref)

Examples:
langsmith evaluator list
Expand All @@ -31,12 +32,14 @@ Examples:
langsmith evaluator get accuracy --session-id <session-id>
langsmith evaluator upload eval.py --name accuracy --function check_accuracy --dataset my-eval-set
langsmith evaluator upload eval.ts --name accuracy --function checkAccuracy --dataset my-eval-set
langsmith evaluator create-llm --name relevance --project my-app --prompt prompt.json --schema schema.json --model-config model.json
langsmith evaluator delete accuracy --yes`,
}

cmd.AddCommand(newEvaluatorGetCmd())
cmd.AddCommand(newEvaluatorListCmd())
cmd.AddCommand(newEvaluatorUploadCmd())
cmd.AddCommand(newEvaluatorCreateLLMCmd())
cmd.AddCommand(newEvaluatorDeleteCmd())
return cmd
}
Expand Down Expand Up @@ -196,6 +199,7 @@ func newEvaluatorUploadCmd() *cobra.Command {
targetDataset string
targetProject string
samplingRate float64
traceFilter string
replace bool
yes bool
)
Expand All @@ -207,27 +211,22 @@ func newEvaluatorUploadCmd() *cobra.Command {
Run: func(cmd *cobra.Command, args []string) {
evaluatorFile := args[0]

if targetDataset == "" && targetProject == "" {
output.OutputJSON(map[string]any{
"error": "Must specify --dataset or --project (global evaluators not supported)",
}, "")
if err := validateEvaluatorTargetFlags(targetDataset, targetProject); err != nil {
output.OutputJSON(map[string]any{"error": err.Error()}, "")
return
}

c := MustGetClient()
ctx := context.Background()

// Resolve targets
var datasetID, projectID string

if targetDataset != "" {
ds, err := resolveDataset(ctx, c, targetDataset)
if err != nil {
ExitErrorf("%v", err)
}
datasetID = ds.ID
}

if targetProject != "" {
sid, err := c.ResolveSessionID(ctx, targetProject)
if err != nil {
Expand All @@ -236,34 +235,6 @@ func newEvaluatorUploadCmd() *cobra.Command {
projectID = sid
}

// Check for existing evaluator
rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{})
if err != nil {
ExitErrorf("checking existing evaluators: %v", err)
}

existing := findEvaluator(*rules, name, datasetID, projectID)
if existing != nil {
if !replace {
output.OutputJSON(map[string]any{
"error": fmt.Sprintf("Evaluator '%s' already exists (use --replace to overwrite)", name),
"id": existing.ID,
}, "")
return
}
if !yes {
fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name)
var confirm string
_, _ = fmt.Scanln(&confirm)
if strings.ToLower(confirm) != "y" {
ExitError("aborted")
}
}
if err := c.RawDelete(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), nil); err != nil {
ExitErrorf("deleting existing evaluator: %v", err)
}
}

// Read and prepare function source
source, err := os.ReadFile(evaluatorFile)
if err != nil {
Expand Down Expand Up @@ -292,7 +263,6 @@ func newEvaluatorUploadCmd() *cobra.Command {
sourceStr = renameJSFunction(sourceStr, funcName)
}

// Build payload
payload := map[string]any{
"display_name": name,
"sampling_rate": samplingRate,
Expand All @@ -302,29 +272,60 @@ func newEvaluatorUploadCmd() *cobra.Command {
{"code": sourceStr, "language": language},
},
}

if datasetID != "" {
payload["dataset_id"] = datasetID
}
if projectID != "" {
payload["session_id"] = projectID
}
if traceFilter != "" {
payload["trace_filter"] = traceFilter
}

// Prepare the new evaluator before touching any existing one. If this is
// a replacement, the old evaluator stays in place until the new version is ready.
rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{})
if err != nil {
ExitErrorf("checking existing evaluators: %v", err)
}

existing := findEvaluator(*rules, name, datasetID, projectID)
if existing != nil {
if !replace {
output.OutputJSON(map[string]any{
"error": fmt.Sprintf("Evaluator '%s' already exists (use --replace to overwrite)", name),
"id": existing.ID,
}, "")
return
}
if !yes {
fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name)
var confirm string
_, _ = fmt.Scanln(&confirm)
if strings.ToLower(confirm) != "y" {
ExitError("aborted")
}
}
}

var result map[string]any
if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil {
if existing != nil {
if err := c.RawPatch(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), payload, &result); err != nil {
ExitErrorf("replacing evaluator: %v", err)
}
} else if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil {
Comment on lines 311 to +316

@devin-ai-integration devin-ai-integration Bot Jul 20, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Replacing an evaluator with a different type can leave stale scoring logic attached

When replacing an existing evaluator that is a different type than the new one (RawPatch at internal/cmd/evaluator.go:313 and internal/cmd/evaluator.go:410), the update only sends the new type's block and never clears the old one, so a rule can end up carrying both the old and new scoring logic.
Impact: A user who replaces a code evaluator with an LLM one (or vice versa) under the same name/target may unknowingly keep the old evaluator running alongside the new one.

PATCH merge vs. previous full-replacement behavior

The previous implementation deleted the existing rule and created a fresh one, guaranteeing the rule contained exactly the new evaluator type. The reworked flow issues a PATCH to /runs/rules/{id} with a payload that includes only code_evaluators (upload, internal/cmd/evaluator.go:299-301) or only evaluators (create-llm, internal/cmd/evaluator_llm.go:201). If the backend PATCH treats absent fields as "leave unchanged" (typical PATCH semantics for list fields), then replacing a code evaluator with an LLM evaluator (or vice versa) under the same name+target would leave the previous type's block in place, producing a rule with both code_evaluators and evaluators. findEvaluator (internal/cmd/evaluator.go:639-652) matches purely on name + dataset/project, so it can match an existing rule of the opposite type. Same-type replacement is fine because the array is overwritten wholesale. Whether this manifests depends on backend PATCH semantics, so it warrants verification.

Prompt for agents 
The upload and create-llm commands now replace an existing evaluator via PATCH /runs/rules/{id} instead of the previous delete-then-create. The PATCH payload for upload includes only code_evaluators, and for create-llm includes only evaluators. findEvaluator matches an existing rule solely by display name plus dataset/project, so it can match a rule of the OPPOSITE evaluator type. If the backend PATCH leaves fields absent from the payload unchanged, then replacing a code evaluator with an LLM evaluator (or vice versa) would leave the old evaluator block attached, producing a rule that runs both. Verify the backend PATCH semantics for /runs/rules; if it does not clear omitted list fields, ensure the replacement payload explicitly clears the opposite evaluator array (e.g. send an empty code_evaluators when creating an LLM evaluator and an empty evaluators when uploading a code evaluator), or fall back to delete-then-create when the existing rule's type differs from the new one. Relevant code: internal/cmd/evaluator.go around lines 311-316 (upload) and internal/cmd/evaluator.go around lines 409-415 plus internal/cmd/evaluator_llm.go buildLLMEvaluatorPayload.
Open in Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

ExitErrorf("uploading evaluator: %v", err)
}

target := "project"
targetLabel := "project"
if datasetID != "" {
target = "dataset"
targetLabel = "dataset"
}

output.OutputJSON(map[string]any{
"status": "uploaded",
"id": result["id"],
"name": name,
"target": target,
"target": targetLabel,
}, "")
},
}
Expand All @@ -334,6 +335,7 @@ func newEvaluatorUploadCmd() *cobra.Command {
cmd.Flags().StringVar(&targetDataset, "dataset", "", "Target dataset name (offline evaluator)")
cmd.Flags().StringVar(&targetProject, "project", "", "Target project name (online evaluator)")
cmd.Flags().Float64Var(&samplingRate, "sampling-rate", 1.0, "Fraction of runs to evaluate (0.0-1.0)")
cmd.Flags().StringVar(&traceFilter, "trace-filter", "", "Filter expression for which runs to evaluate")
cmd.Flags().BoolVar(&replace, "replace", false, "Replace existing evaluator with same name")
cmd.Flags().BoolVar(&yes, "yes", false, "Skip confirmation prompt when replacing")
_ = cmd.MarkFlagRequired("name")
Expand All @@ -342,6 +344,104 @@ func newEvaluatorUploadCmd() *cobra.Command {
return cmd
}

// Registers create-llm for LLM-as-judge evaluators.
func newEvaluatorCreateLLMCmd() *cobra.Command {
var (
name string
targetDataset string
targetProject string
samplingRate float64
traceFilter string
hubRef string
promptPath string
schemaPath string
modelConfigPath string
variableMapping string
replace bool
yes bool
)

cmd := &cobra.Command{
Use: "create-llm",
Short: "Create an LLM-as-judge evaluator rule",
Long: `Create an LLM-as-judge run rule. --model-config is always required.

Provide the judge prompt inline with --prompt and --schema, or point at Prompt Hub
with --hub-ref (which replaces --prompt and --schema). The model is configured separately.

Examples:
langsmith evaluator create-llm --name relevance --project my-app \
--prompt prompt.json --schema schema.json --model-config model.json
langsmith evaluator create-llm --name relevance --project my-app \
--hub-ref my-org/relevance:latest --model-config model.json`,
Run: func(cmd *cobra.Command, args []string) {
c := MustGetClient()
ctx := context.Background()

target, err := resolveLLMEvaluatorTarget(ctx, c, targetDataset, targetProject)
if err != nil {
ExitErrorf("%v", err)
}
mapping, err := parseVariableMapping(variableMapping)
if err != nil {
ExitErrorf("%v", err)
}
payload, err := buildLLMEvaluatorPayload(
name, target, samplingRate, traceFilter, hubRef,
promptPath, schemaPath, modelConfigPath, mapping,
)
if err != nil {
ExitErrorf("%v", err)
}
existing, err := findLLMEvaluatorForCreate(ctx, c, name, target, replace, yes)
if err != nil {
if err.Error() == "aborted" {
ExitError("aborted")
}
if existing != nil {
output.OutputJSON(map[string]any{"error": err.Error(), "id": existing.ID}, "")
return
}
ExitErrorf("%v", err)
}

var result map[string]any
if existing != nil {
if err := c.RawPatch(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), payload, &result); err != nil {
ExitErrorf("replacing LLM evaluator: %v", err)
}
} else if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil {
ExitErrorf("creating LLM evaluator: %v", err)
}
targetLabel := "project"
if target.datasetID != "" {
targetLabel = "dataset"
}
output.OutputJSON(map[string]any{
"status": "created", "type": "llm",
"id": result["id"], "name": name, "target": targetLabel,
}, "")
},
}

cmd.Flags().StringVar(&name, "name", "", "Display name (required)")
cmd.Flags().StringVar(&targetDataset, "dataset", "", "Target dataset name")
cmd.Flags().StringVar(&targetProject, "project", "", "Target project name")
cmd.Flags().Float64Var(&samplingRate, "sampling-rate", 1.0, "Fraction of runs to evaluate (0.0-1.0)")
cmd.Flags().StringVar(&traceFilter, "trace-filter", "", "Filter expression for which runs to evaluate")
cmd.Flags().StringVar(&hubRef, "hub-ref", "", "Prompt Hub reference; replaces --prompt and --schema (e.g. my-org/prompt:latest)")
cmd.Flags().StringVar(&promptPath, "prompt", "", "Prompt JSON file ([[role,content],...] or [{role,content},...]); omit if --hub-ref is set")
cmd.Flags().StringVar(&schemaPath, "schema", "", "JSON schema file for structured output; omit if --hub-ref is set")
cmd.Flags().StringVar(&modelConfigPath, "model-config", "", "Serialized LangChain model JSON (required; copy from UI or GET /runs/rules)")
cmd.Flags().StringVar(&variableMapping, "variable-mapping", "", `Map prompt vars to trace paths (JSON or @file.json)`)
cmd.Flags().BoolVar(&replace, "replace", false, "Replace existing evaluator with same name")
cmd.Flags().BoolVar(&yes, "yes", false, "Skip confirmation when replacing")
_ = cmd.MarkFlagRequired("name")
_ = cmd.MarkFlagRequired("model-config")

return cmd
}

func newEvaluatorDeleteCmd() *cobra.Command {
var yes bool

Expand Down
Loading
Loading