diff --git a/README.md b/README.md index 600dfb9a..a01d1c51 100644 --- a/README.md +++ b/README.md @@ -265,6 +265,18 @@ langsmith evaluator upload evals.py \ # Delete an evaluator langsmith evaluator delete accuracy --yes + +# Create an LLM-as-judge evaluator (--model-config is always required) +# model.json: copy the structured.model block from an existing evaluator or the UI. +langsmith evaluator create-llm \ + --name relevance --project my-app \ + --prompt prompt.json --schema schema.json --model-config model.json \ + --variable-mapping '{"input":"input.question","output":"output.answer"}' + +# Or reference an existing Prompt Hub commit (--hub-ref replaces --prompt and --schema) +langsmith evaluator create-llm \ + --name relevance --project my-app \ + --hub-ref my-org/relevance:latest --model-config model.json ``` ### `experiment` — Query experiment results diff --git a/internal/cmd/evaluator.go b/internal/cmd/evaluator.go index b936e723..d48c1e59 100644 --- a/internal/cmd/evaluator.go +++ b/internal/cmd/evaluator.go @@ -20,9 +20,10 @@ func newEvaluatorCmd() *cobra.Command { Long: `Manage online and offline evaluator rules. Evaluators automatically score runs against a dataset (offline) or a project -(online). Two evaluator types are supported: +(online). Supported evaluator types: - upload Code evaluator from a Python or JavaScript/TypeScript file + upload Code evaluator from a Python or JavaScript/TypeScript file + create-llm LLM-as-judge evaluator (--model-config required; prompt via files or --hub-ref) Examples: langsmith evaluator list @@ -31,12 +32,14 @@ Examples: langsmith evaluator get accuracy --session-id langsmith evaluator upload eval.py --name accuracy --function check_accuracy --dataset my-eval-set langsmith evaluator upload eval.ts --name accuracy --function checkAccuracy --dataset my-eval-set + langsmith evaluator create-llm --name relevance --project my-app --prompt prompt.json --schema schema.json --model-config model.json langsmith evaluator delete accuracy --yes`, } cmd.AddCommand(newEvaluatorGetCmd()) cmd.AddCommand(newEvaluatorListCmd()) cmd.AddCommand(newEvaluatorUploadCmd()) + cmd.AddCommand(newEvaluatorCreateLLMCmd()) cmd.AddCommand(newEvaluatorDeleteCmd()) return cmd } @@ -196,6 +199,7 @@ func newEvaluatorUploadCmd() *cobra.Command { targetDataset string targetProject string samplingRate float64 + traceFilter string replace bool yes bool ) @@ -207,19 +211,15 @@ func newEvaluatorUploadCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { evaluatorFile := args[0] - if targetDataset == "" && targetProject == "" { - output.OutputJSON(map[string]any{ - "error": "Must specify --dataset or --project (global evaluators not supported)", - }, "") + if err := validateEvaluatorTargetFlags(targetDataset, targetProject); err != nil { + output.OutputJSON(map[string]any{"error": err.Error()}, "") return } c := MustGetClient() ctx := context.Background() - // Resolve targets var datasetID, projectID string - if targetDataset != "" { ds, err := resolveDataset(ctx, c, targetDataset) if err != nil { @@ -227,7 +227,6 @@ func newEvaluatorUploadCmd() *cobra.Command { } datasetID = ds.ID } - if targetProject != "" { sid, err := c.ResolveSessionID(ctx, targetProject) if err != nil { @@ -236,34 +235,6 @@ func newEvaluatorUploadCmd() *cobra.Command { projectID = sid } - // Check for existing evaluator - rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) - if err != nil { - ExitErrorf("checking existing evaluators: %v", err) - } - - existing := findEvaluator(*rules, name, datasetID, projectID) - if existing != nil { - if !replace { - output.OutputJSON(map[string]any{ - "error": fmt.Sprintf("Evaluator '%s' already exists (use --replace to overwrite)", name), - "id": existing.ID, - }, "") - return - } - if !yes { - fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) - var confirm string - _, _ = fmt.Scanln(&confirm) - if strings.ToLower(confirm) != "y" { - ExitError("aborted") - } - } - if err := c.RawDelete(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), nil); err != nil { - ExitErrorf("deleting existing evaluator: %v", err) - } - } - // Read and prepare function source source, err := os.ReadFile(evaluatorFile) if err != nil { @@ -292,7 +263,6 @@ func newEvaluatorUploadCmd() *cobra.Command { sourceStr = renameJSFunction(sourceStr, funcName) } - // Build payload payload := map[string]any{ "display_name": name, "sampling_rate": samplingRate, @@ -302,29 +272,60 @@ func newEvaluatorUploadCmd() *cobra.Command { {"code": sourceStr, "language": language}, }, } - if datasetID != "" { payload["dataset_id"] = datasetID } if projectID != "" { payload["session_id"] = projectID } + if traceFilter != "" { + payload["trace_filter"] = traceFilter + } + + // Prepare the new evaluator before touching any existing one. If this is + // a replacement, the old evaluator stays in place until the new version is ready. + rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) + if err != nil { + ExitErrorf("checking existing evaluators: %v", err) + } + + existing := findEvaluator(*rules, name, datasetID, projectID) + if existing != nil { + if !replace { + output.OutputJSON(map[string]any{ + "error": fmt.Sprintf("Evaluator '%s' already exists (use --replace to overwrite)", name), + "id": existing.ID, + }, "") + return + } + if !yes { + fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) + var confirm string + _, _ = fmt.Scanln(&confirm) + if strings.ToLower(confirm) != "y" { + ExitError("aborted") + } + } + } var result map[string]any - if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { + if existing != nil { + if err := c.RawPatch(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), payload, &result); err != nil { + ExitErrorf("replacing evaluator: %v", err) + } + } else if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { ExitErrorf("uploading evaluator: %v", err) } - target := "project" + targetLabel := "project" if datasetID != "" { - target = "dataset" + targetLabel = "dataset" } - output.OutputJSON(map[string]any{ "status": "uploaded", "id": result["id"], "name": name, - "target": target, + "target": targetLabel, }, "") }, } @@ -334,6 +335,7 @@ func newEvaluatorUploadCmd() *cobra.Command { cmd.Flags().StringVar(&targetDataset, "dataset", "", "Target dataset name (offline evaluator)") cmd.Flags().StringVar(&targetProject, "project", "", "Target project name (online evaluator)") cmd.Flags().Float64Var(&samplingRate, "sampling-rate", 1.0, "Fraction of runs to evaluate (0.0-1.0)") + cmd.Flags().StringVar(&traceFilter, "trace-filter", "", "Filter expression for which runs to evaluate") cmd.Flags().BoolVar(&replace, "replace", false, "Replace existing evaluator with same name") cmd.Flags().BoolVar(&yes, "yes", false, "Skip confirmation prompt when replacing") _ = cmd.MarkFlagRequired("name") @@ -342,6 +344,104 @@ func newEvaluatorUploadCmd() *cobra.Command { return cmd } +// Registers create-llm for LLM-as-judge evaluators. +func newEvaluatorCreateLLMCmd() *cobra.Command { + var ( + name string + targetDataset string + targetProject string + samplingRate float64 + traceFilter string + hubRef string + promptPath string + schemaPath string + modelConfigPath string + variableMapping string + replace bool + yes bool + ) + + cmd := &cobra.Command{ + Use: "create-llm", + Short: "Create an LLM-as-judge evaluator rule", + Long: `Create an LLM-as-judge run rule. --model-config is always required. + +Provide the judge prompt inline with --prompt and --schema, or point at Prompt Hub +with --hub-ref (which replaces --prompt and --schema). The model is configured separately. + +Examples: + langsmith evaluator create-llm --name relevance --project my-app \ + --prompt prompt.json --schema schema.json --model-config model.json + langsmith evaluator create-llm --name relevance --project my-app \ + --hub-ref my-org/relevance:latest --model-config model.json`, + Run: func(cmd *cobra.Command, args []string) { + c := MustGetClient() + ctx := context.Background() + + target, err := resolveLLMEvaluatorTarget(ctx, c, targetDataset, targetProject) + if err != nil { + ExitErrorf("%v", err) + } + mapping, err := parseVariableMapping(variableMapping) + if err != nil { + ExitErrorf("%v", err) + } + payload, err := buildLLMEvaluatorPayload( + name, target, samplingRate, traceFilter, hubRef, + promptPath, schemaPath, modelConfigPath, mapping, + ) + if err != nil { + ExitErrorf("%v", err) + } + existing, err := findLLMEvaluatorForCreate(ctx, c, name, target, replace, yes) + if err != nil { + if err.Error() == "aborted" { + ExitError("aborted") + } + if existing != nil { + output.OutputJSON(map[string]any{"error": err.Error(), "id": existing.ID}, "") + return + } + ExitErrorf("%v", err) + } + + var result map[string]any + if existing != nil { + if err := c.RawPatch(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), payload, &result); err != nil { + ExitErrorf("replacing LLM evaluator: %v", err) + } + } else if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { + ExitErrorf("creating LLM evaluator: %v", err) + } + targetLabel := "project" + if target.datasetID != "" { + targetLabel = "dataset" + } + output.OutputJSON(map[string]any{ + "status": "created", "type": "llm", + "id": result["id"], "name": name, "target": targetLabel, + }, "") + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Display name (required)") + cmd.Flags().StringVar(&targetDataset, "dataset", "", "Target dataset name") + cmd.Flags().StringVar(&targetProject, "project", "", "Target project name") + cmd.Flags().Float64Var(&samplingRate, "sampling-rate", 1.0, "Fraction of runs to evaluate (0.0-1.0)") + cmd.Flags().StringVar(&traceFilter, "trace-filter", "", "Filter expression for which runs to evaluate") + cmd.Flags().StringVar(&hubRef, "hub-ref", "", "Prompt Hub reference; replaces --prompt and --schema (e.g. my-org/prompt:latest)") + cmd.Flags().StringVar(&promptPath, "prompt", "", "Prompt JSON file ([[role,content],...] or [{role,content},...]); omit if --hub-ref is set") + cmd.Flags().StringVar(&schemaPath, "schema", "", "JSON schema file for structured output; omit if --hub-ref is set") + cmd.Flags().StringVar(&modelConfigPath, "model-config", "", "Serialized LangChain model JSON (required; copy from UI or GET /runs/rules)") + cmd.Flags().StringVar(&variableMapping, "variable-mapping", "", `Map prompt vars to trace paths (JSON or @file.json)`) + cmd.Flags().BoolVar(&replace, "replace", false, "Replace existing evaluator with same name") + cmd.Flags().BoolVar(&yes, "yes", false, "Skip confirmation when replacing") + _ = cmd.MarkFlagRequired("name") + _ = cmd.MarkFlagRequired("model-config") + + return cmd +} + func newEvaluatorDeleteCmd() *cobra.Command { var yes bool diff --git a/internal/cmd/evaluator_llm.go b/internal/cmd/evaluator_llm.go new file mode 100644 index 00000000..6b734439 --- /dev/null +++ b/internal/cmd/evaluator_llm.go @@ -0,0 +1,213 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/langchain-ai/langsmith-cli/internal/client" + langsmith "github.com/langchain-ai/langsmith-go" +) + +type llmEvaluatorTarget struct { + datasetID, projectID string +} + +func loadJSONFile(path string, dest any) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + if err := json.Unmarshal(data, dest); err != nil { + return fmt.Errorf("parsing JSON in %s: %w", path, err) + } + return nil +} + +// Accepts a plain schema file or one already saved from the LangSmith UI. +func loadEvaluatorSchema(path string) (map[string]any, error) { + var raw map[string]any + if err := loadJSONFile(path, &raw); err != nil { + return nil, err + } + if _, ok := raw["parameters"]; ok { + return raw, nil + } + return map[string]any{"name": "eval", "description": "", "parameters": raw}, nil +} + +func validatePromptMessagePair(i int, path string, msg []string) error { + if len(msg) != 2 || msg[0] == "" || msg[1] == "" { + return fmt.Errorf("prompt message %d in %s must be [role, content] with non-empty values", i, path) + } + return nil +} + +// Loads the judge's system/user messages from a prompt file. +func loadPromptMessagesFromJSON(path string, data []byte) ([][]string, error) { + var pairs [][]string + if err := json.Unmarshal(data, &pairs); err == nil && len(pairs) > 0 { + for i, msg := range pairs { + if err := validatePromptMessagePair(i, path, msg); err != nil { + return nil, err + } + } + return pairs, nil + } + var objects []struct { + Role, Content string + } + if err := json.Unmarshal(data, &objects); err != nil || len(objects) == 0 { + return nil, fmt.Errorf( + "prompt file %s: expected [[role,content],...] or [{role,content},...]", path, + ) + } + pairs = make([][]string, len(objects)) + for i, obj := range objects { + if err := validatePromptMessagePair(i, path, []string{obj.Role, obj.Content}); err != nil { + return nil, err + } + pairs[i] = []string{obj.Role, obj.Content} + } + return pairs, nil +} + +// Maps {{placeholders}} in the prompt to fields on each trace. +func parseVariableMapping(raw string) (map[string]string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } + if strings.HasPrefix(raw, "@") { + var mapping map[string]string + if err := loadJSONFile(raw[1:], &mapping); err != nil { + return nil, err + } + return mapping, nil + } + var mapping map[string]string + if err := json.Unmarshal([]byte(raw), &mapping); err != nil { + return nil, fmt.Errorf("parsing --variable-mapping: %w", err) + } + return mapping, nil +} + +func validateEvaluatorTargetFlags(dataset, project string) error { + if dataset == "" && project == "" { + return fmt.Errorf("must specify --dataset or --project (global evaluators not supported)") + } + if dataset != "" && project != "" { + return fmt.Errorf("cannot specify both --dataset and --project") + } + return nil +} + +// Finds the dataset or project this evaluator should run on. +func resolveLLMEvaluatorTarget(ctx context.Context, c *client.Client, dataset, project string) (llmEvaluatorTarget, error) { + if err := validateEvaluatorTargetFlags(dataset, project); err != nil { + return llmEvaluatorTarget{}, err + } + var target llmEvaluatorTarget + if dataset != "" { + ds, err := resolveDataset(ctx, c, dataset) + if err != nil { + return llmEvaluatorTarget{}, err + } + target.datasetID = ds.ID + } + if project != "" { + sid, err := c.ResolveSessionID(ctx, project) + if err != nil { + return llmEvaluatorTarget{}, err + } + target.projectID = sid + } + return target, nil +} + +// Finds an existing evaluator with the same name on this dataset or project. +// When the evaluator already exists and --replace is not set, returns the existing +// rule alongside the error so callers can reuse it without another list call. +func findLLMEvaluatorForCreate(ctx context.Context, c *client.Client, name string, target llmEvaluatorTarget, replace, yes bool) (*langsmith.Evaluator, error) { + rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) + if err != nil { + return nil, fmt.Errorf("checking existing evaluators: %w", err) + } + existing := findEvaluator(*rules, name, target.datasetID, target.projectID) + if existing == nil { + return nil, nil + } + if !replace { + return existing, fmt.Errorf("evaluator %q already exists (use --replace to overwrite)", name) + } + if !yes { + fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) + var confirm string + _, _ = fmt.Scanln(&confirm) + if strings.ToLower(confirm) != "y" { + return nil, fmt.Errorf("aborted") + } + } + return existing, nil +} + +// Packages prompt, schema, model, and targeting into the create-evaluator request. +func buildLLMEvaluatorPayload( + name string, + target llmEvaluatorTarget, + samplingRate float64, + traceFilter, hubRef, promptPath, schemaPath, modelConfigPath string, + variableMapping map[string]string, +) (map[string]any, error) { + if modelConfigPath == "" { + return nil, fmt.Errorf("--model-config is required") + } + var model map[string]any + if err := loadJSONFile(modelConfigPath, &model); err != nil { + return nil, err + } + + structured := map[string]any{"model": model} + if hubRef != "" { + structured["hub_ref"] = hubRef + } else { + if promptPath == "" || schemaPath == "" { + return nil, fmt.Errorf("--prompt and --schema are required unless --hub-ref is set") + } + data, err := os.ReadFile(promptPath) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", promptPath, err) + } + messages, err := loadPromptMessagesFromJSON(promptPath, data) + if err != nil { + return nil, err + } + schema, err := loadEvaluatorSchema(schemaPath) + if err != nil { + return nil, err + } + structured["prompt"] = messages + structured["schema"] = schema + structured["template_format"] = "mustache" + } + if len(variableMapping) > 0 { + structured["variable_mapping"] = variableMapping + } + payload := map[string]any{ + "display_name": name, "sampling_rate": samplingRate, "is_enabled": true, + "include_extended_stats": false, + "evaluators": []map[string]any{{"structured": structured}}, + } + if traceFilter != "" { + payload["trace_filter"] = traceFilter + } + if target.datasetID != "" { + payload["dataset_id"] = target.datasetID + } + if target.projectID != "" { + payload["session_id"] = target.projectID + } + return payload, nil +} diff --git a/internal/cmd/evaluator_test.go b/internal/cmd/evaluator_test.go index 49368212..ae5c4823 100644 --- a/internal/cmd/evaluator_test.go +++ b/internal/cmd/evaluator_test.go @@ -3,6 +3,7 @@ package cmd import ( "encoding/json" "net/http" + "os" "strings" "testing" @@ -366,11 +367,100 @@ func TestFindEvaluator_NameMatchButNoTarget(t *testing.T) { } } +func TestValidateEvaluatorTargetFlags(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + dataset string + project string + wantErr string + }{ + {name: "requires one target", wantErr: "must specify"}, + {name: "rejects both targets", dataset: "ds", project: "proj", wantErr: "cannot specify both"}, + {name: "dataset only", dataset: "ds"}, + {name: "project only", project: "proj"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := validateEvaluatorTargetFlags(tt.dataset, tt.project) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected %q error, got %v", tt.wantErr, err) + } + }) + } +} + +func TestLoadPromptMessagesFromJSON(t *testing.T) { + t.Parallel() + + t.Run("pair format", func(t *testing.T) { + t.Parallel() + msgs, err := loadPromptMessagesFromJSON("prompt.json", []byte( + `[["system","You are a judge."],["user","Q: {{input}}"]]`, + )) + if err != nil { + t.Fatal(err) + } + if len(msgs) != 2 || msgs[0][0] != "system" { + t.Fatalf("unexpected messages: %#v", msgs) + } + }) + + t.Run("role content objects", func(t *testing.T) { + t.Parallel() + msgs, err := loadPromptMessagesFromJSON("prompt.json", []byte( + `[{"role":"system","content":"You are a judge."},{"role":"user","content":"Q: {{input}}"}]`, + )) + if err != nil { + t.Fatal(err) + } + if len(msgs) != 2 || msgs[1][1] != "Q: {{input}}" { + t.Fatalf("unexpected messages: %#v", msgs) + } + }) + + t.Run("invalid format", func(t *testing.T) { + t.Parallel() + _, err := loadPromptMessagesFromJSON("prompt.json", []byte(`{"messages":[]}`)) + if err == nil || !strings.Contains(err.Error(), "expected [[role,content]") { + t.Fatalf("expected format hint, got %v", err) + } + }) + + t.Run("rejects single-element pair", func(t *testing.T) { + t.Parallel() + _, err := loadPromptMessagesFromJSON("prompt.json", []byte(`[["system"]]`)) + if err == nil || !strings.Contains(err.Error(), "must be [role, content]") { + t.Fatalf("expected length error, got %v", err) + } + }) +} + +func TestBuildLLMEvaluatorPayload_requiresModelConfig(t *testing.T) { + t.Parallel() + + _, err := buildLLMEvaluatorPayload( + "relevance", llmEvaluatorTarget{projectID: "proj-1"}, + 1.0, "", "", "prompt.json", "schema.json", "", nil, + ) + if err == nil || !strings.Contains(err.Error(), "--model-config is required") { + t.Fatalf("expected model-config error, got %v", err) + } +} + // ==================== Command structure tests ==================== func TestEvaluatorCmd_Subcommands(t *testing.T) { cmd := newEvaluatorCmd() - expected := map[string]bool{"get": false, "list": false, "upload": false, "delete": false} + expected := map[string]bool{"get": false, "list": false, "upload": false, "create-llm": false, "delete": false} for _, sub := range cmd.Commands() { if _, ok := expected[sub.Name()]; ok { expected[sub.Name()] = true @@ -444,6 +534,7 @@ func TestEvaluatorUploadCmd_Flags(t *testing.T) { "dataset": "", "project": "", "sampling-rate": "1", + "trace-filter": "", "replace": "false", "yes": "false", } @@ -639,6 +730,194 @@ func TestEvaluatorListCmd_VerifiesAPIKeyHeader(t *testing.T) { } } +func TestEvaluatorUploadReplacePatchesExistingCodeEvaluator(t *testing.T) { + evaluatorFile := t.TempDir() + "/eval.py" + if err := os.WriteFile( + evaluatorFile, + []byte("def check_accuracy(run, example):\n return {\"score\": 1}\n"), + 0o600, + ); err != nil { + t.Fatal(err) + } + + var sawDelete bool + var patchBody map[string]any + ts := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/api/v1/sessions" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]map[string]any{ + {"id": "project-1", "name": "my-project"}, + }) + case r.URL.Path == "/api/v1/runs/rules" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]testRule{ + { + ID: "existing-rule", + DisplayName: "accuracy", + SamplingRate: 0.25, + IsEnabled: true, + SessionID: "project-1", + CodeEvaluators: []testCodeEval{ + {Code: "def perform_eval(run, example):\n return {\"score\": 0}", Language: "python"}, + }, + }, + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "PATCH": + if err := json.NewDecoder(r.Body).Decode(&patchBody); err != nil { + t.Fatalf("decoding patch body: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "existing-rule", + "display_name": "accuracy", + "session_id": "project-1", + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "DELETE": + sawDelete = true + http.Error(w, "delete should not be called", http.StatusInternalServerError) + default: + http.Error(w, "not found", http.StatusNotFound) + } + }) + + cleanup := setupTestEnv(t, ts.URL) + defer cleanup() + flagOutputFormat = "json" + + out := captureStdout(t, func() { + cmd := newEvaluatorUploadCmd() + _ = cmd.Flags().Set("name", "accuracy") + _ = cmd.Flags().Set("function", "check_accuracy") + _ = cmd.Flags().Set("project", "my-project") + _ = cmd.Flags().Set("sampling-rate", "0.5") + _ = cmd.Flags().Set("replace", "true") + _ = cmd.Flags().Set("yes", "true") + cmd.Run(cmd, []string{evaluatorFile}) + }) + + if sawDelete { + t.Fatal("upload --replace should patch the existing evaluator, not delete it") + } + if patchBody == nil { + t.Fatal("expected PATCH body") + } + if patchBody["display_name"] != "accuracy" { + t.Errorf("expected display_name=accuracy, got %v", patchBody["display_name"]) + } + if patchBody["session_id"] != "project-1" { + t.Errorf("expected session_id=project-1, got %v", patchBody["session_id"]) + } + if patchBody["sampling_rate"] != 0.5 { + t.Errorf("expected sampling_rate=0.5, got %v", patchBody["sampling_rate"]) + } + evaluators, ok := patchBody["code_evaluators"].([]any) + if !ok || len(evaluators) != 1 { + t.Fatalf("expected one code evaluator, got %#v", patchBody["code_evaluators"]) + } + codeEvaluator, ok := evaluators[0].(map[string]any) + if !ok { + t.Fatalf("expected code evaluator object, got %#v", evaluators[0]) + } + if codeEvaluator["language"] != "python" { + t.Errorf("expected language=python, got %v", codeEvaluator["language"]) + } + code, _ := codeEvaluator["code"].(string) + if !strings.Contains(code, "def perform_eval(") { + t.Errorf("expected uploaded code to be renamed to perform_eval, got:\n%s", code) + } + + var result map[string]any + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("failed to parse output JSON: %v\noutput: %s", err, out) + } + if result["id"] != "existing-rule" { + t.Errorf("expected output id=existing-rule, got %v", result["id"]) + } +} + +func TestEvaluatorCreateLLMReplacePatchesExistingEvaluator(t *testing.T) { + modelConfigPath := t.TempDir() + "/model.json" + if err := os.WriteFile( + modelConfigPath, + []byte(`{"type":"chat","config":{"model":"test-model"}}`), + 0o600, + ); err != nil { + t.Fatal(err) + } + + var sawDelete bool + var patchBody map[string]any + ts := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/api/v1/sessions" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]map[string]any{ + {"id": "project-1", "name": "my-project"}, + }) + case r.URL.Path == "/api/v1/runs/rules" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]testRule{ + { + ID: "existing-rule", + DisplayName: "relevance", + SamplingRate: 0.25, + IsEnabled: true, + SessionID: "project-1", + Evaluators: []testLLMEval{ + {Structured: testLLMStructured{HubRef: "my-org/old:latest"}}, + }, + }, + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "PATCH": + if err := json.NewDecoder(r.Body).Decode(&patchBody); err != nil { + t.Fatalf("decoding patch body: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "existing-rule", + "display_name": "relevance", + "session_id": "project-1", + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "DELETE": + sawDelete = true + http.Error(w, "delete should not be called", http.StatusInternalServerError) + default: + http.Error(w, "not found", http.StatusNotFound) + } + }) + + cleanup := setupTestEnv(t, ts.URL) + defer cleanup() + flagOutputFormat = "json" + + out := captureStdout(t, func() { + cmd := newEvaluatorCreateLLMCmd() + _ = cmd.Flags().Set("name", "relevance") + _ = cmd.Flags().Set("project", "my-project") + _ = cmd.Flags().Set("hub-ref", "my-org/relevance:latest") + _ = cmd.Flags().Set("model-config", modelConfigPath) + _ = cmd.Flags().Set("sampling-rate", "0.5") + _ = cmd.Flags().Set("replace", "true") + _ = cmd.Flags().Set("yes", "true") + cmd.Run(cmd, nil) + }) + + if sawDelete { + t.Fatal("create-llm --replace should patch the existing evaluator, not delete it") + } + if patchBody == nil { + t.Fatal("expected PATCH body") + } + if patchBody["display_name"] != "relevance" || patchBody["evaluators"] == nil { + t.Fatalf("expected LLM evaluator replacement payload, got %#v", patchBody) + } + + var result map[string]any + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("failed to parse output JSON: %v\noutput: %s", err, out) + } + if result["id"] != "existing-rule" { + t.Errorf("expected output id=existing-rule, got %v", result["id"]) + } +} + // ==================== evaluator get ==================== func TestEvaluatorGetCmd_UseField(t *testing.T) {