diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ad39e9..1f0db1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,9 +52,6 @@ jobs: os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index 600dfb9..9f7168a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # langsmith-cli +> [!IMPORTANT] +> **Alpha** — This CLI is under active development. Commands, flags, and output schemas may change between releases. Feedback and bug reports welcome via [GitHub Issues](https://github.com/langchain-ai/langsmith-cli/issues). + An agent-first CLI for querying and managing [LangSmith](https://smith.langchain.com) resources. Built for AI coding agents (deepagents, Claude Code, Cursor, etc.) and developers who need fast, scriptable access to projects, traces, runs, datasets, evaluators, experiments, and threads. @@ -265,6 +268,18 @@ langsmith evaluator upload evals.py \ # Delete an evaluator langsmith evaluator delete accuracy --yes + +# Create an LLM-as-judge evaluator (--model-config is always required) +# model.json: copy the structured.model block from an existing evaluator or the UI. +langsmith evaluator create-llm \ + --name relevance --project my-app \ + --prompt prompt.json --schema schema.json --model-config model.json \ + --variable-mapping '{"input":"input.question","output":"output.answer"}' + +# Or reference an existing Prompt Hub commit (--hub-ref replaces --prompt and --schema) +langsmith evaluator create-llm \ + --name relevance --project my-app \ + --hub-ref my-org/relevance:latest --model-config model.json ``` ### `experiment` — Query experiment results @@ -316,19 +331,6 @@ langsmith self-update --dry-run langsmith self-update ``` -If langsmith was installed through a package manager, `self-update` won't replace the -binary in place — it points you at the right command instead: - -| Installed via | Update with | -| --- | --- | -| Homebrew | `brew upgrade langchain-ai/tap/langsmith-cli` | -| Scoop | `scoop update langsmith-cli` | -| `go install` | `go install github.com/langchain-ai/langsmith-cli/cmd/langsmith@latest` | - -Installs from the `install.sh`/`install.ps1` scripts or a direct GitHub Releases download -are updated in place as usual. Pass `--force` to update in place regardless of how -langsmith was installed. - ### `trace setup` — Trace coding agents to LangSmith Configure Claude Code or Codex to send full-content traces (prompts, responses, tool diff --git a/go.mod b/go.mod index c75e138..96ba171 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.25.0 require ( github.com/google/uuid v1.6.0 github.com/itchyny/gojq v0.12.15 - github.com/langchain-ai/langsmith-go v0.18.2 + github.com/langchain-ai/langsmith-go v0.19.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pelletier/go-toml/v2 v2.2.4 github.com/spf13/cobra v1.8.1 @@ -21,10 +21,10 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect - github.com/klauspost/compress v1.19.0 // indirect + github.com/klauspost/compress v1.18.6 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -34,18 +34,18 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel v1.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 // indirect - go.opentelemetry.io/otel/metric v1.44.0 // indirect - go.opentelemetry.io/otel/sdk v1.44.0 // indirect - go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.81.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/grpc v1.80.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 942172c..ebe81f4 100644 --- a/go.sum +++ b/go.sum @@ -17,22 +17,20 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/itchyny/gojq v0.12.15 h1:WC1Nxbx4Ifw5U2oQWACYz32JK8G9qxNtHzrvW4KEcqI= github.com/itchyny/gojq v0.12.15/go.mod h1:uWAHCbCIla1jiNxmeT5/B5mOjSdfkCq6p8vxWg+BM10= github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= -github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= -github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/langchain-ai/langsmith-go v0.18.2 h1:/Hs4KoJ2nqsPahFZz47RpXmdPZL/NkL89KC9mj6rSEA= -github.com/langchain-ai/langsmith-go v0.18.2/go.mod h1:2NXq2u8YbQkpP9BaKG7RKs0g0ZoCeqVzAZsBu7nr58Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -70,20 +68,20 @@ github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -98,12 +96,12 @@ golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= -google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/client/client.go b/internal/client/client.go index 4f13138..2da3119 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -58,17 +58,13 @@ func NewWithOptions(options Options) *Client { normalized := NormalizeURL(options.APIURL) var opts []option.RequestOption - // Auth precedence mirrors resolveClientOptions. A resolved profile is routed - // through WithProfile so an explicit selection replaces the config's - // current_profile (clearing any inherited tenant/base URL); WithProfile - // supplies the profile's own auth (API key or OAuth). A profile-less explicit - // key or bearer is applied directly. - switch { - case options.ProfileName != "": + if options.ProfileName != "" && options.APIKey == "" { opts = append(opts, langsmith.WithProfile(options.ProfileName)) - case options.APIKey != "": + } + if options.APIKey != "" { opts = append(opts, option.WithAPIKey(options.APIKey)) - case options.OAuthAccessToken != "": + } + if options.OAuthAccessToken != "" && options.ProfileName == "" && options.APIKey == "" { opts = append(opts, option.WithHeader("authorization", "Bearer "+options.OAuthAccessToken)) } // Only set base URL if not the default (the SDK reads LANGSMITH_ENDPOINT too). diff --git a/internal/client/client_test.go b/internal/client/client_test.go index 41aeb26..241b72d 100644 --- a/internal/client/client_test.go +++ b/internal/client/client_test.go @@ -136,52 +136,7 @@ func TestNewWithOptions_ProfileNameDelegatesAuthToSDK(t *testing.T) { } } -func TestNewWithOptions_APIKeyProfileRoutesThroughSDK(t *testing.T) { - var gotAPIKey string - var gotAuth string - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotAPIKey = r.Header.Get("X-API-Key") - gotAuth = r.Header.Get("Authorization") - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]any{"version": "test"}) - })) - defer ts.Close() - - path := filepath.Join(t.TempDir(), "config.json") - t.Setenv("LANGSMITH_CONFIG_FILE", path) - t.Setenv("LANGSMITH_PROFILE", "") - t.Setenv("LANGSMITH_API_KEY", "") - if err := os.WriteFile(path, []byte(`{ - "profiles": { - "prod": { - "api_key": "prod-api-key" - } - } -} -`), 0600); err != nil { - t.Fatal(err) - } - - // An api-key profile carries both ProfileName and its own APIKey (as - // resolveClientOptions produces). It routes through WithProfile: the profile's - // key is sent, no bearer. - c := NewWithOptions(Options{ - APIKey: "prod-api-key", - ProfileName: "prod", - APIURL: ts.URL, - }) - if _, err := c.SDK.Info.List(context.Background()); err != nil { - t.Fatal(err) - } - if gotAPIKey != "prod-api-key" { - t.Fatalf("expected profile api key, got %q", gotAPIKey) - } - if gotAuth != "" { - t.Fatalf("expected no bearer for an api-key profile, got %q", gotAuth) - } -} - -func TestNewWithOptions_ProfileNameTakesPrecedenceOverExplicitAPIKey(t *testing.T) { +func TestNewWithOptions_APIKeyOverridesProfileName(t *testing.T) { var gotAPIKey string var gotAuth string ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -209,10 +164,6 @@ func TestNewWithOptions_ProfileNameTakesPrecedenceOverExplicitAPIKey(t *testing. t.Fatal(err) } - // resolveClientOptions never produces this combo, but NewWithOptions is - // exported: when a caller passes an explicit APIKey alongside a ProfileName, - // the profile wins (routes through WithProfile) and the explicit key is not - // sent for SDK calls. c := NewWithOptions(Options{ APIKey: "explicit-key", ProfileName: "prod", @@ -221,11 +172,11 @@ func TestNewWithOptions_ProfileNameTakesPrecedenceOverExplicitAPIKey(t *testing. if _, err := c.SDK.Info.List(context.Background()); err != nil { t.Fatal(err) } - if gotAuth != "Bearer prod-access-token" { - t.Fatalf("expected profile OAuth bearer to win, got %q", gotAuth) + if gotAPIKey != "explicit-key" { + t.Fatalf("expected explicit API key, got %q", gotAPIKey) } - if gotAPIKey != "" { - t.Fatalf("expected explicit API key not sent for SDK calls, got %q", gotAPIKey) + if gotAuth != "" { + t.Fatalf("expected no profile bearer auth when API key wins, got %q", gotAuth) } } diff --git a/internal/cmd/evaluator.go b/internal/cmd/evaluator.go index b936e72..d48c1e5 100644 --- a/internal/cmd/evaluator.go +++ b/internal/cmd/evaluator.go @@ -20,9 +20,10 @@ func newEvaluatorCmd() *cobra.Command { Long: `Manage online and offline evaluator rules. Evaluators automatically score runs against a dataset (offline) or a project -(online). Two evaluator types are supported: +(online). Supported evaluator types: - upload Code evaluator from a Python or JavaScript/TypeScript file + upload Code evaluator from a Python or JavaScript/TypeScript file + create-llm LLM-as-judge evaluator (--model-config required; prompt via files or --hub-ref) Examples: langsmith evaluator list @@ -31,12 +32,14 @@ Examples: langsmith evaluator get accuracy --session-id langsmith evaluator upload eval.py --name accuracy --function check_accuracy --dataset my-eval-set langsmith evaluator upload eval.ts --name accuracy --function checkAccuracy --dataset my-eval-set + langsmith evaluator create-llm --name relevance --project my-app --prompt prompt.json --schema schema.json --model-config model.json langsmith evaluator delete accuracy --yes`, } cmd.AddCommand(newEvaluatorGetCmd()) cmd.AddCommand(newEvaluatorListCmd()) cmd.AddCommand(newEvaluatorUploadCmd()) + cmd.AddCommand(newEvaluatorCreateLLMCmd()) cmd.AddCommand(newEvaluatorDeleteCmd()) return cmd } @@ -196,6 +199,7 @@ func newEvaluatorUploadCmd() *cobra.Command { targetDataset string targetProject string samplingRate float64 + traceFilter string replace bool yes bool ) @@ -207,19 +211,15 @@ func newEvaluatorUploadCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { evaluatorFile := args[0] - if targetDataset == "" && targetProject == "" { - output.OutputJSON(map[string]any{ - "error": "Must specify --dataset or --project (global evaluators not supported)", - }, "") + if err := validateEvaluatorTargetFlags(targetDataset, targetProject); err != nil { + output.OutputJSON(map[string]any{"error": err.Error()}, "") return } c := MustGetClient() ctx := context.Background() - // Resolve targets var datasetID, projectID string - if targetDataset != "" { ds, err := resolveDataset(ctx, c, targetDataset) if err != nil { @@ -227,7 +227,6 @@ func newEvaluatorUploadCmd() *cobra.Command { } datasetID = ds.ID } - if targetProject != "" { sid, err := c.ResolveSessionID(ctx, targetProject) if err != nil { @@ -236,34 +235,6 @@ func newEvaluatorUploadCmd() *cobra.Command { projectID = sid } - // Check for existing evaluator - rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) - if err != nil { - ExitErrorf("checking existing evaluators: %v", err) - } - - existing := findEvaluator(*rules, name, datasetID, projectID) - if existing != nil { - if !replace { - output.OutputJSON(map[string]any{ - "error": fmt.Sprintf("Evaluator '%s' already exists (use --replace to overwrite)", name), - "id": existing.ID, - }, "") - return - } - if !yes { - fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) - var confirm string - _, _ = fmt.Scanln(&confirm) - if strings.ToLower(confirm) != "y" { - ExitError("aborted") - } - } - if err := c.RawDelete(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), nil); err != nil { - ExitErrorf("deleting existing evaluator: %v", err) - } - } - // Read and prepare function source source, err := os.ReadFile(evaluatorFile) if err != nil { @@ -292,7 +263,6 @@ func newEvaluatorUploadCmd() *cobra.Command { sourceStr = renameJSFunction(sourceStr, funcName) } - // Build payload payload := map[string]any{ "display_name": name, "sampling_rate": samplingRate, @@ -302,29 +272,60 @@ func newEvaluatorUploadCmd() *cobra.Command { {"code": sourceStr, "language": language}, }, } - if datasetID != "" { payload["dataset_id"] = datasetID } if projectID != "" { payload["session_id"] = projectID } + if traceFilter != "" { + payload["trace_filter"] = traceFilter + } + + // Prepare the new evaluator before touching any existing one. If this is + // a replacement, the old evaluator stays in place until the new version is ready. + rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) + if err != nil { + ExitErrorf("checking existing evaluators: %v", err) + } + + existing := findEvaluator(*rules, name, datasetID, projectID) + if existing != nil { + if !replace { + output.OutputJSON(map[string]any{ + "error": fmt.Sprintf("Evaluator '%s' already exists (use --replace to overwrite)", name), + "id": existing.ID, + }, "") + return + } + if !yes { + fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) + var confirm string + _, _ = fmt.Scanln(&confirm) + if strings.ToLower(confirm) != "y" { + ExitError("aborted") + } + } + } var result map[string]any - if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { + if existing != nil { + if err := c.RawPatch(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), payload, &result); err != nil { + ExitErrorf("replacing evaluator: %v", err) + } + } else if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { ExitErrorf("uploading evaluator: %v", err) } - target := "project" + targetLabel := "project" if datasetID != "" { - target = "dataset" + targetLabel = "dataset" } - output.OutputJSON(map[string]any{ "status": "uploaded", "id": result["id"], "name": name, - "target": target, + "target": targetLabel, }, "") }, } @@ -334,6 +335,7 @@ func newEvaluatorUploadCmd() *cobra.Command { cmd.Flags().StringVar(&targetDataset, "dataset", "", "Target dataset name (offline evaluator)") cmd.Flags().StringVar(&targetProject, "project", "", "Target project name (online evaluator)") cmd.Flags().Float64Var(&samplingRate, "sampling-rate", 1.0, "Fraction of runs to evaluate (0.0-1.0)") + cmd.Flags().StringVar(&traceFilter, "trace-filter", "", "Filter expression for which runs to evaluate") cmd.Flags().BoolVar(&replace, "replace", false, "Replace existing evaluator with same name") cmd.Flags().BoolVar(&yes, "yes", false, "Skip confirmation prompt when replacing") _ = cmd.MarkFlagRequired("name") @@ -342,6 +344,104 @@ func newEvaluatorUploadCmd() *cobra.Command { return cmd } +// Registers create-llm for LLM-as-judge evaluators. +func newEvaluatorCreateLLMCmd() *cobra.Command { + var ( + name string + targetDataset string + targetProject string + samplingRate float64 + traceFilter string + hubRef string + promptPath string + schemaPath string + modelConfigPath string + variableMapping string + replace bool + yes bool + ) + + cmd := &cobra.Command{ + Use: "create-llm", + Short: "Create an LLM-as-judge evaluator rule", + Long: `Create an LLM-as-judge run rule. --model-config is always required. + +Provide the judge prompt inline with --prompt and --schema, or point at Prompt Hub +with --hub-ref (which replaces --prompt and --schema). The model is configured separately. + +Examples: + langsmith evaluator create-llm --name relevance --project my-app \ + --prompt prompt.json --schema schema.json --model-config model.json + langsmith evaluator create-llm --name relevance --project my-app \ + --hub-ref my-org/relevance:latest --model-config model.json`, + Run: func(cmd *cobra.Command, args []string) { + c := MustGetClient() + ctx := context.Background() + + target, err := resolveLLMEvaluatorTarget(ctx, c, targetDataset, targetProject) + if err != nil { + ExitErrorf("%v", err) + } + mapping, err := parseVariableMapping(variableMapping) + if err != nil { + ExitErrorf("%v", err) + } + payload, err := buildLLMEvaluatorPayload( + name, target, samplingRate, traceFilter, hubRef, + promptPath, schemaPath, modelConfigPath, mapping, + ) + if err != nil { + ExitErrorf("%v", err) + } + existing, err := findLLMEvaluatorForCreate(ctx, c, name, target, replace, yes) + if err != nil { + if err.Error() == "aborted" { + ExitError("aborted") + } + if existing != nil { + output.OutputJSON(map[string]any{"error": err.Error(), "id": existing.ID}, "") + return + } + ExitErrorf("%v", err) + } + + var result map[string]any + if existing != nil { + if err := c.RawPatch(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), payload, &result); err != nil { + ExitErrorf("replacing LLM evaluator: %v", err) + } + } else if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { + ExitErrorf("creating LLM evaluator: %v", err) + } + targetLabel := "project" + if target.datasetID != "" { + targetLabel = "dataset" + } + output.OutputJSON(map[string]any{ + "status": "created", "type": "llm", + "id": result["id"], "name": name, "target": targetLabel, + }, "") + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Display name (required)") + cmd.Flags().StringVar(&targetDataset, "dataset", "", "Target dataset name") + cmd.Flags().StringVar(&targetProject, "project", "", "Target project name") + cmd.Flags().Float64Var(&samplingRate, "sampling-rate", 1.0, "Fraction of runs to evaluate (0.0-1.0)") + cmd.Flags().StringVar(&traceFilter, "trace-filter", "", "Filter expression for which runs to evaluate") + cmd.Flags().StringVar(&hubRef, "hub-ref", "", "Prompt Hub reference; replaces --prompt and --schema (e.g. my-org/prompt:latest)") + cmd.Flags().StringVar(&promptPath, "prompt", "", "Prompt JSON file ([[role,content],...] or [{role,content},...]); omit if --hub-ref is set") + cmd.Flags().StringVar(&schemaPath, "schema", "", "JSON schema file for structured output; omit if --hub-ref is set") + cmd.Flags().StringVar(&modelConfigPath, "model-config", "", "Serialized LangChain model JSON (required; copy from UI or GET /runs/rules)") + cmd.Flags().StringVar(&variableMapping, "variable-mapping", "", `Map prompt vars to trace paths (JSON or @file.json)`) + cmd.Flags().BoolVar(&replace, "replace", false, "Replace existing evaluator with same name") + cmd.Flags().BoolVar(&yes, "yes", false, "Skip confirmation when replacing") + _ = cmd.MarkFlagRequired("name") + _ = cmd.MarkFlagRequired("model-config") + + return cmd +} + func newEvaluatorDeleteCmd() *cobra.Command { var yes bool diff --git a/internal/cmd/evaluator_llm.go b/internal/cmd/evaluator_llm.go new file mode 100644 index 0000000..6b73443 --- /dev/null +++ b/internal/cmd/evaluator_llm.go @@ -0,0 +1,213 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/langchain-ai/langsmith-cli/internal/client" + langsmith "github.com/langchain-ai/langsmith-go" +) + +type llmEvaluatorTarget struct { + datasetID, projectID string +} + +func loadJSONFile(path string, dest any) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + if err := json.Unmarshal(data, dest); err != nil { + return fmt.Errorf("parsing JSON in %s: %w", path, err) + } + return nil +} + +// Accepts a plain schema file or one already saved from the LangSmith UI. +func loadEvaluatorSchema(path string) (map[string]any, error) { + var raw map[string]any + if err := loadJSONFile(path, &raw); err != nil { + return nil, err + } + if _, ok := raw["parameters"]; ok { + return raw, nil + } + return map[string]any{"name": "eval", "description": "", "parameters": raw}, nil +} + +func validatePromptMessagePair(i int, path string, msg []string) error { + if len(msg) != 2 || msg[0] == "" || msg[1] == "" { + return fmt.Errorf("prompt message %d in %s must be [role, content] with non-empty values", i, path) + } + return nil +} + +// Loads the judge's system/user messages from a prompt file. +func loadPromptMessagesFromJSON(path string, data []byte) ([][]string, error) { + var pairs [][]string + if err := json.Unmarshal(data, &pairs); err == nil && len(pairs) > 0 { + for i, msg := range pairs { + if err := validatePromptMessagePair(i, path, msg); err != nil { + return nil, err + } + } + return pairs, nil + } + var objects []struct { + Role, Content string + } + if err := json.Unmarshal(data, &objects); err != nil || len(objects) == 0 { + return nil, fmt.Errorf( + "prompt file %s: expected [[role,content],...] or [{role,content},...]", path, + ) + } + pairs = make([][]string, len(objects)) + for i, obj := range objects { + if err := validatePromptMessagePair(i, path, []string{obj.Role, obj.Content}); err != nil { + return nil, err + } + pairs[i] = []string{obj.Role, obj.Content} + } + return pairs, nil +} + +// Maps {{placeholders}} in the prompt to fields on each trace. +func parseVariableMapping(raw string) (map[string]string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } + if strings.HasPrefix(raw, "@") { + var mapping map[string]string + if err := loadJSONFile(raw[1:], &mapping); err != nil { + return nil, err + } + return mapping, nil + } + var mapping map[string]string + if err := json.Unmarshal([]byte(raw), &mapping); err != nil { + return nil, fmt.Errorf("parsing --variable-mapping: %w", err) + } + return mapping, nil +} + +func validateEvaluatorTargetFlags(dataset, project string) error { + if dataset == "" && project == "" { + return fmt.Errorf("must specify --dataset or --project (global evaluators not supported)") + } + if dataset != "" && project != "" { + return fmt.Errorf("cannot specify both --dataset and --project") + } + return nil +} + +// Finds the dataset or project this evaluator should run on. +func resolveLLMEvaluatorTarget(ctx context.Context, c *client.Client, dataset, project string) (llmEvaluatorTarget, error) { + if err := validateEvaluatorTargetFlags(dataset, project); err != nil { + return llmEvaluatorTarget{}, err + } + var target llmEvaluatorTarget + if dataset != "" { + ds, err := resolveDataset(ctx, c, dataset) + if err != nil { + return llmEvaluatorTarget{}, err + } + target.datasetID = ds.ID + } + if project != "" { + sid, err := c.ResolveSessionID(ctx, project) + if err != nil { + return llmEvaluatorTarget{}, err + } + target.projectID = sid + } + return target, nil +} + +// Finds an existing evaluator with the same name on this dataset or project. +// When the evaluator already exists and --replace is not set, returns the existing +// rule alongside the error so callers can reuse it without another list call. +func findLLMEvaluatorForCreate(ctx context.Context, c *client.Client, name string, target llmEvaluatorTarget, replace, yes bool) (*langsmith.Evaluator, error) { + rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) + if err != nil { + return nil, fmt.Errorf("checking existing evaluators: %w", err) + } + existing := findEvaluator(*rules, name, target.datasetID, target.projectID) + if existing == nil { + return nil, nil + } + if !replace { + return existing, fmt.Errorf("evaluator %q already exists (use --replace to overwrite)", name) + } + if !yes { + fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) + var confirm string + _, _ = fmt.Scanln(&confirm) + if strings.ToLower(confirm) != "y" { + return nil, fmt.Errorf("aborted") + } + } + return existing, nil +} + +// Packages prompt, schema, model, and targeting into the create-evaluator request. +func buildLLMEvaluatorPayload( + name string, + target llmEvaluatorTarget, + samplingRate float64, + traceFilter, hubRef, promptPath, schemaPath, modelConfigPath string, + variableMapping map[string]string, +) (map[string]any, error) { + if modelConfigPath == "" { + return nil, fmt.Errorf("--model-config is required") + } + var model map[string]any + if err := loadJSONFile(modelConfigPath, &model); err != nil { + return nil, err + } + + structured := map[string]any{"model": model} + if hubRef != "" { + structured["hub_ref"] = hubRef + } else { + if promptPath == "" || schemaPath == "" { + return nil, fmt.Errorf("--prompt and --schema are required unless --hub-ref is set") + } + data, err := os.ReadFile(promptPath) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", promptPath, err) + } + messages, err := loadPromptMessagesFromJSON(promptPath, data) + if err != nil { + return nil, err + } + schema, err := loadEvaluatorSchema(schemaPath) + if err != nil { + return nil, err + } + structured["prompt"] = messages + structured["schema"] = schema + structured["template_format"] = "mustache" + } + if len(variableMapping) > 0 { + structured["variable_mapping"] = variableMapping + } + payload := map[string]any{ + "display_name": name, "sampling_rate": samplingRate, "is_enabled": true, + "include_extended_stats": false, + "evaluators": []map[string]any{{"structured": structured}}, + } + if traceFilter != "" { + payload["trace_filter"] = traceFilter + } + if target.datasetID != "" { + payload["dataset_id"] = target.datasetID + } + if target.projectID != "" { + payload["session_id"] = target.projectID + } + return payload, nil +} diff --git a/internal/cmd/evaluator_test.go b/internal/cmd/evaluator_test.go index 4936821..ae5c482 100644 --- a/internal/cmd/evaluator_test.go +++ b/internal/cmd/evaluator_test.go @@ -3,6 +3,7 @@ package cmd import ( "encoding/json" "net/http" + "os" "strings" "testing" @@ -366,11 +367,100 @@ func TestFindEvaluator_NameMatchButNoTarget(t *testing.T) { } } +func TestValidateEvaluatorTargetFlags(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + dataset string + project string + wantErr string + }{ + {name: "requires one target", wantErr: "must specify"}, + {name: "rejects both targets", dataset: "ds", project: "proj", wantErr: "cannot specify both"}, + {name: "dataset only", dataset: "ds"}, + {name: "project only", project: "proj"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := validateEvaluatorTargetFlags(tt.dataset, tt.project) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected %q error, got %v", tt.wantErr, err) + } + }) + } +} + +func TestLoadPromptMessagesFromJSON(t *testing.T) { + t.Parallel() + + t.Run("pair format", func(t *testing.T) { + t.Parallel() + msgs, err := loadPromptMessagesFromJSON("prompt.json", []byte( + `[["system","You are a judge."],["user","Q: {{input}}"]]`, + )) + if err != nil { + t.Fatal(err) + } + if len(msgs) != 2 || msgs[0][0] != "system" { + t.Fatalf("unexpected messages: %#v", msgs) + } + }) + + t.Run("role content objects", func(t *testing.T) { + t.Parallel() + msgs, err := loadPromptMessagesFromJSON("prompt.json", []byte( + `[{"role":"system","content":"You are a judge."},{"role":"user","content":"Q: {{input}}"}]`, + )) + if err != nil { + t.Fatal(err) + } + if len(msgs) != 2 || msgs[1][1] != "Q: {{input}}" { + t.Fatalf("unexpected messages: %#v", msgs) + } + }) + + t.Run("invalid format", func(t *testing.T) { + t.Parallel() + _, err := loadPromptMessagesFromJSON("prompt.json", []byte(`{"messages":[]}`)) + if err == nil || !strings.Contains(err.Error(), "expected [[role,content]") { + t.Fatalf("expected format hint, got %v", err) + } + }) + + t.Run("rejects single-element pair", func(t *testing.T) { + t.Parallel() + _, err := loadPromptMessagesFromJSON("prompt.json", []byte(`[["system"]]`)) + if err == nil || !strings.Contains(err.Error(), "must be [role, content]") { + t.Fatalf("expected length error, got %v", err) + } + }) +} + +func TestBuildLLMEvaluatorPayload_requiresModelConfig(t *testing.T) { + t.Parallel() + + _, err := buildLLMEvaluatorPayload( + "relevance", llmEvaluatorTarget{projectID: "proj-1"}, + 1.0, "", "", "prompt.json", "schema.json", "", nil, + ) + if err == nil || !strings.Contains(err.Error(), "--model-config is required") { + t.Fatalf("expected model-config error, got %v", err) + } +} + // ==================== Command structure tests ==================== func TestEvaluatorCmd_Subcommands(t *testing.T) { cmd := newEvaluatorCmd() - expected := map[string]bool{"get": false, "list": false, "upload": false, "delete": false} + expected := map[string]bool{"get": false, "list": false, "upload": false, "create-llm": false, "delete": false} for _, sub := range cmd.Commands() { if _, ok := expected[sub.Name()]; ok { expected[sub.Name()] = true @@ -444,6 +534,7 @@ func TestEvaluatorUploadCmd_Flags(t *testing.T) { "dataset": "", "project": "", "sampling-rate": "1", + "trace-filter": "", "replace": "false", "yes": "false", } @@ -639,6 +730,194 @@ func TestEvaluatorListCmd_VerifiesAPIKeyHeader(t *testing.T) { } } +func TestEvaluatorUploadReplacePatchesExistingCodeEvaluator(t *testing.T) { + evaluatorFile := t.TempDir() + "/eval.py" + if err := os.WriteFile( + evaluatorFile, + []byte("def check_accuracy(run, example):\n return {\"score\": 1}\n"), + 0o600, + ); err != nil { + t.Fatal(err) + } + + var sawDelete bool + var patchBody map[string]any + ts := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/api/v1/sessions" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]map[string]any{ + {"id": "project-1", "name": "my-project"}, + }) + case r.URL.Path == "/api/v1/runs/rules" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]testRule{ + { + ID: "existing-rule", + DisplayName: "accuracy", + SamplingRate: 0.25, + IsEnabled: true, + SessionID: "project-1", + CodeEvaluators: []testCodeEval{ + {Code: "def perform_eval(run, example):\n return {\"score\": 0}", Language: "python"}, + }, + }, + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "PATCH": + if err := json.NewDecoder(r.Body).Decode(&patchBody); err != nil { + t.Fatalf("decoding patch body: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "existing-rule", + "display_name": "accuracy", + "session_id": "project-1", + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "DELETE": + sawDelete = true + http.Error(w, "delete should not be called", http.StatusInternalServerError) + default: + http.Error(w, "not found", http.StatusNotFound) + } + }) + + cleanup := setupTestEnv(t, ts.URL) + defer cleanup() + flagOutputFormat = "json" + + out := captureStdout(t, func() { + cmd := newEvaluatorUploadCmd() + _ = cmd.Flags().Set("name", "accuracy") + _ = cmd.Flags().Set("function", "check_accuracy") + _ = cmd.Flags().Set("project", "my-project") + _ = cmd.Flags().Set("sampling-rate", "0.5") + _ = cmd.Flags().Set("replace", "true") + _ = cmd.Flags().Set("yes", "true") + cmd.Run(cmd, []string{evaluatorFile}) + }) + + if sawDelete { + t.Fatal("upload --replace should patch the existing evaluator, not delete it") + } + if patchBody == nil { + t.Fatal("expected PATCH body") + } + if patchBody["display_name"] != "accuracy" { + t.Errorf("expected display_name=accuracy, got %v", patchBody["display_name"]) + } + if patchBody["session_id"] != "project-1" { + t.Errorf("expected session_id=project-1, got %v", patchBody["session_id"]) + } + if patchBody["sampling_rate"] != 0.5 { + t.Errorf("expected sampling_rate=0.5, got %v", patchBody["sampling_rate"]) + } + evaluators, ok := patchBody["code_evaluators"].([]any) + if !ok || len(evaluators) != 1 { + t.Fatalf("expected one code evaluator, got %#v", patchBody["code_evaluators"]) + } + codeEvaluator, ok := evaluators[0].(map[string]any) + if !ok { + t.Fatalf("expected code evaluator object, got %#v", evaluators[0]) + } + if codeEvaluator["language"] != "python" { + t.Errorf("expected language=python, got %v", codeEvaluator["language"]) + } + code, _ := codeEvaluator["code"].(string) + if !strings.Contains(code, "def perform_eval(") { + t.Errorf("expected uploaded code to be renamed to perform_eval, got:\n%s", code) + } + + var result map[string]any + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("failed to parse output JSON: %v\noutput: %s", err, out) + } + if result["id"] != "existing-rule" { + t.Errorf("expected output id=existing-rule, got %v", result["id"]) + } +} + +func TestEvaluatorCreateLLMReplacePatchesExistingEvaluator(t *testing.T) { + modelConfigPath := t.TempDir() + "/model.json" + if err := os.WriteFile( + modelConfigPath, + []byte(`{"type":"chat","config":{"model":"test-model"}}`), + 0o600, + ); err != nil { + t.Fatal(err) + } + + var sawDelete bool + var patchBody map[string]any + ts := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/api/v1/sessions" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]map[string]any{ + {"id": "project-1", "name": "my-project"}, + }) + case r.URL.Path == "/api/v1/runs/rules" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]testRule{ + { + ID: "existing-rule", + DisplayName: "relevance", + SamplingRate: 0.25, + IsEnabled: true, + SessionID: "project-1", + Evaluators: []testLLMEval{ + {Structured: testLLMStructured{HubRef: "my-org/old:latest"}}, + }, + }, + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "PATCH": + if err := json.NewDecoder(r.Body).Decode(&patchBody); err != nil { + t.Fatalf("decoding patch body: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "existing-rule", + "display_name": "relevance", + "session_id": "project-1", + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "DELETE": + sawDelete = true + http.Error(w, "delete should not be called", http.StatusInternalServerError) + default: + http.Error(w, "not found", http.StatusNotFound) + } + }) + + cleanup := setupTestEnv(t, ts.URL) + defer cleanup() + flagOutputFormat = "json" + + out := captureStdout(t, func() { + cmd := newEvaluatorCreateLLMCmd() + _ = cmd.Flags().Set("name", "relevance") + _ = cmd.Flags().Set("project", "my-project") + _ = cmd.Flags().Set("hub-ref", "my-org/relevance:latest") + _ = cmd.Flags().Set("model-config", modelConfigPath) + _ = cmd.Flags().Set("sampling-rate", "0.5") + _ = cmd.Flags().Set("replace", "true") + _ = cmd.Flags().Set("yes", "true") + cmd.Run(cmd, nil) + }) + + if sawDelete { + t.Fatal("create-llm --replace should patch the existing evaluator, not delete it") + } + if patchBody == nil { + t.Fatal("expected PATCH body") + } + if patchBody["display_name"] != "relevance" || patchBody["evaluators"] == nil { + t.Fatalf("expected LLM evaluator replacement payload, got %#v", patchBody) + } + + var result map[string]any + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("failed to parse output JSON: %v\noutput: %s", err, out) + } + if result["id"] != "existing-rule" { + t.Errorf("expected output id=existing-rule, got %v", result["id"]) + } +} + // ==================== evaluator get ==================== func TestEvaluatorGetCmd_UseField(t *testing.T) { diff --git a/internal/cmd/helpers.go b/internal/cmd/helpers.go index 282872d..1089e79 100644 --- a/internal/cmd/helpers.go +++ b/internal/cmd/helpers.go @@ -183,14 +183,12 @@ func buildRunSelectV2(includeIO, includeFeedback bool) []langsmith.RunQueryV2Par langsmith.RunQueryV2ParamsSelectTotalCost, langsmith.RunQueryV2ParamsSelectLatencySeconds, langsmith.RunQueryV2ParamsSelectAppPath, - langsmith.RunQueryV2ParamsSelectFirstTokenTime, } if includeIO { fields = append(fields, langsmith.RunQueryV2ParamsSelectInputs, langsmith.RunQueryV2ParamsSelectOutputs, langsmith.RunQueryV2ParamsSelectError, - langsmith.RunQueryV2ParamsSelectEvents, ) } if includeFeedback { @@ -201,7 +199,7 @@ func buildRunSelectV2(includeIO, includeFeedback bool) []langsmith.RunQueryV2Par // runV2ToSchema converts a v2 Run into the legacy v1 RunSchema shape so the // existing extract/output pipeline can consume it unchanged. -func runV2ToSchema(r langsmith.Run) langsmith.RunSchema { +func runV2ToSchema(r langsmith.QueryRunResponse) langsmith.RunSchema { extra := asMap(r.Extra) if md := asMap(r.Metadata); md != nil { if extra == nil { @@ -242,7 +240,6 @@ func runV2ToSchema(r langsmith.Run) langsmith.RunSchema { Outputs: asMap(r.Outputs), Extra: extra, FeedbackStats: convertFeedbackStats(r.FeedbackStats), - Events: convertEvents(r.Events), } if len(r.ParentRunIDs) > 0 { out.ParentRunIDs = r.ParentRunIDs @@ -263,7 +260,7 @@ func asMap(v interface{}) map[string]interface{} { // convertFeedbackStats round-trips the generated feedback-stats shape into the // loosely-typed map the downstream pipeline expects. Returns nil on empty input // or any marshalling error. -func convertFeedbackStats(stats map[string]langsmith.RunFeedbackStat) map[string]map[string]interface{} { +func convertFeedbackStats(stats map[string]langsmith.QueryRunResponseFeedbackStat) map[string]map[string]interface{} { if len(stats) == 0 { return nil } @@ -278,24 +275,6 @@ func convertFeedbackStats(stats map[string]langsmith.RunFeedbackStat) map[string return m } -// convertEvents round-trips the generated v2 event shape into the -// loosely-typed maps the legacy RunSchema.Events field expects. Returns nil on -// empty input or any marshalling error. -func convertEvents(events []langsmith.RunEvent) []map[string]interface{} { - if len(events) == 0 { - return nil - } - raw, err := json.Marshal(events) - if err != nil { - return nil - } - var m []map[string]interface{} - if err := json.Unmarshal(raw, &m); err != nil { - return nil - } - return m -} - // buildRunSelect returns the Select fields needed for the given include flags. // Returns nil when neither IO nor feedback is requested, letting the API use its defaults. // When set, includes all base/metadata fields so they aren't stripped from the response. @@ -316,7 +295,6 @@ func buildRunSelect(includeIO, includeFeedback bool) []langsmith.RunQueryParamsS langsmith.RunQueryParamsSelectStatus, // Metadata fields langsmith.RunQueryParamsSelectExtra, - langsmith.RunQueryParamsSelectFirstTokenTime, langsmith.RunQueryParamsSelectPromptTokens, langsmith.RunQueryParamsSelectCompletionTokens, langsmith.RunQueryParamsSelectTotalTokens, @@ -331,7 +309,6 @@ func buildRunSelect(includeIO, includeFeedback bool) []langsmith.RunQueryParamsS langsmith.RunQueryParamsSelectInputs, langsmith.RunQueryParamsSelectOutputs, langsmith.RunQueryParamsSelectError, - langsmith.RunQueryParamsSelectEvents, ) } diff --git a/internal/cmd/helpers_test.go b/internal/cmd/helpers_test.go index 3744954..0cc0bf0 100644 --- a/internal/cmd/helpers_test.go +++ b/internal/cmd/helpers_test.go @@ -404,19 +404,6 @@ func TestBuildRunSelect_IncludesMetadataFields(t *testing.T) { } } -func TestBuildRunSelect_IncludesFirstTokenTimeAndEvents(t *testing.T) { - // first_token_time and events are native Run fields that --full is - // documented to return; a prior version of this select list silently - // dropped both even when --include-io/--include-metadata were requested. - has := selectSet(buildRunSelect(true, true)) - if !has[langsmith.RunQueryParamsSelectFirstTokenTime] { - t.Error("missing first_token_time field") - } - if !has[langsmith.RunQueryParamsSelectEvents] { - t.Error("missing events field (should be requested alongside inputs/outputs/error)") - } -} - // selectSet converts a slice to a set for easy lookup. func selectSet(sel []langsmith.RunQueryParamsSelect) map[langsmith.RunQueryParamsSelect]bool { m := make(map[langsmith.RunQueryParamsSelect]bool, len(sel)) @@ -497,56 +484,3 @@ func TestRunTreeData_AllFields(t *testing.T) { t.Error("unexpected RunTreeData field values") } } - -// ---------- buildRunSelectV2 ---------- - -func TestBuildRunSelectV2_IncludesFirstTokenTimeAndEvents(t *testing.T) { - // Mirrors TestBuildRunSelect_IncludesFirstTokenTimeAndEvents for the v2 - // select builder: first_token_time is a base field always requested, - // events is requested alongside inputs/outputs/error under --include-io. - base := buildRunSelectV2(false, false) - baseHas := map[langsmith.RunQueryV2ParamsSelect]bool{} - for _, f := range base { - baseHas[f] = true - } - if !baseHas[langsmith.RunQueryV2ParamsSelectFirstTokenTime] { - t.Error("missing first_token_time field in base v2 select set") - } - if baseHas[langsmith.RunQueryV2ParamsSelectEvents] { - t.Error("events should not be requested without --include-io") - } - - withIO := buildRunSelectV2(true, false) - ioHas := map[langsmith.RunQueryV2ParamsSelect]bool{} - for _, f := range withIO { - ioHas[f] = true - } - if !ioHas[langsmith.RunQueryV2ParamsSelectEvents] { - t.Error("missing events field when --include-io requested") - } -} - -// ---------- runV2ToSchema ---------- - -func TestRunV2ToSchema_MapsFirstTokenTimeAndEvents(t *testing.T) { - firstToken := time.Date(2026, 1, 15, 10, 30, 1, 0, time.UTC) - v2 := langsmith.Run{ - ID: "run-123", - FirstTokenTime: firstToken, - Events: []langsmith.RunEvent{ - {Name: "new_token", Kwargs: map[string]interface{}{"token": "hi"}}, - }, - } - - out := runV2ToSchema(v2) - - if !out.FirstTokenTime.Equal(firstToken) { - t.Errorf("expected FirstTokenTime=%v, got %v", firstToken, out.FirstTokenTime) - } - if len(out.Events) != 1 { - t.Fatalf("expected 1 event, got %d", len(out.Events)) - } - if out.Events[0]["name"] != "new_token" { - t.Errorf("expected event name=new_token, got %v", out.Events[0]["name"]) - } -} diff --git a/internal/cmd/install_method.go b/internal/cmd/install_method.go deleted file mode 100644 index 987aa39..0000000 --- a/internal/cmd/install_method.go +++ /dev/null @@ -1,101 +0,0 @@ -package cmd - -import ( - "path/filepath" - "strings" -) - -// installMethod identifies how the langsmith binary was installed, which -// determines whether self-update may replace the binary in place. -type installMethod int - -const ( - // methodManaged means self-update owns the binary: install.sh, install.ps1, - // or a manual download from GitHub Releases. In-place replacement is correct. - methodManaged installMethod = iota - // methodHomebrew means the binary lives in a Homebrew Cellar. - methodHomebrew - // methodScoop means the binary was installed by Scoop. - methodScoop - // methodGo means the binary was installed via `go install`. - methodGo - // methodDev means a local/development build with no release version. - methodDev -) - -// externalManagers describes installs owned by a package manager: how to name -// the manager in output and the command the user should run to update. Only the -// methods self-update must defer to appear here — methodManaged and methodDev are -// handled by self-update itself and intentionally have no entry, so membership in -// this map is the single source of truth for "must defer to a package manager". -var externalManagers = map[installMethod]struct { - label string // machine-readable name, for JSON output - display string // human-readable name, for the pretty message - command string // the command the user should run to update -}{ - methodHomebrew: {"homebrew", "Homebrew", "brew upgrade langchain-ai/tap/langsmith-cli"}, - methodScoop: {"scoop", "Scoop", "scoop update langsmith-cli"}, - methodGo: {"go", "`go install`", "go install github.com/langchain-ai/langsmith-cli/cmd/langsmith@latest"}, -} - -// detectInstallMethod classifies the install method from the resolved executable -// path, the build version, and the relevant environment values. It is pure: all -// I/O (resolving the path, reading env) happens in the caller. -// -// - execPath must already be resolved through filepath.EvalSymlinks so that -// Homebrew's bin symlinks point at the Cellar. -// - goos is runtime.GOOS. -// - gobin/gopath/home come from $GOBIN, $GOPATH, and the user's home dir. -func detectInstallMethod(execPath, version, goos, gobin, gopath, home string) installMethod { - // Normalize both separators to "/" so a single "/segment/" check works for - // Windows paths regardless of the OS this code runs on (matters for tests). - lower := strings.ToLower(strings.ReplaceAll(execPath, `\`, "/")) - - if strings.Contains(lower, "/cellar/") { - return methodHomebrew - } - if strings.Contains(lower, "/scoop/") { - return methodScoop - } - - dir := filepath.Dir(execPath) - for _, bin := range goBinDirs(gobin, gopath, home) { - if bin != "" && filepath.Clean(dir) == filepath.Clean(bin) { - return methodGo - } - } - - if version == "dev" { - return methodDev - } - return methodManaged -} - -// goBinDirs returns the candidate directories where `go install` places binaries, -// in precedence order: $GOBIN, then $GOPATH/bin, then $HOME/go/bin (Go's default -// GOPATH when unset). -func goBinDirs(gobin, gopath, home string) []string { - if gobin != "" { - return []string{gobin} - } - if gopath != "" { - // GOPATH may contain multiple entries; go install uses the first. - first := filepath.SplitList(gopath) - if len(first) > 0 && first[0] != "" { - return []string{filepath.Join(first[0], "bin")} - } - } - if home != "" { - return []string{filepath.Join(home, "go", "bin")} - } - return nil -} - -// shouldDeferToManager reports whether self-update must defer to a package -// manager instead of replacing the binary in place. --force overrides it. -// Membership in externalManagers is the source of truth; methodManaged and -// methodDev are not externally managed and so are never deferred. -func shouldDeferToManager(method installMethod, force bool) bool { - _, external := externalManagers[method] - return external && !force -} diff --git a/internal/cmd/install_method_test.go b/internal/cmd/install_method_test.go deleted file mode 100644 index 78aa832..0000000 --- a/internal/cmd/install_method_test.go +++ /dev/null @@ -1,233 +0,0 @@ -package cmd - -import ( - "encoding/json" - "strings" - "testing" -) - -func TestDetectInstallMethod(t *testing.T) { - tests := []struct { - name string - execPath string - version string - goos string - gobin string - gopath string - home string - want installMethod - }{ - { - name: "homebrew apple silicon cellar", - execPath: "/opt/homebrew/Cellar/langsmith-cli/0.2.38/bin/langsmith", - version: "0.2.38", - goos: "darwin", - home: "/Users/me", - want: methodHomebrew, - }, - { - name: "homebrew intel cellar", - execPath: "/usr/local/Cellar/langsmith-cli/0.2.38/bin/langsmith", - version: "0.2.38", - goos: "darwin", - home: "/Users/me", - want: methodHomebrew, - }, - { - name: "homebrew linux cellar", - execPath: "/home/linuxbrew/.linuxbrew/Cellar/langsmith-cli/0.2.38/bin/langsmith", - version: "0.2.38", - goos: "linux", - home: "/home/me", - want: methodHomebrew, - }, - { - name: "scoop apps", - execPath: `C:\Users\me\scoop\apps\langsmith-cli\current\langsmith.exe`, - version: "0.2.38", - goos: "windows", - home: `C:\Users\me`, - want: methodScoop, - }, - { - name: "scoop shims", - execPath: `C:\Users\me\scoop\shims\langsmith.exe`, - version: "0.2.38", - goos: "windows", - home: `C:\Users\me`, - want: methodScoop, - }, - { - name: "go install via GOBIN", - execPath: "/Users/me/dev/gobin/langsmith", - version: "0.2.38", - goos: "darwin", - gobin: "/Users/me/dev/gobin", - home: "/Users/me", - want: methodGo, - }, - { - name: "go install via GOPATH bin", - execPath: "/Users/me/gopath/bin/langsmith", - version: "0.2.38", - goos: "darwin", - gopath: "/Users/me/gopath", - home: "/Users/me", - want: methodGo, - }, - { - name: "go install via default HOME/go/bin", - execPath: "/Users/me/go/bin/langsmith", - version: "0.2.38", - goos: "darwin", - home: "/Users/me", - want: methodGo, - }, - { - name: "go install with dev version still detected as go", - execPath: "/Users/me/go/bin/langsmith", - version: "dev", - goos: "darwin", - home: "/Users/me", - want: methodGo, - }, - { - name: "managed install.sh usr local bin", - execPath: "/usr/local/bin/langsmith", - version: "0.2.38", - goos: "darwin", - home: "/Users/me", - want: methodManaged, - }, - { - name: "managed install.sh local bin", - execPath: "/Users/me/.local/bin/langsmith", - version: "0.2.38", - goos: "linux", - home: "/Users/me", - want: methodManaged, - }, - { - name: "dev build outside package dirs", - execPath: "/Users/me/repos/langsmith-cli/bin/langsmith", - version: "dev", - goos: "darwin", - home: "/Users/me", - want: methodDev, - }, - { - name: "GOBIN takes precedence over GOPATH", - execPath: "/Users/me/gopath/bin/langsmith", - version: "0.2.38", - goos: "darwin", - gobin: "/Users/me/gobin", - gopath: "/Users/me/gopath", - home: "/Users/me", - // exec is in GOPATH/bin, but GOBIN is set and differs → not the go bin dir - want: methodManaged, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := detectInstallMethod(tt.execPath, tt.version, tt.goos, tt.gobin, tt.gopath, tt.home) - if got != tt.want { - t.Errorf("detectInstallMethod() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestExternalManagers(t *testing.T) { - want := map[installMethod]struct{ label, command string }{ - methodHomebrew: {"homebrew", "brew upgrade langchain-ai/tap/langsmith-cli"}, - methodScoop: {"scoop", "scoop update langsmith-cli"}, - methodGo: {"go", "go install github.com/langchain-ai/langsmith-cli/cmd/langsmith@latest"}, - } - for method, exp := range want { - mgr, ok := externalManagers[method] - if !ok { - t.Errorf("externalManagers missing entry for %v", method) - continue - } - if mgr.label != exp.label { - t.Errorf("externalManagers[%v].label = %q, want %q", method, mgr.label, exp.label) - } - if mgr.command != exp.command { - t.Errorf("externalManagers[%v].command = %q, want %q", method, mgr.command, exp.command) - } - if mgr.display == "" { - t.Errorf("externalManagers[%v].display is empty", method) - } - } - // managed and dev are handled by self-update itself and must not be listed. - for _, method := range []installMethod{methodManaged, methodDev} { - if _, ok := externalManagers[method]; ok { - t.Errorf("externalManagers should not contain %v", method) - } - } -} - -func TestShouldDeferToManager(t *testing.T) { - tests := []struct { - method installMethod - force bool - want bool - }{ - {methodHomebrew, false, true}, - {methodHomebrew, true, false}, // --force overrides the defer - {methodScoop, false, true}, - {methodScoop, true, false}, - {methodGo, false, true}, - {methodGo, true, false}, - {methodManaged, false, false}, - {methodManaged, true, false}, - {methodDev, false, false}, - {methodDev, true, false}, - } - for _, tt := range tests { - if got := shouldDeferToManager(tt.method, tt.force); got != tt.want { - t.Errorf("shouldDeferToManager(%v, force=%v) = %v, want %v", tt.method, tt.force, got, tt.want) - } - } -} - -func TestReportManagedExternally_JSON(t *testing.T) { - oldFmt := flagOutputFormat - flagOutputFormat = "json" - defer func() { flagOutputFormat = oldFmt }() - - output := captureStdout(t, func() { - if err := reportManagedExternally(methodHomebrew, "json"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - }) - - var result map[string]string - if err := json.Unmarshal([]byte(output), &result); err != nil { - t.Fatalf("invalid JSON output: %v, output: %q", err, output) - } - if result["status"] != "managed-externally" { - t.Errorf("expected status managed-externally, got %q", result["status"]) - } - if result["install_method"] != "homebrew" { - t.Errorf("expected install_method homebrew, got %q", result["install_method"]) - } - if result["update_command"] != "brew upgrade langchain-ai/tap/langsmith-cli" { - t.Errorf("unexpected update_command: %q", result["update_command"]) - } -} - -func TestReportManagedExternally_Pretty(t *testing.T) { - output := captureStdout(t, func() { - if err := reportManagedExternally(methodScoop, "pretty"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - }) - if !strings.Contains(output, "scoop update langsmith-cli") { - t.Errorf("expected pretty output to contain the scoop command, got %q", output) - } - if !strings.Contains(output, "--force") { - t.Errorf("expected pretty output to mention --force, got %q", output) - } -} diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 9c8992b..30f514c 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -198,10 +198,6 @@ func resolveClientOptions(refreshOAuth bool) (client.Options, error) { opts.OAuthAccessToken = profile.AccessToken() case hasProfile && profile.APIKey != "": opts.APIKey = profile.APIKey - // Route the resolved profile through the SDK (WithProfile) so an explicit - // selection replaces the config's current_profile instead of inheriting - // its tenant/base URL. APIKey is kept for the raw-HTTP helpers. - opts.ProfileName = profileName } if cfgErr != nil && opts.APIKey == "" { return opts, cfgErr diff --git a/internal/cmd/root_test.go b/internal/cmd/root_test.go index d3734c4..0e8431e 100644 --- a/internal/cmd/root_test.go +++ b/internal/cmd/root_test.go @@ -313,61 +313,6 @@ func TestResolveClientOptionsRefreshesProfileWithoutAccessToken(t *testing.T) { } } -func TestResolveClientOptions_SetsProfileNameForAPIKeyProfile(t *testing.T) { - oldKey := flagAPIKey - oldURL := flagAPIURL - oldProfile := flagProfile - oldWS := flagWorkspaceID - defer func() { - flagAPIKey = oldKey - flagAPIURL = oldURL - flagProfile = oldProfile - flagWorkspaceID = oldWS - }() - flagAPIKey = "" - flagAPIURL = "" - flagWorkspaceID = "" - flagProfile = "aws" - - path := filepath.Join(t.TempDir(), "config.json") - t.Setenv("LANGSMITH_CONFIG_FILE", path) - t.Setenv("LANGSMITH_API_KEY", "") - t.Setenv("LANGSMITH_ENDPOINT", "") - t.Setenv("LANGSMITH_WORKSPACE_ID", "") - t.Setenv("LANGSMITH_TENANT_ID", "") - t.Setenv("LANGSMITH_PROFILE", "") - if err := os.WriteFile(path, []byte(`{ - "current_profile": "prod", - "profiles": { - "prod": { - "api_key": "prod-api-key", - "workspace_id": "prod-workspace-id" - }, - "aws": { - "api_key": "aws-api-key" - } - } -} -`), 0600); err != nil { - t.Fatal(err) - } - - opts, err := resolveClientOptions(false) - if err != nil { - t.Fatalf("resolveClientOptions returned error: %v", err) - } - if opts.APIKey != "aws-api-key" { - t.Fatalf("expected profile api key aws-api-key, got %q", opts.APIKey) - } - // The resolved profile must reach the SDK via WithProfile so an explicit - // --profile replaces current_profile and clears its inherited tenant. Without - // ProfileName set, NewWithOptions passes only WithAPIKey and current_profile's - // tenant leaks (403 cross-workspace). - if opts.ProfileName != "aws" { - t.Fatalf("expected ProfileName=aws, got %q", opts.ProfileName) - } -} - func TestGetOAuthAccessToken_ProfileFallback(t *testing.T) { oldKey := flagAPIKey oldURL := flagAPIURL diff --git a/internal/cmd/run.go b/internal/cmd/run.go index 5b5d358..a3aea47 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -95,8 +95,8 @@ func newRunListCmd() *cobra.Command { addCommonFilterFlags(cmd, &ff, true) addVersionFlag(cmd, &ff) - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") @@ -179,8 +179,8 @@ func newRunGetCmd() *cobra.Command { cmd.Flags().StringVar(&since, "since", "", "Only include runs after this timestamp, e.g. 2024-01-15T00:00:00Z (overrides 7-day default)") cmd.Flags().IntVar(&lastNMinutes, "last-n-minutes", 0, "Only include runs from the last N minutes, e.g. 60 (overrides 7-day default)") cmd.Flags().StringVar(&version, "version", "", `Query API version: "" (v1, default) or "v2" (SmithDB)`) - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") @@ -244,8 +244,8 @@ func newRunExportCmd() *cobra.Command { addCommonFilterFlags(cmd, &ff, true) addVersionFlag(cmd, &ff) - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") diff --git a/internal/cmd/sandbox_box.go b/internal/cmd/sandbox_box.go index 049cec3..3738387 100644 --- a/internal/cmd/sandbox_box.go +++ b/internal/cmd/sandbox_box.go @@ -16,7 +16,7 @@ import ( const defaultBoxPollInterval = 2 * time.Second -func waitForBoxReady(ctx context.Context, c *client.Client, name string) (*langsmith.SandboxStatusResponse, error) { +func waitForBoxReady(ctx context.Context, c *client.Client, name string) (*langsmith.SandboxBoxGetStatusResponse, error) { for { resp, err := c.SDK.Sandboxes.Boxes.GetStatus(ctx, name) if err != nil { diff --git a/internal/cmd/sandbox_test.go b/internal/cmd/sandbox_test.go index 1c7f3d3..30030d3 100644 --- a/internal/cmd/sandbox_test.go +++ b/internal/cmd/sandbox_test.go @@ -288,7 +288,7 @@ func TestSandboxBoxDetailRenderSupportsSDKResponseTypes(t *testing.T) { }{ { name: "new response", - model: langsmith.SandboxResponse{ + model: langsmith.SandboxBoxNewResponse{ ID: "box-new", Name: "new-vm", Status: "running", @@ -305,7 +305,7 @@ func TestSandboxBoxDetailRenderSupportsSDKResponseTypes(t *testing.T) { }, { name: "get response", - model: langsmith.SandboxResponse{ + model: langsmith.SandboxBoxGetResponse{ ID: "box-get", Name: "get-vm", Status: "running", @@ -322,7 +322,7 @@ func TestSandboxBoxDetailRenderSupportsSDKResponseTypes(t *testing.T) { }, { name: "update response", - model: langsmith.SandboxResponse{ + model: langsmith.SandboxBoxUpdateResponse{ ID: "box-update", Name: "update-vm", Status: "stopped", diff --git a/internal/cmd/thread.go b/internal/cmd/thread.go index 92ab76c..57d4eef 100644 --- a/internal/cmd/thread.go +++ b/internal/cmd/thread.go @@ -267,8 +267,8 @@ func newThreadGetCmd() *cobra.Command { } cmd.Flags().StringVar(&project, "project", "", "Project name [env: LANGSMITH_PROJECT]") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().IntVarP(&limit, "limit", "n", 0, "Maximum number of runs (turns) to return") diff --git a/internal/cmd/trace.go b/internal/cmd/trace.go index ef78232..bc001d2 100644 --- a/internal/cmd/trace.go +++ b/internal/cmd/trace.go @@ -144,8 +144,8 @@ func newTraceListCmd() *cobra.Command { addCommonFilterFlags(cmd, &ff, false) cmd.Flags().StringVar(&ff.ProjectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&includeFlagged, "include-flagged", false, "Add flagged_comment field populated from user-flagged trace feedback") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") @@ -222,8 +222,8 @@ func newTraceGetCmd() *cobra.Command { cmd.Flags().StringVar(&projectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") cmd.Flags().StringVar(&since, "since", "", "Only include runs after this timestamp, e.g. 2024-01-15T00:00:00Z (overrides 7-day default)") cmd.Flags().IntVar(&lastNMinutes, "last-n-minutes", 0, "Only include runs from the last N minutes, e.g. 60 (overrides 7-day default)") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") @@ -332,8 +332,8 @@ func newTraceExportCmd() *cobra.Command { addCommonFilterFlags(cmd, &ff, false) cmd.Flags().StringVar(&ff.ProjectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVar(&filenamePattern, "filename-pattern", "{trace_id}.jsonl", diff --git a/internal/cmd/update.go b/internal/cmd/update.go index d84deb1..19ba244 100644 --- a/internal/cmd/update.go +++ b/internal/cmd/update.go @@ -28,48 +28,26 @@ var githubDownloadBaseURL = "https://github.com" func newUpdateCmd(rawVersion string) *cobra.Command { var dryRun bool - var force bool cmd := &cobra.Command{ Use: "self-update", Short: "Update langsmith to the latest version", Long: "Check for and install the latest version of the langsmith CLI.", RunE: func(cmd *cobra.Command, args []string) error { - return runUpdate(cmd.Context(), rawVersion, dryRun, force) + return runUpdate(cmd.Context(), rawVersion, dryRun) }, } cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Check for updates without installing") - cmd.Flags().BoolVar(&force, "force", false, "Replace the binary in place even if a package manager (brew, scoop, go) manages this install") return cmd } -func runUpdate(ctx context.Context, currentVersion string, dryRun, force bool) error { - // Resolve the current binary path first: it drives both install-method - // detection and the eventual in-place replacement. - execPath, err := os.Executable() - if err != nil { - return fmt.Errorf("resolving executable path: %w", err) - } - execPath, err = filepath.EvalSymlinks(execPath) - if err != nil { - return fmt.Errorf("resolving symlinks: %w", err) - } - - home, _ := os.UserHomeDir() - method := detectInstallMethod(execPath, currentVersion, runtime.GOOS, os.Getenv("GOBIN"), os.Getenv("GOPATH"), home) - - if method == methodDev { +func runUpdate(ctx context.Context, currentVersion string, dryRun bool) error { + if currentVersion == "dev" { return fmt.Errorf("cannot update a development build; install from a release") } - // For package-manager-owned installs, refuse to touch the binary and point - // the user at the right command instead. --force overrides this. - if shouldDeferToManager(method, force) { - return reportManagedExternally(method, GetFormat()) - } - latest, err := fetchLatestVersion(ctx) if err != nil { return fmt.Errorf("checking for updates: %w", err) @@ -105,6 +83,16 @@ func runUpdate(ctx context.Context, currentVersion string, dryRun, force bool) e return nil } + // Resolve current binary path + execPath, err := os.Executable() + if err != nil { + return fmt.Errorf("resolving executable path: %w", err) + } + execPath, err = filepath.EvalSymlinks(execPath) + if err != nil { + return fmt.Errorf("resolving symlinks: %w", err) + } + archiveName := buildArchiveName() archiveURL := fmt.Sprintf("%s/langchain-ai/langsmith-cli/releases/download/v%s/%s", githubDownloadBaseURL, latest, archiveName) checksumURL := fmt.Sprintf("%s/langchain-ai/langsmith-cli/releases/download/v%s/checksums.txt", githubDownloadBaseURL, latest) @@ -158,31 +146,6 @@ func runUpdate(ctx context.Context, currentVersion string, dryRun, force bool) e return nil } -// reportManagedExternally informs the user that their install is managed by a -// package manager and prints the command to update it, without modifying the -// binary. It returns nil (a successful, informational outcome). -func reportManagedExternally(method installMethod, format string) error { - mgr, ok := externalManagers[method] - if !ok { - // Programming error: only externally-managed methods should reach here - // (methodManaged updates in place, methodDev errors out earlier). - return fmt.Errorf("internal error: install method %d is not externally managed", method) - } - - if format == "pretty" { - fmt.Printf("langsmith was installed with %s, which manages updates for you.\n"+ - "To update, run:\n\n %s\n\n(Use --force to update in place anyway.)\n", mgr.display, mgr.command) - } else { - out, _ := json.Marshal(map[string]string{ - "status": "managed-externally", - "install_method": mgr.label, - "update_command": mgr.command, - }) - fmt.Println(string(out)) - } - return nil -} - // fetchLatestVersion queries the GitHub Releases API for the latest release tag. func fetchLatestVersion(ctx context.Context) (string, error) { url := fmt.Sprintf("%s/repos/langchain-ai/langsmith-cli/releases/latest", githubReleasesBaseURL) diff --git a/internal/cmd/update_test.go b/internal/cmd/update_test.go index 8e52280..7580434 100644 --- a/internal/cmd/update_test.go +++ b/internal/cmd/update_test.go @@ -203,7 +203,7 @@ func TestRunUpdate_AlreadyUpToDate(t *testing.T) { defer func() { flagOutputFormat = oldFmt }() output := captureStdout(t, func() { - err := runUpdate(context.Background(), "0.1.7", false, false) + err := runUpdate(context.Background(), "0.1.7", false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -235,7 +235,7 @@ func TestRunUpdate_DryRun(t *testing.T) { defer func() { flagOutputFormat = oldFmt }() output := captureStdout(t, func() { - err := runUpdate(context.Background(), "0.1.7", true, false) + err := runUpdate(context.Background(), "0.1.7", true) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -254,7 +254,7 @@ func TestRunUpdate_DryRun(t *testing.T) { } func TestRunUpdate_DevBuild(t *testing.T) { - err := runUpdate(context.Background(), "dev", false, false) + err := runUpdate(context.Background(), "dev", false) if err == nil { t.Error("expected error for dev build") } diff --git a/internal/cmdutil/resolve.go b/internal/cmdutil/resolve.go index 7ea4f83..0efa9de 100644 --- a/internal/cmdutil/resolve.go +++ b/internal/cmdutil/resolve.go @@ -187,10 +187,6 @@ func ResolveClientOptions(cmd *cobra.Command, refreshOAuth bool) (client.Options opts.OAuthAccessToken = profile.AccessToken() case hasProfile && profile.APIKey != "": opts.APIKey = profile.APIKey - // Route the resolved profile through the SDK (WithProfile) so an explicit - // selection replaces the config's current_profile instead of inheriting - // its tenant/base URL. APIKey is kept for the raw-HTTP helpers. - opts.ProfileName = profileName } if cfgErr != nil && opts.APIKey == "" { return opts, cfgErr diff --git a/internal/cmdutil/resolve_test.go b/internal/cmdutil/resolve_test.go index 7dacf5e..ba7ec5a 100644 --- a/internal/cmdutil/resolve_test.go +++ b/internal/cmdutil/resolve_test.go @@ -205,44 +205,6 @@ func TestResolveClientOptions_ProfileFlagSetsProfileName(t *testing.T) { } } -func TestResolveClientOptions_APIKeyProfileSetsProfileName(t *testing.T) { - path := filepath.Join(t.TempDir(), "config.json") - t.Setenv("LANGSMITH_CONFIG_FILE", path) - t.Setenv("LANGSMITH_API_KEY", "") - t.Setenv("LANGSMITH_ENDPOINT", "") - if err := os.WriteFile(path, []byte(`{ - "current_profile": "prod", - "profiles": { - "prod": { - "api_key": "prod-api-key", - "workspace_id": "ws-prod" - }, - "aws": { - "api_key": "aws-api-key" - } - } -} -`), 0600); err != nil { - t.Fatal(err) - } - - cmd := newTestCmd() - _ = cmd.PersistentFlags().Set("profile", "aws") - opts, err := ResolveClientOptions(cmd, false) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if opts.APIKey != "aws-api-key" { - t.Fatalf("expected profile api key, got %q", opts.APIKey) - } - // An api-key profile must route through WithProfile too, so it replaces - // current_profile and clears the inherited tenant. This resolver (used by the - // api/sandbox/ssh subcommands) previously omitted ProfileName here. - if opts.ProfileName != "aws" { - t.Fatalf("expected ProfileName=aws, got %q", opts.ProfileName) - } -} - func TestResolveClientOptions_WorkspaceFlagOverridesEnvAndProfile(t *testing.T) { path := filepath.Join(t.TempDir(), "config.json") t.Setenv("LANGSMITH_CONFIG_FILE", path) diff --git a/internal/extract/extract.go b/internal/extract/extract.go index c0dc1e4..5e63ed9 100644 --- a/internal/extract/extract.go +++ b/internal/extract/extract.go @@ -70,7 +70,6 @@ func ExtractRun(run langsmith.RunSchema, includeMetadata, includeIO, includeFeed result["status"] = run.Status result["duration_ms"] = durationMs - result["first_token_time"] = formatTimeNullable(run.FirstTokenTime) result["custom_metadata"] = customMetadata result["token_usage"] = tokenUsage result["costs"] = costs @@ -81,7 +80,6 @@ func ExtractRun(run langsmith.RunSchema, includeMetadata, includeIO, includeFeed result["inputs"] = nilIfEmptyMap(run.Inputs) result["outputs"] = nilIfEmptyMap(run.Outputs) result["error"] = nilIfEmpty(run.Error) - result["events"] = nilIfEmptyEvents(run.Events) } if includeFeedback { @@ -131,13 +129,6 @@ func nilIfEmptyMap(m map[string]any) any { return m } -func nilIfEmptyEvents(events []map[string]any) any { - if len(events) == 0 { - return nil - } - return events -} - // FormatDurationHuman formats milliseconds as human-readable string. func FormatDurationHuman(ms int64) string { if ms < 1000 { diff --git a/internal/extract/extract_test.go b/internal/extract/extract_test.go index 20060c7..79c3974 100644 --- a/internal/extract/extract_test.go +++ b/internal/extract/extract_test.go @@ -51,7 +51,6 @@ func TestExtractRunBase(t *testing.T) { func TestExtractRunWithMetadata(t *testing.T) { start := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC) end := time.Date(2024, 1, 15, 10, 30, 5, 0, time.UTC) - firstToken := time.Date(2024, 1, 15, 10, 30, 1, 0, time.UTC) run := langsmith.RunSchema{ ID: "run-123", @@ -60,7 +59,6 @@ func TestExtractRunWithMetadata(t *testing.T) { RunType: "llm", StartTime: start, EndTime: end, - FirstTokenTime: firstToken, Status: "success", PromptTokens: 100, CompletionTokens: 50, @@ -78,10 +76,6 @@ func TestExtractRunWithMetadata(t *testing.T) { t.Errorf("expected status=success, got %v", result["status"]) } - if result["first_token_time"] != firstToken.Format(time.RFC3339Nano) { - t.Errorf("expected first_token_time=%s, got %v", firstToken.Format(time.RFC3339Nano), result["first_token_time"]) - } - durationMs, ok := result["duration_ms"].(int64) if !ok || durationMs != 5000 { t.Errorf("expected duration_ms=5000, got %v", result["duration_ms"]) @@ -121,7 +115,6 @@ func TestExtractRunWithIO(t *testing.T) { Inputs: map[string]any{"query": "hello"}, Outputs: map[string]any{"response": "world"}, Error: "some error", - Events: []map[string]interface{}{{"name": "new_token", "kwargs": map[string]interface{}{"token": "hi"}}}, } result := ExtractRun(run, false, true, false) @@ -139,30 +132,6 @@ func TestExtractRunWithIO(t *testing.T) { if result["error"] != "some error" { t.Errorf("expected error='some error', got %v", result["error"]) } - - events, ok := result["events"].([]map[string]interface{}) - if !ok { - t.Fatalf("expected events to be []map[string]interface{}, got %T", result["events"]) - } - if len(events) != 1 || events[0]["name"] != "new_token" { - t.Errorf("expected one new_token event, got %v", events) - } -} - -func TestExtractRunWithIOEmptyEvents(t *testing.T) { - run := langsmith.RunSchema{ - ID: "run-123", - TraceID: "trace-456", - Name: "ChatOpenAI", - RunType: "llm", - StartTime: time.Now(), - } - - result := ExtractRun(run, false, true, false) - - if result["events"] != nil { - t.Errorf("expected events=nil for run with no events, got %v", result["events"]) - } } func TestExtractRunNilParent(t *testing.T) { diff --git a/scripts/install.sh b/scripts/install.sh index 284ff6c..57dc130 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -39,29 +39,9 @@ fi # Get version if [ -z "$VERSION" ]; then - API_URL="https://api.github.com/repos/${REPO}/releases/latest" - - fetch_latest() { - if [ -n "$GITHUB_TOKEN" ]; then - curl -fsSL -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_TOKEN" "$API_URL" - else - curl -fsSL -H "Accept: application/vnd.github+json" "$API_URL" - fi - } - - VERSION="" - i=1 - while [ "$i" -le 3 ]; do - VERSION="$(fetch_latest | grep '"tag_name"' | sed -E 's/.*"tag_name": *"([^"]+)".*/\1/')" - [ -n "$VERSION" ] && break - [ "$i" -lt 3 ] && sleep "$i" - i=$((i + 1)) - done - + VERSION="$(curl -sSL "https://api.github.com/repos/${REPO}/releases/latest" | grep '"tag_name"' | sed -E 's/.*"tag_name": *"([^"]+)".*/\1/')" if [ -z "$VERSION" ]; then - echo "Failed to determine latest version (GitHub API unreachable or rate-limited)." >&2 - echo "Set GITHUB_TOKEN to raise the rate limit, or set VERSION to install a specific release." >&2 + echo "Failed to determine latest version" >&2 exit 1 fi fi