From 1a4e9093edde4449aa0bae5c51461a84dd3f5eb3 Mon Sep 17 00:00:00 2001 From: Ramon Nogueira Date: Mon, 15 Jun 2026 16:02:02 -0400 Subject: [PATCH 1/9] chore(sdks): include vendored SDK GitHub workflows in copybara [INF-0000] [copybara] (#27446) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop excluding `.github/**` from the Copybara workflows so the vendored SDKs' GitHub Actions are actually imported (and managed in both directions). - Removed the `.github/**` exclusion from both the **vendor** (`origin_files`) and **export** (`destination_files`) globs for the cli, go, and terraform SDKs. This monorepo now imports each SDK's `.github/` and is authoritative over it on export. - Dropped the now-dead cli `core.replace` (RSA-key obfuscation): the OSS repo already stores the obfuscated split form, so the transform was a guaranteed no-op and broke `vendor_cli`. - Re-vendored all three SDKs from their OSS `main`. Beyond the new `.github/workflows`, this pulls in accumulated forward-sync drift (verified as real upstream commits, not local edits) — e.g. go's `datasetplaygroundexperiment.go` deletion from langsmith-go `#105`, and terraform's `docs/`/`examples/`/`templates/`. `[copybara]` is in the title because the change touches `sdks/go/`, which `check_sdk_copybara.yml` gates. ## Self-Hosted Release Note none ## Test Plan - [x] none GitOrigin-RevId: db2713ebcf6b634274bb91055dae87f50e29dfdf --- .github/workflows/ci.yml | 3 - .github/workflows/sync-rc-branches.yml | 65 ++++++ README.md | 24 +-- go.mod | 34 +-- go.sum | 72 +++---- internal/client/client.go | 14 +- internal/client/client_test.go | 59 +----- internal/cmd/filters.go | 7 - internal/cmd/helpers.go | 262 +----------------------- internal/cmd/helpers_test.go | 110 ---------- internal/cmd/hub.go | 1 - internal/cmd/hub_list.go | 10 +- internal/cmd/hub_list_test.go | 15 +- internal/cmd/hub_repo_sdk.go | 1 - internal/cmd/install_method.go | 101 --------- internal/cmd/install_method_test.go | 233 --------------------- internal/cmd/message.go | 253 ++--------------------- internal/cmd/message_trajectory_test.go | 177 ---------------- internal/cmd/root.go | 4 - internal/cmd/root_test.go | 55 ----- internal/cmd/run.go | 92 +++------ internal/cmd/sandbox_box.go | 2 +- internal/cmd/sandbox_test.go | 6 +- internal/cmd/thread.go | 8 +- internal/cmd/trace.go | 53 +++-- internal/cmd/trace_stats.go | 16 +- internal/cmd/update.go | 63 ++---- internal/cmd/update_test.go | 6 +- internal/cmdutil/resolve.go | 4 - internal/cmdutil/resolve_test.go | 38 ---- internal/extract/extract.go | 24 +-- internal/extract/extract_test.go | 37 +--- scripts/install.sh | 24 +-- 33 files changed, 268 insertions(+), 1605 deletions(-) create mode 100644 .github/workflows/sync-rc-branches.yml delete mode 100644 internal/cmd/install_method.go delete mode 100644 internal/cmd/install_method_test.go delete mode 100644 internal/cmd/message_trajectory_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ad39e9..1f0db1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,9 +52,6 @@ jobs: os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/sync-rc-branches.yml b/.github/workflows/sync-rc-branches.yml new file mode 100644 index 0000000..fb6b10d --- /dev/null +++ b/.github/workflows/sync-rc-branches.yml @@ -0,0 +1,65 @@ +name: Sync RC branches with upstream + +# Keeps the engine-facing RC branches in sync with their upstream integration +# branches while the v2 runs-query work is consumed pre-release: +# - langsmith-go rc/runs-v2-engine <- next (custom code lives off `next`) +# - langsmith-cli rc/runs-v2-engine <- main +# +# Merges (never rebases/force-pushes) so engine's go.mod pin to the RC HEAD +# stays valid. On conflict it leaves the last-good RC untouched and opens a +# single deduped issue for manual resolution. +# +# Requires a secret RC_SYNC_TOKEN with contents:write + issues:write on BOTH +# repos. The default GITHUB_TOKEN is repo-scoped and cannot push cross-repo. + +permissions: {} + +on: + schedule: + - cron: '23 */4 * * *' # every 4 hours + workflow_dispatch: {} + +concurrency: + group: sync-rc-runs-v2-engine + cancel-in-progress: false + +jobs: + sync: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { repo: langchain-ai/langsmith-go, upstream: next, rc: rc/runs-v2-engine } + - { repo: langchain-ai/langsmith-cli, upstream: main, rc: rc/runs-v2-engine } + steps: + - name: Merge ${{ matrix.upstream }} into ${{ matrix.rc }} (${{ matrix.repo }}) + env: + GH_TOKEN: ${{ secrets.RC_SYNC_TOKEN }} + REPO: ${{ matrix.repo }} + UPSTREAM: ${{ matrix.upstream }} + RC: ${{ matrix.rc }} + run: | + set -euo pipefail + git config --global user.name "langsmith-rc-sync[bot]" + git config --global user.email "rc-sync@users.noreply.github.com" + + git clone --no-tags "https://x-access-token:${GH_TOKEN}@github.com/${REPO}.git" repo + cd repo + git fetch origin "$UPSTREAM" "$RC" + git checkout "$RC" + + if git merge --no-edit "origin/$UPSTREAM"; then + git push origin "HEAD:$RC" + echo "Synced $REPO: $UPSTREAM -> $RC" + else + git merge --abort + echo "::warning::Merge conflict syncing $UPSTREAM into $RC in $REPO" + TITLE="RC sync conflict: $UPSTREAM -> $RC" + BODY="Automated RC sync could not merge \`$UPSTREAM\` into \`$RC\`. Resolve manually: \`git checkout $RC && git merge origin/$UPSTREAM\`, then push." + EXISTING="$(gh issue list --repo "$REPO" --state open --search "$TITLE in:title" --json number --jq '.[0].number // empty')" + if [ -z "$EXISTING" ]; then + gh issue create --repo "$REPO" --title "$TITLE" --body "$BODY" + fi + exit 1 + fi diff --git a/README.md b/README.md index 600dfb9..8c5a9d7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # langsmith-cli +> [!IMPORTANT] +> **Alpha** — This CLI is under active development. Commands, flags, and output schemas may change between releases. Feedback and bug reports welcome via [GitHub Issues](https://github.com/langchain-ai/langsmith-cli/issues). + An agent-first CLI for querying and managing [LangSmith](https://smith.langchain.com) resources. Built for AI coding agents (deepagents, Claude Code, Cursor, etc.) and developers who need fast, scriptable access to projects, traces, runs, datasets, evaluators, experiments, and threads. @@ -168,13 +171,6 @@ langsmith run get --full langsmith run export llm_calls.jsonl --project my-app --run-type llm --full ``` -> **For agents querying runs:** prefer `--version v2` first (SmithDB-backed; faster on tenants that are rolled out). If the call fails with a 4xx (typically 403, 404, or 422), retry the **same command without** `--version` to fall back to v1. Example: -> -> ```bash -> langsmith run list --project my-app --version v2 \ -> || langsmith run list --project my-app -> ``` - ### `thread` — Query conversation threads A thread groups multiple root runs sharing a thread_id (multi-turn conversations). @@ -297,7 +293,6 @@ langsmith hub pull acme/my-skill:production --dir ./out # Discover, inspect, delete langsmith hub list --type skill --query foo -langsmith hub list --type skill --source external langsmith hub get acme/my-skill langsmith hub delete acme/my-skill --yes ``` @@ -316,19 +311,6 @@ langsmith self-update --dry-run langsmith self-update ``` -If langsmith was installed through a package manager, `self-update` won't replace the -binary in place — it points you at the right command instead: - -| Installed via | Update with | -| --- | --- | -| Homebrew | `brew upgrade langchain-ai/tap/langsmith-cli` | -| Scoop | `scoop update langsmith-cli` | -| `go install` | `go install github.com/langchain-ai/langsmith-cli/cmd/langsmith@latest` | - -Installs from the `install.sh`/`install.ps1` scripts or a direct GitHub Releases download -are updated in place as usual. Pass `--force` to update in place regardless of how -langsmith was installed. - ### `trace setup` — Trace coding agents to LangSmith Configure Claude Code or Codex to send full-content traces (prompts, responses, tool diff --git a/go.mod b/go.mod index c75e138..30bdbf3 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,14 @@ go 1.25.0 require ( github.com/google/uuid v1.6.0 github.com/itchyny/gojq v0.12.15 - github.com/langchain-ai/langsmith-go v0.18.2 + github.com/langchain-ai/langsmith-go v0.10.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pelletier/go-toml/v2 v2.2.4 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.11.1 github.com/xlab/treeprint v1.2.0 - golang.org/x/net v0.55.0 - golang.org/x/term v0.43.0 + golang.org/x/net v0.52.0 + golang.org/x/term v0.41.0 ) require ( @@ -21,31 +21,31 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect - github.com/klauspost/compress v1.19.0 // indirect + github.com/klauspost/compress v1.18.4 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tidwall/gjson v1.19.0 // indirect + github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel v1.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 // indirect - go.opentelemetry.io/otel/metric v1.44.0 // indirect - go.opentelemetry.io/otel/sdk v1.44.0 // indirect - go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.81.1 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/grpc v1.80.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 942172c..9711832 100644 --- a/go.sum +++ b/go.sum @@ -17,22 +17,22 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/itchyny/gojq v0.12.15 h1:WC1Nxbx4Ifw5U2oQWACYz32JK8G9qxNtHzrvW4KEcqI= github.com/itchyny/gojq v0.12.15/go.mod h1:uWAHCbCIla1jiNxmeT5/B5mOjSdfkCq6p8vxWg+BM10= github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= -github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= -github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= +github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/langchain-ai/langsmith-go v0.18.2 h1:/Hs4KoJ2nqsPahFZz47RpXmdPZL/NkL89KC9mj6rSEA= -github.com/langchain-ai/langsmith-go v0.18.2/go.mod h1:2NXq2u8YbQkpP9BaKG7RKs0g0ZoCeqVzAZsBu7nr58Y= +github.com/langchain-ai/langsmith-go v0.10.0 h1:crrioqgBBKY+rTYhFW0NujhxBUoq6iARv12bP9xR7mg= +github.com/langchain-ai/langsmith-go v0.10.0/go.mod h1:hUFPP9siu6sGSodcZo/c3uUG5xcXRdqZ8gN899R1OyE= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -57,8 +57,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= -github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -70,40 +70,40 @@ github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= -google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/client/client.go b/internal/client/client.go index 4f13138..2da3119 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -58,17 +58,13 @@ func NewWithOptions(options Options) *Client { normalized := NormalizeURL(options.APIURL) var opts []option.RequestOption - // Auth precedence mirrors resolveClientOptions. A resolved profile is routed - // through WithProfile so an explicit selection replaces the config's - // current_profile (clearing any inherited tenant/base URL); WithProfile - // supplies the profile's own auth (API key or OAuth). A profile-less explicit - // key or bearer is applied directly. - switch { - case options.ProfileName != "": + if options.ProfileName != "" && options.APIKey == "" { opts = append(opts, langsmith.WithProfile(options.ProfileName)) - case options.APIKey != "": + } + if options.APIKey != "" { opts = append(opts, option.WithAPIKey(options.APIKey)) - case options.OAuthAccessToken != "": + } + if options.OAuthAccessToken != "" && options.ProfileName == "" && options.APIKey == "" { opts = append(opts, option.WithHeader("authorization", "Bearer "+options.OAuthAccessToken)) } // Only set base URL if not the default (the SDK reads LANGSMITH_ENDPOINT too). diff --git a/internal/client/client_test.go b/internal/client/client_test.go index 41aeb26..241b72d 100644 --- a/internal/client/client_test.go +++ b/internal/client/client_test.go @@ -136,52 +136,7 @@ func TestNewWithOptions_ProfileNameDelegatesAuthToSDK(t *testing.T) { } } -func TestNewWithOptions_APIKeyProfileRoutesThroughSDK(t *testing.T) { - var gotAPIKey string - var gotAuth string - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotAPIKey = r.Header.Get("X-API-Key") - gotAuth = r.Header.Get("Authorization") - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]any{"version": "test"}) - })) - defer ts.Close() - - path := filepath.Join(t.TempDir(), "config.json") - t.Setenv("LANGSMITH_CONFIG_FILE", path) - t.Setenv("LANGSMITH_PROFILE", "") - t.Setenv("LANGSMITH_API_KEY", "") - if err := os.WriteFile(path, []byte(`{ - "profiles": { - "prod": { - "api_key": "prod-api-key" - } - } -} -`), 0600); err != nil { - t.Fatal(err) - } - - // An api-key profile carries both ProfileName and its own APIKey (as - // resolveClientOptions produces). It routes through WithProfile: the profile's - // key is sent, no bearer. - c := NewWithOptions(Options{ - APIKey: "prod-api-key", - ProfileName: "prod", - APIURL: ts.URL, - }) - if _, err := c.SDK.Info.List(context.Background()); err != nil { - t.Fatal(err) - } - if gotAPIKey != "prod-api-key" { - t.Fatalf("expected profile api key, got %q", gotAPIKey) - } - if gotAuth != "" { - t.Fatalf("expected no bearer for an api-key profile, got %q", gotAuth) - } -} - -func TestNewWithOptions_ProfileNameTakesPrecedenceOverExplicitAPIKey(t *testing.T) { +func TestNewWithOptions_APIKeyOverridesProfileName(t *testing.T) { var gotAPIKey string var gotAuth string ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -209,10 +164,6 @@ func TestNewWithOptions_ProfileNameTakesPrecedenceOverExplicitAPIKey(t *testing. t.Fatal(err) } - // resolveClientOptions never produces this combo, but NewWithOptions is - // exported: when a caller passes an explicit APIKey alongside a ProfileName, - // the profile wins (routes through WithProfile) and the explicit key is not - // sent for SDK calls. c := NewWithOptions(Options{ APIKey: "explicit-key", ProfileName: "prod", @@ -221,11 +172,11 @@ func TestNewWithOptions_ProfileNameTakesPrecedenceOverExplicitAPIKey(t *testing. if _, err := c.SDK.Info.List(context.Background()); err != nil { t.Fatal(err) } - if gotAuth != "Bearer prod-access-token" { - t.Fatalf("expected profile OAuth bearer to win, got %q", gotAuth) + if gotAPIKey != "explicit-key" { + t.Fatalf("expected explicit API key, got %q", gotAPIKey) } - if gotAPIKey != "" { - t.Fatalf("expected explicit API key not sent for SDK calls, got %q", gotAPIKey) + if gotAuth != "" { + t.Fatalf("expected no profile bearer auth when API key wins, got %q", gotAuth) } } diff --git a/internal/cmd/filters.go b/internal/cmd/filters.go index 98ffb8c..a8ae94b 100644 --- a/internal/cmd/filters.go +++ b/internal/cmd/filters.go @@ -15,7 +15,6 @@ type FilterFlags struct { TraceIDs string Limit int Project string - ProjectID string LastNMinutes int Since string Before string @@ -30,7 +29,6 @@ type FilterFlags struct { Tags string Metadata string RawFilter string - Version string } // addCommonFilterFlags attaches shared filter flags to a command. @@ -57,11 +55,6 @@ func addCommonFilterFlags(cmd *cobra.Command, f *FilterFlags, includeRunType boo } } -// addVersionFlag attaches the --version flag for selecting the runs query backend. -func addVersionFlag(cmd *cobra.Command, f *FilterFlags) { - cmd.Flags().StringVar(&f.Version, "version", "", `Query API version: "" (v1, default) or "v2" (SmithDB)`) -} - // resolveStartTime returns the start time for a query. // Priority: lastNMinutes > since > default (7 days ago). func resolveStartTime(since string, lastNMinutes int) time.Time { diff --git a/internal/cmd/helpers.go b/internal/cmd/helpers.go index 282872d..8eccdff 100644 --- a/internal/cmd/helpers.go +++ b/internal/cmd/helpers.go @@ -2,9 +2,7 @@ package cmd import ( "context" - "encoding/json" "fmt" - "strings" "time" "github.com/langchain-ai/langsmith-cli/internal/client" @@ -15,34 +13,15 @@ import ( "github.com/google/uuid" ) -// resolveSessionID resolves the target session (project) UUID for a command from -// either an explicit project ID or a project name. Precedence: -// -// --project-id (an explicit session UUID; takes precedence) → -// --project / $LANGSMITH_PROJECT (a project name, looked up via the API). -// -// projectID, when set, must be a well-formed UUID and is returned as-is without a -// name lookup (saving a Sessions.List round-trip). cmdName is used only to build -// the "required" error message. -func resolveSessionID(ctx context.Context, c *client.Client, projectName, projectID, cmdName string) (string, error) { - if projectID != "" { - if _, err := uuid.Parse(projectID); err != nil { - return "", fmt.Errorf("invalid --project-id %q: must be a project (session) UUID", projectID) - } - return projectID, nil - } - name := ResolveProject(projectName) - if name == "" { - return "", fmt.Errorf("--project or --project-id is required for %s (or set LANGSMITH_PROJECT)", cmdName) - } - return c.ResolveSessionID(ctx, name) -} - -// queryRuns queries runs with the given params, scoped to sessionID when non-empty. -// The caller is responsible for resolving sessionID (see resolveSessionID). +// queryRuns queries runs with the given params and optional session resolution. // minTokens > 0 enables client-side filtering by total_tokens (not supported server-side). -func queryRuns(ctx context.Context, c *client.Client, params langsmith.RunQueryParams, sessionID string, limit int, minTokens int) ([]langsmith.RunSchema, error) { - if sessionID != "" { +func queryRuns(ctx context.Context, c *client.Client, params langsmith.RunQueryParams, projectName string, limit int, minTokens int) ([]langsmith.RunSchema, error) { + // Resolve project name → session ID + if projectName != "" { + sessionID, err := c.ResolveSessionID(ctx, projectName) + if err != nil { + return nil, err + } params.Session = langsmith.F([]string{sessionID}) } @@ -68,234 +47,15 @@ func queryRuns(ctx context.Context, c *client.Client, params langsmith.RunQueryP } // Check for next cursor - if resp.Cursors.Next == "" || remaining <= 0 { + if resp.Cursors == nil || resp.Cursors["next"] == "" || remaining <= 0 { break } - params.Cursor = langsmith.F(resp.Cursors.Next) + params.Cursor = langsmith.F(resp.Cursors["next"]) } return allRuns, nil } -// queryRunsV2 queries runs against the v2 (SmithDB) endpoint, resolves the -// project name to a session UUID, paginates on next_cursor, and normalizes -// each v2 Run back into v1's RunSchema so downstream rendering is unchanged. -// minTokens > 0 enables client-side filtering by total_tokens. -func queryRunsV2(ctx context.Context, c *client.Client, params langsmith.RunQueryV2Params, sessionID string, limit int, minTokens int) ([]langsmith.RunSchema, error) { - if sessionID != "" { - params.ProjectIDs = langsmith.F([]string{sessionID}) - } - - var allRuns []langsmith.RunSchema - - iter := c.SDK.Runs.QueryV2AutoPaging(ctx, params) - for iter.Next() { - if len(allRuns) >= limit { - break - } - run := runV2ToSchema(iter.Current()) - if minTokens > 0 && run.TotalTokens < int64(minTokens) { - continue - } - allRuns = append(allRuns, run) - } - if err := iter.Err(); err != nil { - return nil, fmt.Errorf("querying runs (v2): %w", err) - } - - return allRuns, nil -} - -// buildRunQueryV2Params translates FilterFlags into a v2 query body. The -// project name is left for queryRunsV2 to resolve into ProjectIDs. -func buildRunQueryV2Params(f *FilterFlags, isRoot bool, defaultLimit int) langsmith.RunQueryV2Params { - var params langsmith.RunQueryV2Params - - limit := defaultLimit - if f.Limit > 0 { - limit = f.Limit - } - params.PageSize = langsmith.F(int64(limit)) - - if isRoot { - params.IsRoot = langsmith.F(true) - } - - params.MinStartTime = langsmith.F(resolveStartTime(f.Since, f.LastNMinutes)) - if f.Before != "" { - t, err := time.Parse(time.RFC3339, f.Before) - if err != nil { - t, err = time.Parse("2006-01-02T15:04:05", f.Before) - if err != nil { - ExitErrorf("invalid --before timestamp: %s", f.Before) - } - } - params.MaxStartTime = langsmith.F(t.UTC()) - } - - if f.RunType != "" { - params.RunType = langsmith.F(langsmith.RunQueryV2ParamsRunType(strings.ToUpper(f.RunType))) - } - - if f.ErrorFlag { - params.HasError = langsmith.F(true) - } else if f.NoErrorFlag { - params.HasError = langsmith.F(false) - } - - if f.TraceIDs != "" { - ids := splitTrim(f.TraceIDs) - if len(ids) == 1 { - params.TraceID = langsmith.F(ids[0]) - } - } - - if s := buildFilterDSL(f); s != "" { - params.Filter = langsmith.F(s) - } - - return params -} - -// buildRunSelectV2 returns the v2 select-field set covering the same base -// fields used by the downstream RunSchema pipeline plus the optional groups -// requested by the include flags. -func buildRunSelectV2(includeIO, includeFeedback bool) []langsmith.RunQueryV2ParamsSelect { - fields := []langsmith.RunQueryV2ParamsSelect{ - langsmith.RunQueryV2ParamsSelectID, - langsmith.RunQueryV2ParamsSelectTraceID, - langsmith.RunQueryV2ParamsSelectName, - langsmith.RunQueryV2ParamsSelectRunType, - langsmith.RunQueryV2ParamsSelectStartTime, - langsmith.RunQueryV2ParamsSelectEndTime, - langsmith.RunQueryV2ParamsSelectParentRunIDs, - langsmith.RunQueryV2ParamsSelectProjectID, - langsmith.RunQueryV2ParamsSelectDottedOrder, - langsmith.RunQueryV2ParamsSelectIsRoot, - langsmith.RunQueryV2ParamsSelectExtra, - langsmith.RunQueryV2ParamsSelectMetadata, - langsmith.RunQueryV2ParamsSelectTags, - langsmith.RunQueryV2ParamsSelectPromptTokens, - langsmith.RunQueryV2ParamsSelectCompletionTokens, - langsmith.RunQueryV2ParamsSelectTotalTokens, - langsmith.RunQueryV2ParamsSelectPromptCost, - langsmith.RunQueryV2ParamsSelectCompletionCost, - langsmith.RunQueryV2ParamsSelectTotalCost, - langsmith.RunQueryV2ParamsSelectLatencySeconds, - langsmith.RunQueryV2ParamsSelectAppPath, - langsmith.RunQueryV2ParamsSelectFirstTokenTime, - } - if includeIO { - fields = append(fields, - langsmith.RunQueryV2ParamsSelectInputs, - langsmith.RunQueryV2ParamsSelectOutputs, - langsmith.RunQueryV2ParamsSelectError, - langsmith.RunQueryV2ParamsSelectEvents, - ) - } - if includeFeedback { - fields = append(fields, langsmith.RunQueryV2ParamsSelectFeedbackStats) - } - return fields -} - -// runV2ToSchema converts a v2 Run into the legacy v1 RunSchema shape so the -// existing extract/output pipeline can consume it unchanged. -func runV2ToSchema(r langsmith.Run) langsmith.RunSchema { - extra := asMap(r.Extra) - if md := asMap(r.Metadata); md != nil { - if extra == nil { - extra = map[string]interface{}{} - } - if _, ok := extra["metadata"]; !ok { - extra["metadata"] = md - } - } - - out := langsmith.RunSchema{ - ID: r.ID, - TraceID: r.TraceID, - Name: r.Name, - RunType: langsmith.RunTypeEnum(strings.ToLower(string(r.RunType))), - StartTime: r.StartTime, - EndTime: r.EndTime, - FirstTokenTime: r.FirstTokenTime, - SessionID: r.ProjectID, - DottedOrder: r.DottedOrder, - Tags: r.Tags, - ThreadID: r.ThreadID, - AppPath: r.AppPath, - TotalTokens: r.TotalTokens, - PromptTokens: r.PromptTokens, - CompletionTokens: r.CompletionTokens, - TotalCost: r.TotalCost, - PromptCost: r.PromptCost, - CompletionCost: r.CompletionCost, - Error: r.Error, - InputsPreview: r.InputsPreview, - OutputsPreview: r.OutputsPreview, - ReferenceExampleID: r.ReferenceExampleID, - ReferenceDatasetID: r.ReferenceDatasetID, - PriceModelID: r.PriceModelID, - InDataset: r.IsInDataset, - Inputs: asMap(r.Inputs), - Outputs: asMap(r.Outputs), - Extra: extra, - FeedbackStats: convertFeedbackStats(r.FeedbackStats), - Events: convertEvents(r.Events), - } - if len(r.ParentRunIDs) > 0 { - out.ParentRunIDs = r.ParentRunIDs - out.ParentRunID = r.ParentRunIDs[len(r.ParentRunIDs)-1] - } - return out -} - -// asMap returns v as a map[string]interface{} when it is one, else nil. -func asMap(v interface{}) map[string]interface{} { - m, ok := v.(map[string]interface{}) - if !ok { - return nil - } - return m -} - -// convertFeedbackStats round-trips the generated feedback-stats shape into the -// loosely-typed map the downstream pipeline expects. Returns nil on empty input -// or any marshalling error. -func convertFeedbackStats(stats map[string]langsmith.RunFeedbackStat) map[string]map[string]interface{} { - if len(stats) == 0 { - return nil - } - raw, err := json.Marshal(stats) - if err != nil { - return nil - } - var m map[string]map[string]interface{} - if err := json.Unmarshal(raw, &m); err != nil { - return nil - } - return m -} - -// convertEvents round-trips the generated v2 event shape into the -// loosely-typed maps the legacy RunSchema.Events field expects. Returns nil on -// empty input or any marshalling error. -func convertEvents(events []langsmith.RunEvent) []map[string]interface{} { - if len(events) == 0 { - return nil - } - raw, err := json.Marshal(events) - if err != nil { - return nil - } - var m []map[string]interface{} - if err := json.Unmarshal(raw, &m); err != nil { - return nil - } - return m -} - // buildRunSelect returns the Select fields needed for the given include flags. // Returns nil when neither IO nor feedback is requested, letting the API use its defaults. // When set, includes all base/metadata fields so they aren't stripped from the response. @@ -316,7 +76,6 @@ func buildRunSelect(includeIO, includeFeedback bool) []langsmith.RunQueryParamsS langsmith.RunQueryParamsSelectStatus, // Metadata fields langsmith.RunQueryParamsSelectExtra, - langsmith.RunQueryParamsSelectFirstTokenTime, langsmith.RunQueryParamsSelectPromptTokens, langsmith.RunQueryParamsSelectCompletionTokens, langsmith.RunQueryParamsSelectTotalTokens, @@ -331,7 +90,6 @@ func buildRunSelect(includeIO, includeFeedback bool) []langsmith.RunQueryParamsS langsmith.RunQueryParamsSelectInputs, langsmith.RunQueryParamsSelectOutputs, langsmith.RunQueryParamsSelectError, - langsmith.RunQueryParamsSelectEvents, ) } diff --git a/internal/cmd/helpers_test.go b/internal/cmd/helpers_test.go index 3744954..631ecdc 100644 --- a/internal/cmd/helpers_test.go +++ b/internal/cmd/helpers_test.go @@ -1,8 +1,6 @@ package cmd import ( - "context" - "strings" "testing" "time" @@ -10,48 +8,6 @@ import ( langsmith "github.com/langchain-ai/langsmith-go" ) -// ---------- resolveSessionID ---------- -// -// These cases all resolve (or error) before any API call, so a nil client is -// safe. The project-name → session-ID lookup path is exercised via the -// command-level tests that stub the client. - -func TestResolveSessionID_ProjectIDTakesPrecedence(t *testing.T) { - // A valid --project-id is returned as-is, even when a project name is set - // via --project / $LANGSMITH_PROJECT — without any name lookup. - t.Setenv("LANGSMITH_PROJECT", "some-project-name") - const id = "3f2504e0-4f89-41d3-9a0c-0305e82c3301" - - got, err := resolveSessionID(context.Background(), nil, "another-name", id, "trace get") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if got != id { - t.Errorf("expected session id %q returned verbatim, got %q", id, got) - } -} - -func TestResolveSessionID_InvalidProjectID(t *testing.T) { - _, err := resolveSessionID(context.Background(), nil, "", "not-a-uuid", "trace get") - if err == nil { - t.Fatal("expected error for malformed --project-id") - } - if !strings.Contains(err.Error(), "--project-id") { - t.Errorf("error should mention --project-id, got %q", err.Error()) - } -} - -func TestResolveSessionID_NeitherProvided(t *testing.T) { - t.Setenv("LANGSMITH_PROJECT", "") - _, err := resolveSessionID(context.Background(), nil, "", "", "trace stats") - if err == nil { - t.Fatal("expected error when neither --project nor --project-id is provided") - } - if !strings.Contains(err.Error(), "trace stats") { - t.Errorf("error should name the command, got %q", err.Error()) - } -} - // ---------- formatTimedelta ---------- func TestFormatTimedelta_Milliseconds(t *testing.T) { @@ -404,19 +360,6 @@ func TestBuildRunSelect_IncludesMetadataFields(t *testing.T) { } } -func TestBuildRunSelect_IncludesFirstTokenTimeAndEvents(t *testing.T) { - // first_token_time and events are native Run fields that --full is - // documented to return; a prior version of this select list silently - // dropped both even when --include-io/--include-metadata were requested. - has := selectSet(buildRunSelect(true, true)) - if !has[langsmith.RunQueryParamsSelectFirstTokenTime] { - t.Error("missing first_token_time field") - } - if !has[langsmith.RunQueryParamsSelectEvents] { - t.Error("missing events field (should be requested alongside inputs/outputs/error)") - } -} - // selectSet converts a slice to a set for easy lookup. func selectSet(sel []langsmith.RunQueryParamsSelect) map[langsmith.RunQueryParamsSelect]bool { m := make(map[langsmith.RunQueryParamsSelect]bool, len(sel)) @@ -497,56 +440,3 @@ func TestRunTreeData_AllFields(t *testing.T) { t.Error("unexpected RunTreeData field values") } } - -// ---------- buildRunSelectV2 ---------- - -func TestBuildRunSelectV2_IncludesFirstTokenTimeAndEvents(t *testing.T) { - // Mirrors TestBuildRunSelect_IncludesFirstTokenTimeAndEvents for the v2 - // select builder: first_token_time is a base field always requested, - // events is requested alongside inputs/outputs/error under --include-io. - base := buildRunSelectV2(false, false) - baseHas := map[langsmith.RunQueryV2ParamsSelect]bool{} - for _, f := range base { - baseHas[f] = true - } - if !baseHas[langsmith.RunQueryV2ParamsSelectFirstTokenTime] { - t.Error("missing first_token_time field in base v2 select set") - } - if baseHas[langsmith.RunQueryV2ParamsSelectEvents] { - t.Error("events should not be requested without --include-io") - } - - withIO := buildRunSelectV2(true, false) - ioHas := map[langsmith.RunQueryV2ParamsSelect]bool{} - for _, f := range withIO { - ioHas[f] = true - } - if !ioHas[langsmith.RunQueryV2ParamsSelectEvents] { - t.Error("missing events field when --include-io requested") - } -} - -// ---------- runV2ToSchema ---------- - -func TestRunV2ToSchema_MapsFirstTokenTimeAndEvents(t *testing.T) { - firstToken := time.Date(2026, 1, 15, 10, 30, 1, 0, time.UTC) - v2 := langsmith.Run{ - ID: "run-123", - FirstTokenTime: firstToken, - Events: []langsmith.RunEvent{ - {Name: "new_token", Kwargs: map[string]interface{}{"token": "hi"}}, - }, - } - - out := runV2ToSchema(v2) - - if !out.FirstTokenTime.Equal(firstToken) { - t.Errorf("expected FirstTokenTime=%v, got %v", firstToken, out.FirstTokenTime) - } - if len(out.Events) != 1 { - t.Fatalf("expected 1 event, got %d", len(out.Events)) - } - if out.Events[0]["name"] != "new_token" { - t.Errorf("expected event name=new_token, got %v", out.Events[0]["name"]) - } -} diff --git a/internal/cmd/hub.go b/internal/cmd/hub.go index 3fccbe8..97bb7e7 100644 --- a/internal/cmd/hub.go +++ b/internal/cmd/hub.go @@ -54,7 +54,6 @@ type hubRepo struct { Owner *string `json:"owner"` RepoHandle string `json:"repo_handle"` RepoType string `json:"repo_type"` - Source *string `json:"source"` Description *string `json:"description"` Readme *string `json:"readme"` IsPublic bool `json:"is_public"` diff --git a/internal/cmd/hub_list.go b/internal/cmd/hub_list.go index 1b020e1..645869a 100644 --- a/internal/cmd/hub_list.go +++ b/internal/cmd/hub_list.go @@ -13,7 +13,6 @@ import ( func newHubListCmd() *cobra.Command { var ( repoType string - source string query string publicOnly bool limit int @@ -34,13 +33,7 @@ func newHubListCmd() *cobra.Command { if repoType != "agent" && repoType != "skill" { return fmt.Errorf("--type must be 'agent' or 'skill' when set (got %q)", repoType) } - params.SingleRepoType = langsmith.F(langsmith.RepoListParamsRepoType(repoType)) - } - if source != "" { - if source != "internal" && source != "external" { - return fmt.Errorf("--source must be 'internal' or 'external' when set (got %q)", source) - } - params.Source = langsmith.F(langsmith.RepoListParamsSource(source)) + params.RepoType = langsmith.F(langsmith.RepoListParamsRepoType(repoType)) } if query != "" { params.Query = langsmith.F(query) @@ -95,7 +88,6 @@ func newHubListCmd() *cobra.Command { } cmd.Flags().StringVar(&repoType, "type", "", "Filter by repo type: agent or skill") - cmd.Flags().StringVar(&source, "source", "", "Filter by source: internal or external") cmd.Flags().StringVar(&query, "query", "", "Filter by name substring") cmd.Flags().BoolVar(&publicOnly, "public", false, "Filter by public/private") cmd.Flags().IntVarP(&limit, "limit", "n", 100, "Maximum number of repos to return") diff --git a/internal/cmd/hub_list_test.go b/internal/cmd/hub_list_test.go index 92cbdfe..3d740b6 100644 --- a/internal/cmd/hub_list_test.go +++ b/internal/cmd/hub_list_test.go @@ -19,7 +19,7 @@ func TestHubList_QueryParams(t *testing.T) { captureStdout(t, func() { cmd := newHubCmd() - cmd.SetArgs([]string{"list", "--type", "skill", "--source", "external", "--query", "foo", "--public", "--limit", "10", "--offset", "5"}) + cmd.SetArgs([]string{"list", "--type", "skill", "--query", "foo", "--public", "--limit", "10", "--offset", "5"}) if err := cmd.Execute(); err != nil { t.Fatalf("execute: %v", err) } @@ -27,7 +27,6 @@ func TestHubList_QueryParams(t *testing.T) { checks := map[string]string{ "repo_type": "skill", - "source": "external", "query": "foo", "match_prefix": "true", "is_public": "true", @@ -54,18 +53,6 @@ func TestHubList_RejectsBadType(t *testing.T) { } } -func TestHubList_RejectsBadSource(t *testing.T) { - srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {}) - defer setupTestEnv(t, srv.URL)() - - cmd := newHubCmd() - cmd.SetArgs([]string{"list", "--source", "unknown"}) - err := cmd.Execute() - if err == nil || !strings.Contains(err.Error(), "internal") { - t.Errorf("expected error mentioning valid sources; got %v", err) - } -} - func TestHubList_RendersTable(t *testing.T) { srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/internal/cmd/hub_repo_sdk.go b/internal/cmd/hub_repo_sdk.go index f3b52fd..c074c5b 100644 --- a/internal/cmd/hub_repo_sdk.go +++ b/internal/cmd/hub_repo_sdk.go @@ -48,7 +48,6 @@ func sdkRepoToHubRepo(repo langsmith.RepoWithLookups) hubRepo { Owner: strPtrOrNil(repo.Owner), RepoHandle: repo.RepoHandle, RepoType: string(repo.RepoType), - Source: strPtrOrNil(string(repo.Source)), Description: strPtrOrNil(repo.Description), Readme: strPtrOrNil(repo.Readme), IsPublic: repo.IsPublic, diff --git a/internal/cmd/install_method.go b/internal/cmd/install_method.go deleted file mode 100644 index 987aa39..0000000 --- a/internal/cmd/install_method.go +++ /dev/null @@ -1,101 +0,0 @@ -package cmd - -import ( - "path/filepath" - "strings" -) - -// installMethod identifies how the langsmith binary was installed, which -// determines whether self-update may replace the binary in place. -type installMethod int - -const ( - // methodManaged means self-update owns the binary: install.sh, install.ps1, - // or a manual download from GitHub Releases. In-place replacement is correct. - methodManaged installMethod = iota - // methodHomebrew means the binary lives in a Homebrew Cellar. - methodHomebrew - // methodScoop means the binary was installed by Scoop. - methodScoop - // methodGo means the binary was installed via `go install`. - methodGo - // methodDev means a local/development build with no release version. - methodDev -) - -// externalManagers describes installs owned by a package manager: how to name -// the manager in output and the command the user should run to update. Only the -// methods self-update must defer to appear here — methodManaged and methodDev are -// handled by self-update itself and intentionally have no entry, so membership in -// this map is the single source of truth for "must defer to a package manager". -var externalManagers = map[installMethod]struct { - label string // machine-readable name, for JSON output - display string // human-readable name, for the pretty message - command string // the command the user should run to update -}{ - methodHomebrew: {"homebrew", "Homebrew", "brew upgrade langchain-ai/tap/langsmith-cli"}, - methodScoop: {"scoop", "Scoop", "scoop update langsmith-cli"}, - methodGo: {"go", "`go install`", "go install github.com/langchain-ai/langsmith-cli/cmd/langsmith@latest"}, -} - -// detectInstallMethod classifies the install method from the resolved executable -// path, the build version, and the relevant environment values. It is pure: all -// I/O (resolving the path, reading env) happens in the caller. -// -// - execPath must already be resolved through filepath.EvalSymlinks so that -// Homebrew's bin symlinks point at the Cellar. -// - goos is runtime.GOOS. -// - gobin/gopath/home come from $GOBIN, $GOPATH, and the user's home dir. -func detectInstallMethod(execPath, version, goos, gobin, gopath, home string) installMethod { - // Normalize both separators to "/" so a single "/segment/" check works for - // Windows paths regardless of the OS this code runs on (matters for tests). - lower := strings.ToLower(strings.ReplaceAll(execPath, `\`, "/")) - - if strings.Contains(lower, "/cellar/") { - return methodHomebrew - } - if strings.Contains(lower, "/scoop/") { - return methodScoop - } - - dir := filepath.Dir(execPath) - for _, bin := range goBinDirs(gobin, gopath, home) { - if bin != "" && filepath.Clean(dir) == filepath.Clean(bin) { - return methodGo - } - } - - if version == "dev" { - return methodDev - } - return methodManaged -} - -// goBinDirs returns the candidate directories where `go install` places binaries, -// in precedence order: $GOBIN, then $GOPATH/bin, then $HOME/go/bin (Go's default -// GOPATH when unset). -func goBinDirs(gobin, gopath, home string) []string { - if gobin != "" { - return []string{gobin} - } - if gopath != "" { - // GOPATH may contain multiple entries; go install uses the first. - first := filepath.SplitList(gopath) - if len(first) > 0 && first[0] != "" { - return []string{filepath.Join(first[0], "bin")} - } - } - if home != "" { - return []string{filepath.Join(home, "go", "bin")} - } - return nil -} - -// shouldDeferToManager reports whether self-update must defer to a package -// manager instead of replacing the binary in place. --force overrides it. -// Membership in externalManagers is the source of truth; methodManaged and -// methodDev are not externally managed and so are never deferred. -func shouldDeferToManager(method installMethod, force bool) bool { - _, external := externalManagers[method] - return external && !force -} diff --git a/internal/cmd/install_method_test.go b/internal/cmd/install_method_test.go deleted file mode 100644 index 78aa832..0000000 --- a/internal/cmd/install_method_test.go +++ /dev/null @@ -1,233 +0,0 @@ -package cmd - -import ( - "encoding/json" - "strings" - "testing" -) - -func TestDetectInstallMethod(t *testing.T) { - tests := []struct { - name string - execPath string - version string - goos string - gobin string - gopath string - home string - want installMethod - }{ - { - name: "homebrew apple silicon cellar", - execPath: "/opt/homebrew/Cellar/langsmith-cli/0.2.38/bin/langsmith", - version: "0.2.38", - goos: "darwin", - home: "/Users/me", - want: methodHomebrew, - }, - { - name: "homebrew intel cellar", - execPath: "/usr/local/Cellar/langsmith-cli/0.2.38/bin/langsmith", - version: "0.2.38", - goos: "darwin", - home: "/Users/me", - want: methodHomebrew, - }, - { - name: "homebrew linux cellar", - execPath: "/home/linuxbrew/.linuxbrew/Cellar/langsmith-cli/0.2.38/bin/langsmith", - version: "0.2.38", - goos: "linux", - home: "/home/me", - want: methodHomebrew, - }, - { - name: "scoop apps", - execPath: `C:\Users\me\scoop\apps\langsmith-cli\current\langsmith.exe`, - version: "0.2.38", - goos: "windows", - home: `C:\Users\me`, - want: methodScoop, - }, - { - name: "scoop shims", - execPath: `C:\Users\me\scoop\shims\langsmith.exe`, - version: "0.2.38", - goos: "windows", - home: `C:\Users\me`, - want: methodScoop, - }, - { - name: "go install via GOBIN", - execPath: "/Users/me/dev/gobin/langsmith", - version: "0.2.38", - goos: "darwin", - gobin: "/Users/me/dev/gobin", - home: "/Users/me", - want: methodGo, - }, - { - name: "go install via GOPATH bin", - execPath: "/Users/me/gopath/bin/langsmith", - version: "0.2.38", - goos: "darwin", - gopath: "/Users/me/gopath", - home: "/Users/me", - want: methodGo, - }, - { - name: "go install via default HOME/go/bin", - execPath: "/Users/me/go/bin/langsmith", - version: "0.2.38", - goos: "darwin", - home: "/Users/me", - want: methodGo, - }, - { - name: "go install with dev version still detected as go", - execPath: "/Users/me/go/bin/langsmith", - version: "dev", - goos: "darwin", - home: "/Users/me", - want: methodGo, - }, - { - name: "managed install.sh usr local bin", - execPath: "/usr/local/bin/langsmith", - version: "0.2.38", - goos: "darwin", - home: "/Users/me", - want: methodManaged, - }, - { - name: "managed install.sh local bin", - execPath: "/Users/me/.local/bin/langsmith", - version: "0.2.38", - goos: "linux", - home: "/Users/me", - want: methodManaged, - }, - { - name: "dev build outside package dirs", - execPath: "/Users/me/repos/langsmith-cli/bin/langsmith", - version: "dev", - goos: "darwin", - home: "/Users/me", - want: methodDev, - }, - { - name: "GOBIN takes precedence over GOPATH", - execPath: "/Users/me/gopath/bin/langsmith", - version: "0.2.38", - goos: "darwin", - gobin: "/Users/me/gobin", - gopath: "/Users/me/gopath", - home: "/Users/me", - // exec is in GOPATH/bin, but GOBIN is set and differs → not the go bin dir - want: methodManaged, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := detectInstallMethod(tt.execPath, tt.version, tt.goos, tt.gobin, tt.gopath, tt.home) - if got != tt.want { - t.Errorf("detectInstallMethod() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestExternalManagers(t *testing.T) { - want := map[installMethod]struct{ label, command string }{ - methodHomebrew: {"homebrew", "brew upgrade langchain-ai/tap/langsmith-cli"}, - methodScoop: {"scoop", "scoop update langsmith-cli"}, - methodGo: {"go", "go install github.com/langchain-ai/langsmith-cli/cmd/langsmith@latest"}, - } - for method, exp := range want { - mgr, ok := externalManagers[method] - if !ok { - t.Errorf("externalManagers missing entry for %v", method) - continue - } - if mgr.label != exp.label { - t.Errorf("externalManagers[%v].label = %q, want %q", method, mgr.label, exp.label) - } - if mgr.command != exp.command { - t.Errorf("externalManagers[%v].command = %q, want %q", method, mgr.command, exp.command) - } - if mgr.display == "" { - t.Errorf("externalManagers[%v].display is empty", method) - } - } - // managed and dev are handled by self-update itself and must not be listed. - for _, method := range []installMethod{methodManaged, methodDev} { - if _, ok := externalManagers[method]; ok { - t.Errorf("externalManagers should not contain %v", method) - } - } -} - -func TestShouldDeferToManager(t *testing.T) { - tests := []struct { - method installMethod - force bool - want bool - }{ - {methodHomebrew, false, true}, - {methodHomebrew, true, false}, // --force overrides the defer - {methodScoop, false, true}, - {methodScoop, true, false}, - {methodGo, false, true}, - {methodGo, true, false}, - {methodManaged, false, false}, - {methodManaged, true, false}, - {methodDev, false, false}, - {methodDev, true, false}, - } - for _, tt := range tests { - if got := shouldDeferToManager(tt.method, tt.force); got != tt.want { - t.Errorf("shouldDeferToManager(%v, force=%v) = %v, want %v", tt.method, tt.force, got, tt.want) - } - } -} - -func TestReportManagedExternally_JSON(t *testing.T) { - oldFmt := flagOutputFormat - flagOutputFormat = "json" - defer func() { flagOutputFormat = oldFmt }() - - output := captureStdout(t, func() { - if err := reportManagedExternally(methodHomebrew, "json"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - }) - - var result map[string]string - if err := json.Unmarshal([]byte(output), &result); err != nil { - t.Fatalf("invalid JSON output: %v, output: %q", err, output) - } - if result["status"] != "managed-externally" { - t.Errorf("expected status managed-externally, got %q", result["status"]) - } - if result["install_method"] != "homebrew" { - t.Errorf("expected install_method homebrew, got %q", result["install_method"]) - } - if result["update_command"] != "brew upgrade langchain-ai/tap/langsmith-cli" { - t.Errorf("unexpected update_command: %q", result["update_command"]) - } -} - -func TestReportManagedExternally_Pretty(t *testing.T) { - output := captureStdout(t, func() { - if err := reportManagedExternally(methodScoop, "pretty"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - }) - if !strings.Contains(output, "scoop update langsmith-cli") { - t.Errorf("expected pretty output to contain the scoop command, got %q", output) - } - if !strings.Contains(output, "--force") { - t.Errorf("expected pretty output to mention --force, got %q", output) - } -} diff --git a/internal/cmd/message.go b/internal/cmd/message.go index 190e27d..11413ba 100644 --- a/internal/cmd/message.go +++ b/internal/cmd/message.go @@ -2,11 +2,8 @@ package cmd import ( "context" - "encoding/json" "fmt" "os" - "regexp" - "strings" "time" langsmith "github.com/langchain-ai/langsmith-go" @@ -16,28 +13,7 @@ import ( "github.com/spf13/cobra" ) -// Char caps for the per-trace digest fields. Bounded so a single huge message -// or tool payload can't blow up the digest (and the context of any agent -// reading it). The full untruncated content is still available in `groups`. -const ( - digestTextMax = 600 - digestArgsMax = 200 - digestResultMax = 200 -) - -// errorishPattern is a heuristic flag for a tool result whose content looks -// like an *error response* (not merely text that mentions the word "error" — -// technical docs and search results say "error" constantly). It requires -// error-shaped context: an error/exception token with `:`/`=`, a quoted -// `"error":` key, an HTTP/status 4xx-5xx, a 4xx-5xx code next to a failure -// word, a traceback, or a common shell/tool failure string. Advisory, not -// authoritative — a reader still confirms by looking at the result. -var errorishPattern = regexp.MustCompile(`(?i)|traceback|command not found|permission denied|no such file|\b(error|exception)\b\s*[:=]|"error"\s*[:=]|\bhttp[ /]?[45]\d\d\b|\b[45]\d\d\b\s+\w*\s*(error|not found|forbidden|unauthorized|internal server)`) - -// trajectoryStep is a compact single-step view of one message/tool-call. It is -// deliberately THIN: the trajectory is scanned across *all* traces in a batch -// (triage), so per-trace content lives in the separate `digest` field, read -// per-trace — not crammed into the bulk-scanned trajectory. +// trajectoryStep is a compact single-step view of one message/tool-call in a trace. type trajectoryStep struct { Role string `json:"role"` ToolName string `json:"tool_name,omitempty"` @@ -48,46 +24,10 @@ type trajectoryStep struct { // traceTrajectory is the compact trajectory for a single trace. type traceTrajectory struct { - TraceID string `json:"trace_id"` - Steps []trajectoryStep `json:"steps"` -} - -// digestToolArg is one tool call's name and its (bounded) JSON args. -type digestToolArg struct { - Name string `json:"name"` - Args string `json:"args"` -} - -// digestGroup is one normalized group in a trace's digest — enough to verdict -// the group without re-parsing the heterogeneously-shaped raw `groups`. -type digestGroup struct { - I int `json:"i"` - Kind string `json:"kind"` - Role string `json:"role,omitempty"` - Chars int `json:"chars"` - Text string `json:"text,omitempty"` // coerced content, ≤digestTextMax - Tools []string `json:"tools,omitempty"` // tool-call names in this group - ToolArgs []digestToolArg `json:"tool_args,omitempty"` // per call: name + ≤digestArgsMax JSON args - ToolResult []string `json:"tool_result,omitempty"` // per call: coerced result, ≤digestResultMax - Errorish bool `json:"errorish,omitempty"` // any tool result looks like an error -} - -// traceDigest is the detailed per-trace view a screener reads for the ONE trace -// it is judging. It is emitted as its own field (NOT folded into the trajectory) -// so this bulk detail is read per-trace, never scanned across the whole batch. -type traceDigest struct { - FirstHuman string `json:"first_human,omitempty"` - FinalAI string `json:"final_ai,omitempty"` - NGroups int `json:"n_groups"` - GroupIndex []digestGroup `json:"group_index"` -} - -// attachTrajectory attaches the thin trajectory and the per-trace digest to a -// trace map for JSON output. The trajectory is scanned across all traces; the -// digest is read per-trace. -func attachTrajectory(trace map[string]any, traj traceTrajectory, digest traceDigest) { - trace["trajectory"] = traj.Steps - trace["digest"] = digest + TraceID string `json:"trace_id"` + InputMessage string `json:"input_message,omitempty"` + OutputMessage string `json:"output_message,omitempty"` + Steps []trajectoryStep `json:"steps"` } func newTraceMessagesCmd() *cobra.Command { @@ -133,7 +73,12 @@ Examples: c := MustGetClient() ctx := context.Background() - sessionID, err := resolveSessionID(ctx, c, ff.Project, ff.ProjectID, "trace messages") + projectName := ResolveProject(ff.Project) + if projectName == "" { + ExitError("--project is required for trace messages (or set LANGSMITH_PROJECT)") + } + + sessionID, err := c.ResolveSessionID(ctx, projectName) if err != nil { ExitErrorf("%v", err) } @@ -196,7 +141,7 @@ Examples: for _, t := range traces { trace, _ := t.(map[string]any) traj := buildTraceTrajectory(trace) - attachTrajectory(trace, traj, buildTraceDigest(trace)) + trace["trajectory"] = traj.Steps } nextCursor, _ := result["next_cursor"].(string) @@ -257,7 +202,7 @@ Examples: for _, t := range allTraces { trace, _ := t.(map[string]any) traj := buildTraceTrajectory(trace) - attachTrajectory(trace, traj, buildTraceDigest(trace)) + trace["trajectory"] = traj.Steps } combined := map[string]any{ @@ -275,9 +220,7 @@ Examples: } addCommonFilterFlags(cmd, &ff, true) - cmd.Flags().StringVar(&ff.ProjectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") - cmd.MarkFlagsMutuallyExclusive("project", "project-id") return cmd } @@ -418,12 +361,11 @@ func printTraceMessages(result map[string]any) { } } -// buildTraceTrajectory converts a raw trace map into a compact, THIN trajectory -// (role/tool/tokens/latency/chars per step). It carries no message content — -// the trajectory is scanned across all traces for triage, so detail lives in -// buildTraceDigest and is read per-trace. +// buildTraceTrajectory converts a raw trace map into a compact trajectory. func buildTraceTrajectory(trace map[string]any) traceTrajectory { traceID, _ := trace["trace_id"].(string) + inputsPreview, _ := trace["root_inputs_preview"].(string) + outputsPreview, _ := trace["root_outputs_preview"].(string) groups, _ := trace["groups"].([]any) var steps []trajectoryStep @@ -431,6 +373,7 @@ func buildTraceTrajectory(trace map[string]any) traceTrajectory { group, _ := g.(map[string]any) gType, _ := group["type"].(string) meta, _ := group["metadata"].(map[string]any) + tokens := trajTokens(meta) latency := trajLatency(meta) @@ -466,85 +409,12 @@ func buildTraceTrajectory(trace map[string]any) traceTrajectory { } } } - return traceTrajectory{TraceID: traceID, Steps: steps} -} - -// buildTraceDigest builds the detailed per-trace digest: the full first-human / -// final-AI messages plus a normalized per-group index (content coerced to -// strings, fields length-bounded). Read per-trace by a screener — kept OUT of -// the bulk-scanned trajectory on purpose. Mirrors the fetch-time digest the -// issues-agent screeners verdict from. -func buildTraceDigest(trace map[string]any) traceDigest { - inputsPreview, _ := trace["root_inputs_preview"].(string) - outputsPreview, _ := trace["root_outputs_preview"].(string) - groups, _ := trace["groups"].([]any) - - var firstHuman, finalAI string - index := make([]digestGroup, 0, len(groups)) - for i, g := range groups { - group, _ := g.(map[string]any) - gType, _ := group["type"].(string) - dg := digestGroup{I: i, Kind: gType} - if dg.Kind == "" { - dg.Kind = "message" - } - - var text string - switch gType { - case "message": - msg, _ := group["message"].(map[string]any) - dg.Role, _ = msg["role"].(string) - text = msgText(msg) - if dg.Role == "human" && firstHuman == "" { - firstHuman = text - } - if dg.Role == "ai" && text != "" { - finalAI = text - } - case "tool_interaction": - aiMsg, _ := group["aiMessage"].(map[string]any) - text = msgText(aiMsg) - if text != "" { - finalAI = text - } - toolCalls, _ := group["toolCalls"].([]any) - for _, tc := range toolCalls { - call, _ := tc.(map[string]any) - name, _ := call["name"].(string) - dg.Tools = append(dg.Tools, name) - dg.ToolArgs = append(dg.ToolArgs, digestToolArg{ - Name: name, - Args: truncateHard(marshalArgs(call["args"]), digestArgsMax), - }) - result, _ := call["result"].(map[string]any) - resultText := "" - if result != nil { - resultText = coerceContent(result["content"]) - } - dg.ToolResult = append(dg.ToolResult, truncateHard(resultText, digestResultMax)) - if resultText != "" && errorishPattern.MatchString(resultText) { - dg.Errorish = true - } - } - } - dg.Chars = len(text) - dg.Text = truncateHard(text, digestTextMax) - index = append(index, dg) - } - // Prefer the full message from groups; fall back to the (~150-char) preview - // only when groups don't carry it. - if firstHuman == "" { - firstHuman = inputsPreview - } - if finalAI == "" { - finalAI = outputsPreview - } - return traceDigest{ - FirstHuman: firstHuman, - FinalAI: finalAI, - NGroups: len(groups), - GroupIndex: index, + return traceTrajectory{ + TraceID: traceID, + InputMessage: inputsPreview, + OutputMessage: outputsPreview, + Steps: steps, } } @@ -594,85 +464,6 @@ func msgChars(msg map[string]any) int { return total } -// coerceContent flattens a message/tool `content` value — which may be a -// string, a list of content blocks, an object, or null — into a plain string. -// Content shape is heterogeneous across integrations; normalizing here lets a -// reader verdict off the trajectory without re-parsing each shape. -func coerceContent(v any) string { - switch c := v.(type) { - case nil: - return "" - case string: - return c - case []any: - var parts []string - for _, block := range c { - switch b := block.(type) { - case string: - parts = append(parts, b) - case map[string]any: - if t, ok := b["text"].(string); ok { - parts = append(parts, t) - } else if t, ok := b["content"].(string); ok { - parts = append(parts, t) - } else { - parts = append(parts, unrenderedContent(b)) - } - default: - parts = append(parts, fmt.Sprintf("%v", b)) - } - } - return strings.Join(parts, " ") - case map[string]any: - if t, ok := c["text"].(string); ok { - return t - } - if t, ok := c["content"].(string); ok { - return t - } - return unrenderedContent(c) - default: - return fmt.Sprintf("%v", c) - } -} - -// unrenderedContent represents a non-null content block that carries no -// plain-text `text`/`content` field (image, tool_use, or other structured -// blocks) as a non-empty string. Collapsing these to "" makes structured -// content indistinguishable from a genuinely empty turn, which misleads a -// downstream reader (e.g. the engine screener) into treating the content as -// absent. Prefer compact JSON (the downstream char caps truncate it); fall back -// to a typed sentinel if the block can't be marshaled. -func unrenderedContent(m map[string]any) string { - if b, err := json.Marshal(m); err == nil { - return string(b) - } - if t, ok := m["type"].(string); ok && t != "" { - return "[" + t + " content]" - } - return "[unrenderable content]" -} - -// msgText returns a message's content coerced to a plain string. -func msgText(msg map[string]any) string { - if msg == nil { - return "" - } - return coerceContent(msg["content"]) -} - -// marshalArgs renders tool-call args as compact JSON, or "" if absent/unmarshalable. -func marshalArgs(args any) string { - if args == nil { - return "" - } - b, err := json.Marshal(args) - if err != nil { - return "" - } - return string(b) -} - // printMessage prints a single message in a compact format. func printMessage(msg map[string]any) { if msg == nil { diff --git a/internal/cmd/message_trajectory_test.go b/internal/cmd/message_trajectory_test.go deleted file mode 100644 index a791266..0000000 --- a/internal/cmd/message_trajectory_test.go +++ /dev/null @@ -1,177 +0,0 @@ -package cmd - -import ( - "strings" - "testing" -) - -// The trajectory must stay THIN — no message content / tool args / results. -// It is scanned across all traces, so per-trace detail belongs in the digest. -func TestBuildTraceTrajectory_IsThin(t *testing.T) { - trace := map[string]any{ - "trace_id": "t1", - "groups": []any{ - map[string]any{"type": "message", "message": map[string]any{"role": "human", "content": "x" + strings.Repeat("y", 5000)}}, - map[string]any{ - "type": "tool_interaction", - "aiMessage": map[string]any{"role": "ai", "content": "thinking"}, - "toolCalls": []any{map[string]any{"name": "search", "args": map[string]any{"q": "z"}, "result": map[string]any{"content": "404 not found"}}}, - }, - }, - } - traj := buildTraceTrajectory(trace) - if len(traj.Steps) != 3 { // human, ai, tool - t.Fatalf("got %d steps, want 3", len(traj.Steps)) - } - // Steps carry only role/tool_name/chars — no content fields exist on the struct. - if traj.Steps[2].Role != "tool" || traj.Steps[2].ToolName != "search" { - t.Errorf("tool step = %+v", traj.Steps[2]) - } - if traj.Steps[0].Chars != 5001 { - t.Errorf("human step chars = %d, want 5001 (count only, no content)", traj.Steps[0].Chars) - } -} - -// buildTraceDigest carries the detail: full first/final messages + a normalized -// per-group index with coerced content, tool args/results, and errorish flags. -func TestBuildTraceDigest_NormalizedGroups(t *testing.T) { - trace := map[string]any{ - "trace_id": "t1", - "root_inputs_preview": "human: how do I use MCP?", - "root_outputs_preview": "tool: clipped preview that isn't the answer", - "groups": []any{ - map[string]any{"type": "message", "message": map[string]any{"role": "human", "content": "how do I use MCP?"}}, - map[string]any{ - "type": "tool_interaction", - "aiMessage": map[string]any{"role": "ai", "content": []any{map[string]any{"type": "text", "text": "searching the docs"}}}, - "toolCalls": []any{map[string]any{"name": "search_docs", "args": map[string]any{"q": "mcp"}, "result": map[string]any{"content": "404 not found"}}}, - }, - map[string]any{"type": "message", "message": map[string]any{"role": "ai", "content": "Use create_agent with the mcp arg."}}, - }, - } - d := buildTraceDigest(trace) - - if d.FirstHuman != "how do I use MCP?" { - t.Errorf("first_human = %q, want full human message from groups", d.FirstHuman) - } - if d.FinalAI != "Use create_agent with the mcp arg." { - t.Errorf("final_ai = %q, want final ai message from groups", d.FinalAI) - } - if d.NGroups != 3 || len(d.GroupIndex) != 3 { - t.Fatalf("n_groups=%d group_index=%d, want 3/3", d.NGroups, len(d.GroupIndex)) - } - // tool_interaction group: content coerced from list-of-blocks, tool fields present. - tg := d.GroupIndex[1] - if tg.Kind != "tool_interaction" || tg.Text != "searching the docs" { - t.Errorf("group[1] kind/text = %q/%q", tg.Kind, tg.Text) - } - if len(tg.Tools) != 1 || tg.Tools[0] != "search_docs" { - t.Errorf("group[1] tools = %v", tg.Tools) - } - if len(tg.ToolArgs) != 1 || tg.ToolArgs[0].Name != "search_docs" || !strings.Contains(tg.ToolArgs[0].Args, `"q":"mcp"`) { - t.Errorf("group[1] tool_args = %+v", tg.ToolArgs) - } - if len(tg.ToolResult) != 1 || tg.ToolResult[0] != "404 not found" { - t.Errorf("group[1] tool_result = %v", tg.ToolResult) - } -} - -// Null/missing content coerces to empty (no panic); first/final fall back to previews. -func TestBuildTraceDigest_NullAndFallback(t *testing.T) { - trace := map[string]any{ - "trace_id": "t2", - "root_inputs_preview": "human: hi", - "root_outputs_preview": "the answer", - "groups": []any{ - map[string]any{ - "type": "tool_interaction", - "aiMessage": map[string]any{"role": "ai", "content": nil}, - "toolCalls": []any{map[string]any{"name": "read_url", "result": map[string]any{"content": nil}}}, - }, - }, - } - d := buildTraceDigest(trace) - if d.FirstHuman != "human: hi" || d.FinalAI != "the answer" { - t.Errorf("fallbacks: first=%q final=%q", d.FirstHuman, d.FinalAI) - } - g := d.GroupIndex[0] - if g.Text != "" || g.ToolResult[0] != "" || g.Errorish { - t.Errorf("null content should be empty/non-errorish, got %+v", g) - } -} - -// Non-null structured content (image / tool_use blocks, or a map with no -// text/content field) must NOT collapse to "" — that is indistinguishable from -// a genuinely empty turn and misleads a downstream reader into thinking the -// content is absent. Only nil stays empty. -func TestCoerceContent_UnrenderedStructuredContent(t *testing.T) { - if got := coerceContent(nil); got != "" { - t.Errorf("nil content = %q, want empty", got) - } - // A structured object with no text/content field (e.g. an image block). - img := map[string]any{"type": "image", "source": map[string]any{"url": "https://x/y.png"}} - if got := coerceContent(img); got == "" || !strings.Contains(got, "image") { - t.Errorf("image block coerced to %q, want non-empty mentioning its type", got) - } - // A list with a renderable text block and an unrenderable tool_use block: - // the tool_use must still contribute non-empty content, not be dropped. - blocks := []any{ - map[string]any{"type": "text", "text": "calling a tool"}, - map[string]any{"type": "tool_use", "name": "search", "input": map[string]any{"q": "mcp"}}, - } - if got := coerceContent(blocks); !strings.Contains(got, "calling a tool") || !strings.Contains(got, "tool_use") { - t.Errorf("block list coerced to %q, want both the text and the tool_use block", got) - } -} - -// errorish must flag error-SHAPED results, not text that merely mentions "error". -func TestBuildTraceDigest_ErrorishPrecision(t *testing.T) { - cases := []struct { - content string - want bool - }{ - {"Error: connection refused", true}, - {"404 not found", true}, - {`{"error": "rate limited"}`, true}, - {"HTTP 503 service unavailable", true}, - {"Traceback (most recent call last):", true}, - {"Evaluators help you catch error cases in your app.", false}, - {"This guide covers error handling best practices.", false}, - {`{"level": "error", "matched": 4}`, false}, - } - for _, c := range cases { - trace := map[string]any{"groups": []any{map[string]any{ - "type": "tool_interaction", - "aiMessage": map[string]any{"role": "ai", "content": ""}, - "toolCalls": []any{map[string]any{"name": "t", "result": map[string]any{"content": c.content}}}, - }}} - got := buildTraceDigest(trace).GroupIndex[0].Errorish - if got != c.want { - t.Errorf("errorish(%q) = %v, want %v", c.content, got, c.want) - } - } -} - -// Long content is bounded to the digest caps. -func TestBuildTraceDigest_Truncation(t *testing.T) { - big := strings.Repeat("x", 5000) - trace := map[string]any{"groups": []any{ - map[string]any{"type": "message", "message": map[string]any{"role": "human", "content": big}}, - map[string]any{ - "type": "tool_interaction", - "aiMessage": map[string]any{"role": "ai", "content": "ok"}, - "toolCalls": []any{map[string]any{"name": "f", "args": map[string]any{"k": big}, "result": map[string]any{"content": big}}}, - }, - }} - d := buildTraceDigest(trace) - if got := len(d.GroupIndex[0].Text); got != digestTextMax { - t.Errorf("group text len = %d, want %d", got, digestTextMax) - } - tg := d.GroupIndex[1] - if got := len(tg.ToolResult[0]); got != digestResultMax { - t.Errorf("tool_result len = %d, want %d", got, digestResultMax) - } - if got := len(tg.ToolArgs[0].Args); got != digestArgsMax { - t.Errorf("tool_args len = %d, want %d", got, digestArgsMax) - } -} diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 9c8992b..30f514c 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -198,10 +198,6 @@ func resolveClientOptions(refreshOAuth bool) (client.Options, error) { opts.OAuthAccessToken = profile.AccessToken() case hasProfile && profile.APIKey != "": opts.APIKey = profile.APIKey - // Route the resolved profile through the SDK (WithProfile) so an explicit - // selection replaces the config's current_profile instead of inheriting - // its tenant/base URL. APIKey is kept for the raw-HTTP helpers. - opts.ProfileName = profileName } if cfgErr != nil && opts.APIKey == "" { return opts, cfgErr diff --git a/internal/cmd/root_test.go b/internal/cmd/root_test.go index d3734c4..0e8431e 100644 --- a/internal/cmd/root_test.go +++ b/internal/cmd/root_test.go @@ -313,61 +313,6 @@ func TestResolveClientOptionsRefreshesProfileWithoutAccessToken(t *testing.T) { } } -func TestResolveClientOptions_SetsProfileNameForAPIKeyProfile(t *testing.T) { - oldKey := flagAPIKey - oldURL := flagAPIURL - oldProfile := flagProfile - oldWS := flagWorkspaceID - defer func() { - flagAPIKey = oldKey - flagAPIURL = oldURL - flagProfile = oldProfile - flagWorkspaceID = oldWS - }() - flagAPIKey = "" - flagAPIURL = "" - flagWorkspaceID = "" - flagProfile = "aws" - - path := filepath.Join(t.TempDir(), "config.json") - t.Setenv("LANGSMITH_CONFIG_FILE", path) - t.Setenv("LANGSMITH_API_KEY", "") - t.Setenv("LANGSMITH_ENDPOINT", "") - t.Setenv("LANGSMITH_WORKSPACE_ID", "") - t.Setenv("LANGSMITH_TENANT_ID", "") - t.Setenv("LANGSMITH_PROFILE", "") - if err := os.WriteFile(path, []byte(`{ - "current_profile": "prod", - "profiles": { - "prod": { - "api_key": "prod-api-key", - "workspace_id": "prod-workspace-id" - }, - "aws": { - "api_key": "aws-api-key" - } - } -} -`), 0600); err != nil { - t.Fatal(err) - } - - opts, err := resolveClientOptions(false) - if err != nil { - t.Fatalf("resolveClientOptions returned error: %v", err) - } - if opts.APIKey != "aws-api-key" { - t.Fatalf("expected profile api key aws-api-key, got %q", opts.APIKey) - } - // The resolved profile must reach the SDK via WithProfile so an explicit - // --profile replaces current_profile and clears its inherited tenant. Without - // ProfileName set, NewWithOptions passes only WithAPIKey and current_profile's - // tenant leaks (403 cross-workspace). - if opts.ProfileName != "aws" { - t.Fatalf("expected ProfileName=aws, got %q", opts.ProfileName) - } -} - func TestGetOAuthAccessToken_ProfileFallback(t *testing.T) { oldKey := flagAPIKey oldURL := flagAPIURL diff --git a/internal/cmd/run.go b/internal/cmd/run.go index 5b5d358..defa44a 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -60,23 +60,16 @@ func newRunListCmd() *cobra.Command { c := MustGetClient() ctx := context.Background() - sessionID, err := resolveSessionID(ctx, c, ff.Project, "", "run list") - if err != nil { - ExitErrorf("%v", err) + projectName := ResolveProject(ff.Project) + if projectName == "" { + ExitError("--project is required for run list (or set LANGSMITH_PROJECT)") } - var runs []langsmith.RunSchema - if ff.Version == "v2" { - body := buildRunQueryV2Params(&ff, false, ff.Limit) - body.Selects = langsmith.F(buildRunSelectV2(includeIO, includeFeedback)) - runs, err = queryRunsV2(ctx, c, body, sessionID, ff.Limit, ff.MinTokens) - } else { - params := BuildRunQueryParams(&ff, false, ff.Limit) - if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { - params.Select = langsmith.F(sel) - } - runs, err = queryRuns(ctx, c, params, sessionID, ff.Limit, ff.MinTokens) + params := BuildRunQueryParams(&ff, false, ff.Limit) + if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { + params.Select = langsmith.F(sel) } + runs, err := queryRuns(ctx, c, params, projectName, ff.Limit, ff.MinTokens) if err != nil { ExitErrorf("%v", err) } @@ -94,9 +87,8 @@ func newRunListCmd() *cobra.Command { } addCommonFilterFlags(cmd, &ff, true) - addVersionFlag(cmd, &ff) - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") @@ -109,7 +101,6 @@ func newRunGetCmd() *cobra.Command { project string since string lastNMinutes int - version string includeMetadata bool includeIO bool includeFeedback bool @@ -132,31 +123,21 @@ func newRunGetCmd() *cobra.Command { c := MustGetClient() ctx := context.Background() - sessionID, err := resolveSessionID(ctx, c, project, "", "run get") - if err != nil { - ExitErrorf("%v", err) + projectName := ResolveProject(project) + if projectName == "" { + ExitError("--project is required for run get (or set LANGSMITH_PROJECT)") } - var runs []langsmith.RunSchema - if version == "v2" { - params := langsmith.RunQueryV2Params{ - IDs: langsmith.F([]string{runID}), - MinStartTime: langsmith.F(resolveStartTime(since, lastNMinutes)), - PageSize: langsmith.F(int64(1)), - Selects: langsmith.F(buildRunSelectV2(includeIO, includeFeedback)), - } - runs, err = queryRunsV2(ctx, c, params, sessionID, 1, 0) - } else { - params := langsmith.RunQueryParams{ - ID: langsmith.F([]string{runID}), - Limit: langsmith.F(int64(1)), - StartTime: langsmith.F(resolveStartTime(since, lastNMinutes)), - } - if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { - params.Select = langsmith.F(sel) - } - runs, err = queryRuns(ctx, c, params, sessionID, 1, 0) + params := langsmith.RunQueryParams{ + ID: langsmith.F([]string{runID}), + Limit: langsmith.F(int64(1)), + StartTime: langsmith.F(resolveStartTime(since, lastNMinutes)), } + if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { + params.Select = langsmith.F(sel) + } + + runs, err := queryRuns(ctx, c, params, projectName, 1, 0) if err != nil { ExitErrorf("fetching run: %v", err) } @@ -178,9 +159,8 @@ func newRunGetCmd() *cobra.Command { cmd.Flags().StringVar(&project, "project", "", "Project name [env: LANGSMITH_PROJECT]") cmd.Flags().StringVar(&since, "since", "", "Only include runs after this timestamp, e.g. 2024-01-15T00:00:00Z (overrides 7-day default)") cmd.Flags().IntVar(&lastNMinutes, "last-n-minutes", 0, "Only include runs from the last N minutes, e.g. 60 (overrides 7-day default)") - cmd.Flags().StringVar(&version, "version", "", `Query API version: "" (v1, default) or "v2" (SmithDB)`) - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") @@ -216,23 +196,16 @@ func newRunExportCmd() *cobra.Command { c := MustGetClient() ctx := context.Background() - sessionID, err := resolveSessionID(ctx, c, ff.Project, "", "run export") - if err != nil { - ExitErrorf("%v", err) + projectName := ResolveProject(ff.Project) + if projectName == "" { + ExitError("--project is required for run export (or set LANGSMITH_PROJECT)") } - var runs []langsmith.RunSchema - if ff.Version == "v2" { - body := buildRunQueryV2Params(&ff, false, ff.Limit) - body.Selects = langsmith.F(buildRunSelectV2(includeIO, includeFeedback)) - runs, err = queryRunsV2(ctx, c, body, sessionID, ff.Limit, ff.MinTokens) - } else { - params := BuildRunQueryParams(&ff, false, ff.Limit) - if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { - params.Select = langsmith.F(sel) - } - runs, err = queryRuns(ctx, c, params, sessionID, ff.Limit, ff.MinTokens) + params := BuildRunQueryParams(&ff, false, ff.Limit) + if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { + params.Select = langsmith.F(sel) } + runs, err := queryRuns(ctx, c, params, projectName, ff.Limit, ff.MinTokens) if err != nil { ExitErrorf("%v", err) } @@ -243,9 +216,8 @@ func newRunExportCmd() *cobra.Command { } addCommonFilterFlags(cmd, &ff, true) - addVersionFlag(cmd, &ff) - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") diff --git a/internal/cmd/sandbox_box.go b/internal/cmd/sandbox_box.go index 049cec3..3738387 100644 --- a/internal/cmd/sandbox_box.go +++ b/internal/cmd/sandbox_box.go @@ -16,7 +16,7 @@ import ( const defaultBoxPollInterval = 2 * time.Second -func waitForBoxReady(ctx context.Context, c *client.Client, name string) (*langsmith.SandboxStatusResponse, error) { +func waitForBoxReady(ctx context.Context, c *client.Client, name string) (*langsmith.SandboxBoxGetStatusResponse, error) { for { resp, err := c.SDK.Sandboxes.Boxes.GetStatus(ctx, name) if err != nil { diff --git a/internal/cmd/sandbox_test.go b/internal/cmd/sandbox_test.go index 1c7f3d3..30030d3 100644 --- a/internal/cmd/sandbox_test.go +++ b/internal/cmd/sandbox_test.go @@ -288,7 +288,7 @@ func TestSandboxBoxDetailRenderSupportsSDKResponseTypes(t *testing.T) { }{ { name: "new response", - model: langsmith.SandboxResponse{ + model: langsmith.SandboxBoxNewResponse{ ID: "box-new", Name: "new-vm", Status: "running", @@ -305,7 +305,7 @@ func TestSandboxBoxDetailRenderSupportsSDKResponseTypes(t *testing.T) { }, { name: "get response", - model: langsmith.SandboxResponse{ + model: langsmith.SandboxBoxGetResponse{ ID: "box-get", Name: "get-vm", Status: "running", @@ -322,7 +322,7 @@ func TestSandboxBoxDetailRenderSupportsSDKResponseTypes(t *testing.T) { }, { name: "update response", - model: langsmith.SandboxResponse{ + model: langsmith.SandboxBoxUpdateResponse{ ID: "box-update", Name: "update-vm", Status: "stopped", diff --git a/internal/cmd/thread.go b/internal/cmd/thread.go index 92ab76c..cde5413 100644 --- a/internal/cmd/thread.go +++ b/internal/cmd/thread.go @@ -105,10 +105,10 @@ func newThreadListCmd() *cobra.Command { threadsMap[tid] = append(threadsMap[tid], m) } } - if resp.Cursors.Next == "" { + if resp.Cursors == nil || resp.Cursors["next"] == "" { break } - cursor = resp.Cursors.Next + cursor = resp.Cursors["next"] } // Build thread summaries @@ -267,8 +267,8 @@ func newThreadGetCmd() *cobra.Command { } cmd.Flags().StringVar(&project, "project", "", "Project name [env: LANGSMITH_PROJECT]") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().IntVarP(&limit, "limit", "n", 0, "Maximum number of runs (turns) to return") diff --git a/internal/cmd/trace.go b/internal/cmd/trace.go index ef78232..c10e4a9 100644 --- a/internal/cmd/trace.go +++ b/internal/cmd/trace.go @@ -68,22 +68,26 @@ func newTraceListCmd() *cobra.Command { c := MustGetClient() ctx := context.Background() - sessionID, err := resolveSessionID(ctx, c, ff.Project, ff.ProjectID, "trace list") - if err != nil { - ExitErrorf("%v", err) + projectName := ResolveProject(ff.Project) + if projectName == "" { + ExitError("--project is required for trace list (or set LANGSMITH_PROJECT)") } params := BuildRunQueryParams(&ff, true, ff.Limit) if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { params.Select = langsmith.F(sel) } - runs, err := queryRuns(ctx, c, params, sessionID, ff.Limit, ff.MinTokens) + runs, err := queryRuns(ctx, c, params, projectName, ff.Limit, ff.MinTokens) if err != nil { ExitErrorf("%v", err) } var flaggedByTrace map[string]string if includeFlagged { + sessionID, err := c.ResolveSessionID(ctx, projectName) + if err != nil { + ExitErrorf("%v", err) + } flagged := FetchFlaggedTraces(ctx, c, sessionID, ff.LastNMinutes) flaggedByTrace = make(map[string]string, len(flagged)) for _, ft := range flagged { @@ -99,7 +103,7 @@ func newTraceListCmd() *cobra.Command { allRuns, err := queryRuns(ctx, c, langsmith.RunQueryParams{ Trace: langsmith.F(run.TraceID), Order: langsmith.F(langsmith.RunQueryParamsOrderAsc), - }, sessionID, 1000, 0) + }, projectName, 1000, 0) if err != nil { ExitErrorf("%v", err) } @@ -120,7 +124,7 @@ func newTraceListCmd() *cobra.Command { var result []map[string]any for _, run := range runs { childParams.Trace = langsmith.F(run.TraceID) - allRuns, err := queryRuns(ctx, c, childParams, sessionID, 1000, 0) + allRuns, err := queryRuns(ctx, c, childParams, projectName, 1000, 0) if err != nil { ExitErrorf("%v", err) } @@ -143,15 +147,13 @@ func newTraceListCmd() *cobra.Command { } addCommonFilterFlags(cmd, &ff, false) - cmd.Flags().StringVar(&ff.ProjectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&includeFlagged, "include-flagged", false, "Add flagged_comment field populated from user-flagged trace feedback") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().BoolVar(&showHierarchy, "show-hierarchy", false, "Fetch the full run tree for each trace") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") - cmd.MarkFlagsMutuallyExclusive("project", "project-id") return cmd } @@ -159,7 +161,6 @@ func newTraceListCmd() *cobra.Command { func newTraceGetCmd() *cobra.Command { var ( project string - projectID string since string lastNMinutes int includeMetadata bool @@ -184,9 +185,9 @@ func newTraceGetCmd() *cobra.Command { c := MustGetClient() ctx := context.Background() - sessionID, err := resolveSessionID(ctx, c, project, projectID, "trace get") - if err != nil { - ExitErrorf("%v", err) + projectName := ResolveProject(project) + if projectName == "" { + ExitError("--project is required for trace get (or set LANGSMITH_PROJECT)") } params := langsmith.RunQueryParams{ @@ -198,7 +199,7 @@ func newTraceGetCmd() *cobra.Command { params.Select = langsmith.F(sel) } - runs, err := queryRuns(ctx, c, params, sessionID, 1000, 0) + runs, err := queryRuns(ctx, c, params, projectName, 1000, 0) if err != nil { ExitErrorf("%v", err) } @@ -219,15 +220,13 @@ func newTraceGetCmd() *cobra.Command { } cmd.Flags().StringVar(&project, "project", "", "Project name [env: LANGSMITH_PROJECT]") - cmd.Flags().StringVar(&projectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") cmd.Flags().StringVar(&since, "since", "", "Only include runs after this timestamp, e.g. 2024-01-15T00:00:00Z (overrides 7-day default)") cmd.Flags().IntVar(&lastNMinutes, "last-n-minutes", 0, "Only include runs from the last N minutes, e.g. 60 (overrides 7-day default)") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") - cmd.MarkFlagsMutuallyExclusive("project", "project-id") return cmd } @@ -265,9 +264,9 @@ func newTraceExportCmd() *cobra.Command { c := MustGetClient() ctx := context.Background() - sessionID, err := resolveSessionID(ctx, c, ff.Project, ff.ProjectID, "trace export") - if err != nil { - ExitErrorf("%v", err) + projectName := ResolveProject(ff.Project) + if projectName == "" { + ExitError("--project is required for trace export (or set LANGSMITH_PROJECT)") } params := BuildRunQueryParams(&ff, true, ff.Limit) @@ -275,7 +274,7 @@ func newTraceExportCmd() *cobra.Command { if sel != nil { params.Select = langsmith.F(sel) } - rootRuns, err := queryRuns(ctx, c, params, sessionID, ff.Limit, ff.MinTokens) + rootRuns, err := queryRuns(ctx, c, params, projectName, ff.Limit, ff.MinTokens) if err != nil { ExitErrorf("%v", err) } @@ -291,7 +290,7 @@ func newTraceExportCmd() *cobra.Command { if sel != nil { childParams.Select = langsmith.F(sel) } - allRuns, err := queryRuns(ctx, c, childParams, sessionID, 1000, 0) + allRuns, err := queryRuns(ctx, c, childParams, projectName, 1000, 0) if err != nil { ExitErrorf("%v", err) } @@ -331,14 +330,12 @@ func newTraceExportCmd() *cobra.Command { } addCommonFilterFlags(cmd, &ff, false) - cmd.Flags().StringVar(&ff.ProjectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVar(&filenamePattern, "filename-pattern", "{trace_id}.jsonl", "Filename pattern. Supports {trace_id} and {name} placeholders.") - cmd.MarkFlagsMutuallyExclusive("project", "project-id") return cmd } diff --git a/internal/cmd/trace_stats.go b/internal/cmd/trace_stats.go index cc4ef93..6e4701d 100644 --- a/internal/cmd/trace_stats.go +++ b/internal/cmd/trace_stats.go @@ -31,7 +31,6 @@ type runStats struct { func newTraceStatsCmd() *cobra.Command { var ( project string - projectID string since string before string lastNMin int @@ -64,9 +63,14 @@ Examples: c := MustGetClient() ctx := context.Background() - sessionID, err := resolveSessionID(ctx, c, project, projectID, "trace stats") + projectName := ResolveProject(project) + if projectName == "" { + return fmt.Errorf("--project is required (or set LANGSMITH_PROJECT)") + } + + sessionID, err := c.ResolveSessionID(ctx, projectName) if err != nil { - return err + return fmt.Errorf("resolving project %q: %w", projectName, err) } primary, err := fetchRunStats(ctx, c, sessionID, since, before, lastNMin, filter) @@ -99,7 +103,6 @@ Examples: } cmd.Flags().StringVar(&project, "project", "", "Project name [env: LANGSMITH_PROJECT]") - cmd.Flags().StringVar(&projectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") cmd.Flags().StringVar(&since, "since", "", "Start of time window (RFC3339 or YYYY-MM-DD; default: 7 days ago)") cmd.Flags().StringVar(&before, "before", "", "End of time window (RFC3339 or YYYY-MM-DD; default: now)") cmd.Flags().IntVar(&lastNMin, "last-n-minutes", 0, "Shorthand: window = last N minutes (overrides --since)") @@ -108,7 +111,6 @@ Examples: cmd.Flags().IntVar(&cmpLastNMin, "compare-last-n-minutes", 0, "Shorthand: comparison window = N minutes before the primary window starts") cmd.Flags().StringVar(&filter, "filter", "", "LangSmith filter DSL (applied to both windows if comparing)") cmd.Flags().StringVar(&outputFile, "output", "", "Write JSON output to file instead of stdout") - cmd.MarkFlagsMutuallyExclusive("project", "project-id") return cmd } @@ -163,13 +165,13 @@ func fetchRunStats(ctx context.Context, c *client.Client, sessionID, since, befo } } -func toRunStats(runCount int64, latencyP50, latencyP99 float64, totalTokens, promptTokens, completionTokens int64, totalCost float64, errorRate float64, feedbackStats map[string]interface{}) runStats { +func toRunStats(runCount int64, latencyP50, latencyP99 float64, totalTokens, promptTokens, completionTokens int64, totalCost string, errorRate float64, feedbackStats map[string]interface{}) runStats { fs := make(map[string]any, len(feedbackStats)) for k, v := range feedbackStats { fs[k] = v } var cost any - if totalCost > 0 { + if totalCost != "" { cost = totalCost } return runStats{ diff --git a/internal/cmd/update.go b/internal/cmd/update.go index d84deb1..19ba244 100644 --- a/internal/cmd/update.go +++ b/internal/cmd/update.go @@ -28,48 +28,26 @@ var githubDownloadBaseURL = "https://github.com" func newUpdateCmd(rawVersion string) *cobra.Command { var dryRun bool - var force bool cmd := &cobra.Command{ Use: "self-update", Short: "Update langsmith to the latest version", Long: "Check for and install the latest version of the langsmith CLI.", RunE: func(cmd *cobra.Command, args []string) error { - return runUpdate(cmd.Context(), rawVersion, dryRun, force) + return runUpdate(cmd.Context(), rawVersion, dryRun) }, } cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Check for updates without installing") - cmd.Flags().BoolVar(&force, "force", false, "Replace the binary in place even if a package manager (brew, scoop, go) manages this install") return cmd } -func runUpdate(ctx context.Context, currentVersion string, dryRun, force bool) error { - // Resolve the current binary path first: it drives both install-method - // detection and the eventual in-place replacement. - execPath, err := os.Executable() - if err != nil { - return fmt.Errorf("resolving executable path: %w", err) - } - execPath, err = filepath.EvalSymlinks(execPath) - if err != nil { - return fmt.Errorf("resolving symlinks: %w", err) - } - - home, _ := os.UserHomeDir() - method := detectInstallMethod(execPath, currentVersion, runtime.GOOS, os.Getenv("GOBIN"), os.Getenv("GOPATH"), home) - - if method == methodDev { +func runUpdate(ctx context.Context, currentVersion string, dryRun bool) error { + if currentVersion == "dev" { return fmt.Errorf("cannot update a development build; install from a release") } - // For package-manager-owned installs, refuse to touch the binary and point - // the user at the right command instead. --force overrides this. - if shouldDeferToManager(method, force) { - return reportManagedExternally(method, GetFormat()) - } - latest, err := fetchLatestVersion(ctx) if err != nil { return fmt.Errorf("checking for updates: %w", err) @@ -105,6 +83,16 @@ func runUpdate(ctx context.Context, currentVersion string, dryRun, force bool) e return nil } + // Resolve current binary path + execPath, err := os.Executable() + if err != nil { + return fmt.Errorf("resolving executable path: %w", err) + } + execPath, err = filepath.EvalSymlinks(execPath) + if err != nil { + return fmt.Errorf("resolving symlinks: %w", err) + } + archiveName := buildArchiveName() archiveURL := fmt.Sprintf("%s/langchain-ai/langsmith-cli/releases/download/v%s/%s", githubDownloadBaseURL, latest, archiveName) checksumURL := fmt.Sprintf("%s/langchain-ai/langsmith-cli/releases/download/v%s/checksums.txt", githubDownloadBaseURL, latest) @@ -158,31 +146,6 @@ func runUpdate(ctx context.Context, currentVersion string, dryRun, force bool) e return nil } -// reportManagedExternally informs the user that their install is managed by a -// package manager and prints the command to update it, without modifying the -// binary. It returns nil (a successful, informational outcome). -func reportManagedExternally(method installMethod, format string) error { - mgr, ok := externalManagers[method] - if !ok { - // Programming error: only externally-managed methods should reach here - // (methodManaged updates in place, methodDev errors out earlier). - return fmt.Errorf("internal error: install method %d is not externally managed", method) - } - - if format == "pretty" { - fmt.Printf("langsmith was installed with %s, which manages updates for you.\n"+ - "To update, run:\n\n %s\n\n(Use --force to update in place anyway.)\n", mgr.display, mgr.command) - } else { - out, _ := json.Marshal(map[string]string{ - "status": "managed-externally", - "install_method": mgr.label, - "update_command": mgr.command, - }) - fmt.Println(string(out)) - } - return nil -} - // fetchLatestVersion queries the GitHub Releases API for the latest release tag. func fetchLatestVersion(ctx context.Context) (string, error) { url := fmt.Sprintf("%s/repos/langchain-ai/langsmith-cli/releases/latest", githubReleasesBaseURL) diff --git a/internal/cmd/update_test.go b/internal/cmd/update_test.go index 8e52280..7580434 100644 --- a/internal/cmd/update_test.go +++ b/internal/cmd/update_test.go @@ -203,7 +203,7 @@ func TestRunUpdate_AlreadyUpToDate(t *testing.T) { defer func() { flagOutputFormat = oldFmt }() output := captureStdout(t, func() { - err := runUpdate(context.Background(), "0.1.7", false, false) + err := runUpdate(context.Background(), "0.1.7", false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -235,7 +235,7 @@ func TestRunUpdate_DryRun(t *testing.T) { defer func() { flagOutputFormat = oldFmt }() output := captureStdout(t, func() { - err := runUpdate(context.Background(), "0.1.7", true, false) + err := runUpdate(context.Background(), "0.1.7", true) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -254,7 +254,7 @@ func TestRunUpdate_DryRun(t *testing.T) { } func TestRunUpdate_DevBuild(t *testing.T) { - err := runUpdate(context.Background(), "dev", false, false) + err := runUpdate(context.Background(), "dev", false) if err == nil { t.Error("expected error for dev build") } diff --git a/internal/cmdutil/resolve.go b/internal/cmdutil/resolve.go index 7ea4f83..0efa9de 100644 --- a/internal/cmdutil/resolve.go +++ b/internal/cmdutil/resolve.go @@ -187,10 +187,6 @@ func ResolveClientOptions(cmd *cobra.Command, refreshOAuth bool) (client.Options opts.OAuthAccessToken = profile.AccessToken() case hasProfile && profile.APIKey != "": opts.APIKey = profile.APIKey - // Route the resolved profile through the SDK (WithProfile) so an explicit - // selection replaces the config's current_profile instead of inheriting - // its tenant/base URL. APIKey is kept for the raw-HTTP helpers. - opts.ProfileName = profileName } if cfgErr != nil && opts.APIKey == "" { return opts, cfgErr diff --git a/internal/cmdutil/resolve_test.go b/internal/cmdutil/resolve_test.go index 7dacf5e..ba7ec5a 100644 --- a/internal/cmdutil/resolve_test.go +++ b/internal/cmdutil/resolve_test.go @@ -205,44 +205,6 @@ func TestResolveClientOptions_ProfileFlagSetsProfileName(t *testing.T) { } } -func TestResolveClientOptions_APIKeyProfileSetsProfileName(t *testing.T) { - path := filepath.Join(t.TempDir(), "config.json") - t.Setenv("LANGSMITH_CONFIG_FILE", path) - t.Setenv("LANGSMITH_API_KEY", "") - t.Setenv("LANGSMITH_ENDPOINT", "") - if err := os.WriteFile(path, []byte(`{ - "current_profile": "prod", - "profiles": { - "prod": { - "api_key": "prod-api-key", - "workspace_id": "ws-prod" - }, - "aws": { - "api_key": "aws-api-key" - } - } -} -`), 0600); err != nil { - t.Fatal(err) - } - - cmd := newTestCmd() - _ = cmd.PersistentFlags().Set("profile", "aws") - opts, err := ResolveClientOptions(cmd, false) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if opts.APIKey != "aws-api-key" { - t.Fatalf("expected profile api key, got %q", opts.APIKey) - } - // An api-key profile must route through WithProfile too, so it replaces - // current_profile and clears the inherited tenant. This resolver (used by the - // api/sandbox/ssh subcommands) previously omitted ProfileName here. - if opts.ProfileName != "aws" { - t.Fatalf("expected ProfileName=aws, got %q", opts.ProfileName) - } -} - func TestResolveClientOptions_WorkspaceFlagOverridesEnvAndProfile(t *testing.T) { path := filepath.Join(t.TempDir(), "config.json") t.Setenv("LANGSMITH_CONFIG_FILE", path) diff --git a/internal/extract/extract.go b/internal/extract/extract.go index c0dc1e4..b92863d 100644 --- a/internal/extract/extract.go +++ b/internal/extract/extract.go @@ -2,6 +2,7 @@ package extract import ( "fmt" + "strconv" "time" langsmith "github.com/langchain-ai/langsmith-go" @@ -47,16 +48,16 @@ func ExtractRun(run langsmith.RunSchema, includeMetadata, includeIO, includeFeed } var costs map[string]any - if run.PromptCost > 0 || run.CompletionCost > 0 || run.TotalCost > 0 { + if run.PromptCost != "" || run.CompletionCost != "" || run.TotalCost != "" { costs = map[string]any{} - if run.PromptCost > 0 { - costs["prompt_cost"] = run.PromptCost + if f, err := strconv.ParseFloat(run.PromptCost, 64); err == nil && f > 0 { + costs["prompt_cost"] = f } - if run.CompletionCost > 0 { - costs["completion_cost"] = run.CompletionCost + if f, err := strconv.ParseFloat(run.CompletionCost, 64); err == nil && f > 0 { + costs["completion_cost"] = f } - if run.TotalCost > 0 { - costs["total_cost"] = run.TotalCost + if f, err := strconv.ParseFloat(run.TotalCost, 64); err == nil && f > 0 { + costs["total_cost"] = f } } if len(costs) == 0 { @@ -70,7 +71,6 @@ func ExtractRun(run langsmith.RunSchema, includeMetadata, includeIO, includeFeed result["status"] = run.Status result["duration_ms"] = durationMs - result["first_token_time"] = formatTimeNullable(run.FirstTokenTime) result["custom_metadata"] = customMetadata result["token_usage"] = tokenUsage result["costs"] = costs @@ -81,7 +81,6 @@ func ExtractRun(run langsmith.RunSchema, includeMetadata, includeIO, includeFeed result["inputs"] = nilIfEmptyMap(run.Inputs) result["outputs"] = nilIfEmptyMap(run.Outputs) result["error"] = nilIfEmpty(run.Error) - result["events"] = nilIfEmptyEvents(run.Events) } if includeFeedback { @@ -131,13 +130,6 @@ func nilIfEmptyMap(m map[string]any) any { return m } -func nilIfEmptyEvents(events []map[string]any) any { - if len(events) == 0 { - return nil - } - return events -} - // FormatDurationHuman formats milliseconds as human-readable string. func FormatDurationHuman(ms int64) string { if ms < 1000 { diff --git a/internal/extract/extract_test.go b/internal/extract/extract_test.go index 20060c7..5399bde 100644 --- a/internal/extract/extract_test.go +++ b/internal/extract/extract_test.go @@ -51,7 +51,6 @@ func TestExtractRunBase(t *testing.T) { func TestExtractRunWithMetadata(t *testing.T) { start := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC) end := time.Date(2024, 1, 15, 10, 30, 5, 0, time.UTC) - firstToken := time.Date(2024, 1, 15, 10, 30, 1, 0, time.UTC) run := langsmith.RunSchema{ ID: "run-123", @@ -60,14 +59,13 @@ func TestExtractRunWithMetadata(t *testing.T) { RunType: "llm", StartTime: start, EndTime: end, - FirstTokenTime: firstToken, Status: "success", PromptTokens: 100, CompletionTokens: 50, TotalTokens: 150, - PromptCost: 0.001, - CompletionCost: 0.0005, - TotalCost: 0.0015, + PromptCost: "0.001", + CompletionCost: "0.0005", + TotalCost: "0.0015", Tags: []string{"production", "v2"}, Extra: map[string]any{"metadata": map[string]any{"model": "gpt-4"}}, } @@ -78,10 +76,6 @@ func TestExtractRunWithMetadata(t *testing.T) { t.Errorf("expected status=success, got %v", result["status"]) } - if result["first_token_time"] != firstToken.Format(time.RFC3339Nano) { - t.Errorf("expected first_token_time=%s, got %v", firstToken.Format(time.RFC3339Nano), result["first_token_time"]) - } - durationMs, ok := result["duration_ms"].(int64) if !ok || durationMs != 5000 { t.Errorf("expected duration_ms=5000, got %v", result["duration_ms"]) @@ -121,7 +115,6 @@ func TestExtractRunWithIO(t *testing.T) { Inputs: map[string]any{"query": "hello"}, Outputs: map[string]any{"response": "world"}, Error: "some error", - Events: []map[string]interface{}{{"name": "new_token", "kwargs": map[string]interface{}{"token": "hi"}}}, } result := ExtractRun(run, false, true, false) @@ -139,30 +132,6 @@ func TestExtractRunWithIO(t *testing.T) { if result["error"] != "some error" { t.Errorf("expected error='some error', got %v", result["error"]) } - - events, ok := result["events"].([]map[string]interface{}) - if !ok { - t.Fatalf("expected events to be []map[string]interface{}, got %T", result["events"]) - } - if len(events) != 1 || events[0]["name"] != "new_token" { - t.Errorf("expected one new_token event, got %v", events) - } -} - -func TestExtractRunWithIOEmptyEvents(t *testing.T) { - run := langsmith.RunSchema{ - ID: "run-123", - TraceID: "trace-456", - Name: "ChatOpenAI", - RunType: "llm", - StartTime: time.Now(), - } - - result := ExtractRun(run, false, true, false) - - if result["events"] != nil { - t.Errorf("expected events=nil for run with no events, got %v", result["events"]) - } } func TestExtractRunNilParent(t *testing.T) { diff --git a/scripts/install.sh b/scripts/install.sh index 284ff6c..57dc130 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -39,29 +39,9 @@ fi # Get version if [ -z "$VERSION" ]; then - API_URL="https://api.github.com/repos/${REPO}/releases/latest" - - fetch_latest() { - if [ -n "$GITHUB_TOKEN" ]; then - curl -fsSL -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_TOKEN" "$API_URL" - else - curl -fsSL -H "Accept: application/vnd.github+json" "$API_URL" - fi - } - - VERSION="" - i=1 - while [ "$i" -le 3 ]; do - VERSION="$(fetch_latest | grep '"tag_name"' | sed -E 's/.*"tag_name": *"([^"]+)".*/\1/')" - [ -n "$VERSION" ] && break - [ "$i" -lt 3 ] && sleep "$i" - i=$((i + 1)) - done - + VERSION="$(curl -sSL "https://api.github.com/repos/${REPO}/releases/latest" | grep '"tag_name"' | sed -E 's/.*"tag_name": *"([^"]+)".*/\1/')" if [ -z "$VERSION" ]; then - echo "Failed to determine latest version (GitHub API unreachable or rate-limited)." >&2 - echo "Set GITHUB_TOKEN to raise the rate limit, or set VERSION to install a specific release." >&2 + echo "Failed to determine latest version" >&2 exit 1 fi fi From 0bb958dbd24df486bced44fcfbc89aa80182f847 Mon Sep 17 00:00:00 2001 From: Ramon Nogueira Date: Tue, 16 Jun 2026 21:14:06 -0400 Subject: [PATCH 2/9] fix(cli): resolve `api` shorthand paths against the cached spec (#27678) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What `langsmith api ` (a path with no leading `/`) hardcoded an `/api/v1/` prefix in both `resolveEndpoint` and `LookupEndpoint`. Any surface not under `/api/v1` — `/v2/*` (all sandboxes, 18 endpoints), `/commits`, `/me`, `/orgs`, etc. — was rewritten to `/api/v1/v2/...` and silently misrouted to the Python backend, returning a confusing `404`. This resolves shorthands against the cached OpenAPI spec: prefer `/api/v1/`, then an exact `/`, then a unique spec path whose trailing segments match (so `sandboxes/boxes` → `/v2/sandboxes/boxes`), falling back to `/api/v1/` when the spec isn't cached. Absolute paths (`/v2/...`) and full URLs are unchanged. Resolution is **cache-only** — it reads the spec the browse commands (`api ls`/`api info`) already populate and never fetches, so there's no network round-trip added to the request hot path. ## Test Plan - [x] `go build ./...` + `go test ./internal/cmd/api/` pass - [x] New table tests in `resolve_test.go` cover spec-backed shorthand resolution, query-string preservation, fallback, and absolute-path passthrough - [x] Verified against dev: `langsmith api sandboxes/boxes` now resolves to `/v2/sandboxes/boxes` (200), previously a `uvicorn` 404 GitOrigin-RevId: 1409b98a30e526cee5f2161d8f62ffebc690a60e --- internal/cmd/api/api.go | 9 ++++++- internal/cmd/api/request.go | 4 +-- internal/cmd/api/request_test.go | 22 +++++++-------- internal/cmd/api/resolve.go | 18 ++++++++++--- internal/cmd/api/resolve_test.go | 35 +++++++++++++++++++++++- internal/cmd/api/spec.go | 46 ++++++++++++++++++++++++++++++-- 6 files changed, 114 insertions(+), 20 deletions(-) diff --git a/internal/cmd/api/api.go b/internal/cmd/api/api.go index 2bbfe14..d3a8131 100644 --- a/internal/cmd/api/api.go +++ b/internal/cmd/api/api.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/langchain-ai/langsmith-cli/internal/cache" "github.com/langchain-ai/langsmith-cli/internal/cmdutil" "github.com/spf13/cobra" ) @@ -65,8 +66,14 @@ Make requests: return err } + // Resolve shorthand paths against the cached spec so non-/api/v1 surfaces (e.g. /v2/sandboxes/boxes) work without a leading slash. + var spec *OpenAPISpec + if !strings.HasPrefix(path, "/") && !strings.HasPrefix(path, "http://") && !strings.HasPrefix(path, "https://") { + spec = cachedSpec(c.APIURL(), cache.DefaultDir()) + } + w := cmd.OutOrStdout() - statusCode, err := runRequest(c, method, path, body, input, params, headers, include, w) + statusCode, err := runRequest(c, method, path, body, input, params, headers, include, spec, w) if err != nil { return err } diff --git a/internal/cmd/api/request.go b/internal/cmd/api/request.go index efeef26..5ff1238 100644 --- a/internal/cmd/api/request.go +++ b/internal/cmd/api/request.go @@ -16,9 +16,9 @@ import ( // runRequest executes an HTTP request and writes the response to w. // Returns the HTTP status code and any transport-level error. -func runRequest(c *client.Client, method, path, body, input string, params map[string]any, headers []string, include bool, w io.Writer) (int, error) { +func runRequest(c *client.Client, method, path, body, input string, params map[string]any, headers []string, include bool, spec *OpenAPISpec, w io.Writer) (int, error) { apiURL := c.APIURL() - fullURL := resolveEndpoint(apiURL, path) + fullURL := resolveEndpoint(apiURL, path, spec) if len(params) > 0 && strings.EqualFold(method, "GET") { fullURL = addQueryParams(fullURL, params) } diff --git a/internal/cmd/api/request_test.go b/internal/cmd/api/request_test.go index 958a1ee..545c8d0 100644 --- a/internal/cmd/api/request_test.go +++ b/internal/cmd/api/request_test.go @@ -31,7 +31,7 @@ func TestRunRequest_GET(t *testing.T) { var out bytes.Buffer c := client.New("test-key", ts.URL) - code, err := runRequest(c, "GET", "sessions", "", "", nil, nil, false, &out) + code, err := runRequest(c, "GET", "sessions", "", "", nil, nil, false, nil, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -61,7 +61,7 @@ func TestRunRequest_POSTWithBody(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) - code, err := runRequest(c, "POST", "sessions", `{"name":"test"}`, "", nil, nil, false, &out) + code, err := runRequest(c, "POST", "sessions", `{"name":"test"}`, "", nil, nil, false, nil, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -94,7 +94,7 @@ func TestRunRequest_POSTWithFields(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) params := map[string]any{"name": "test", "limit": 10} - code, err := runRequest(c, "POST", "sessions", "", "", params, nil, false, &out) + code, err := runRequest(c, "POST", "sessions", "", "", params, nil, false, nil, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -126,7 +126,7 @@ func TestRunRequest_GETWithFieldsAddsQuery(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) params := map[string]any{"name": "test", "limit": 10} - _, err := runRequest(c, "GET", "sessions?existing=1", "", "", params, nil, false, &out) + _, err := runRequest(c, "GET", "sessions?existing=1", "", "", params, nil, false, nil, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -144,7 +144,7 @@ func TestRunRequest_ExtraHeaders(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) - _, err := runRequest(c, "GET", "sessions", "", "", nil, []string{"X-Custom:val"}, false, &out) + _, err := runRequest(c, "GET", "sessions", "", "", nil, []string{"X-Custom:val"}, false, nil, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -160,7 +160,7 @@ func TestRunRequest_Include(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) - _, err := runRequest(c, "GET", "sessions", "", "", nil, nil, true, &out) + _, err := runRequest(c, "GET", "sessions", "", "", nil, nil, true, nil, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -181,7 +181,7 @@ func TestRunRequest_4xxPrintsBody(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) - code, err := runRequest(c, "GET", "sessions", "", "", nil, nil, false, &out) + code, err := runRequest(c, "GET", "sessions", "", "", nil, nil, false, nil, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -207,7 +207,7 @@ func TestRunRequest_BodyFromFile(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) - code, err := runRequest(c, "POST", "sessions", "@"+f.Name(), "", nil, nil, false, &out) + code, err := runRequest(c, "POST", "sessions", "@"+f.Name(), "", nil, nil, false, nil, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -234,7 +234,7 @@ func TestRunRequest_FullURLDifferentHost(t *testing.T) { var out bytes.Buffer c := client.New("key", "https://different.host") - code, err := runRequest(c, "GET", ts.URL+"/custom/endpoint", "", "", nil, nil, false, &out) + code, err := runRequest(c, "GET", ts.URL+"/custom/endpoint", "", "", nil, nil, false, nil, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -256,7 +256,7 @@ func TestRunRequest_MultiValueHeaders(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) - _, err := runRequest(c, "GET", "sessions", "", "", nil, []string{"X-Multi:one", "X-Multi:two"}, false, &out) + _, err := runRequest(c, "GET", "sessions", "", "", nil, []string{"X-Multi:one", "X-Multi:two"}, false, nil, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -284,7 +284,7 @@ func TestRunRequest_PrefixConfusionAttack(t *testing.T) { c := client.New("secret-key", apiURL) var out bytes.Buffer - code, err := runRequest(c, "GET", ts.URL+"/steal", "", "", nil, nil, false, &out) + code, err := runRequest(c, "GET", ts.URL+"/steal", "", "", nil, nil, false, nil, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/internal/cmd/api/resolve.go b/internal/cmd/api/resolve.go index 91203c6..568467c 100644 --- a/internal/cmd/api/resolve.go +++ b/internal/cmd/api/resolve.go @@ -7,8 +7,9 @@ import ( // resolveEndpoint resolves an endpoint argument to a full URL. // - Full URL (http:// or https://) → returned as-is. // - Absolute path (starts with /) → baseURL + path. -// - Shorthand (e.g. "sessions") → baseURL + /api/v1/ + path. -func resolveEndpoint(baseURL, path string) string { +// - Shorthand (e.g. "sessions") → resolved against spec when available, +// falling back to baseURL + /api/v1/ + path. spec may be nil. +func resolveEndpoint(baseURL, path string, spec *OpenAPISpec) string { baseURL = strings.TrimRight(baseURL, "/") if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { return path @@ -16,7 +17,18 @@ func resolveEndpoint(baseURL, path string) string { if strings.HasPrefix(path, "/") { return baseURL + path } - return baseURL + "/api/v1/" + path + rel, query, hasQuery := strings.Cut(path, "?") + resolved := "" + if spec != nil { + resolved = spec.resolveShorthand(rel) + } + if resolved == "" { + resolved = "/api/v1/" + rel + } + if hasQuery { + resolved += "?" + query + } + return baseURL + resolved } // isHTTPMethod returns true if s is an uppercase HTTP method name. diff --git a/internal/cmd/api/resolve_test.go b/internal/cmd/api/resolve_test.go index 172845d..171e2b5 100644 --- a/internal/cmd/api/resolve_test.go +++ b/internal/cmd/api/resolve_test.go @@ -1,6 +1,7 @@ package api import ( + "encoding/json" "testing" ) @@ -22,7 +23,7 @@ func TestResolveEndpoint(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := resolveEndpoint(tt.baseURL, tt.path) + got := resolveEndpoint(tt.baseURL, tt.path, nil) if got != tt.want { t.Errorf("resolveEndpoint(%q, %q) = %q, want %q", tt.baseURL, tt.path, got, tt.want) } @@ -30,6 +31,38 @@ func TestResolveEndpoint(t *testing.T) { } } +func TestResolveEndpointWithSpec(t *testing.T) { + spec := &OpenAPISpec{ + Paths: map[string]map[string]json.RawMessage{ + "/api/v1/sessions": {"get": json.RawMessage(`{}`)}, + "/v2/sandboxes/boxes": {"post": json.RawMessage(`{}`)}, + "/commits": {"post": json.RawMessage(`{}`)}, + "/v2/sandboxes/widgets": {"get": json.RawMessage(`{}`)}, + }, + } + base := "https://api.smith.langchain.com" + tests := []struct { + name string + path string + want string + }{ + {"api/v1 shorthand still preferred", "sessions", base + "/api/v1/sessions"}, + {"v2 shorthand resolves via spec", "sandboxes/boxes", base + "/v2/sandboxes/boxes"}, + {"non-api-v1 root resolves", "commits", base + "/commits"}, + {"shorthand with query preserved", "sandboxes/boxes?foo=1", base + "/v2/sandboxes/boxes?foo=1"}, + {"unknown shorthand falls back to /api/v1", "nope/missing", base + "/api/v1/nope/missing"}, + {"absolute path untouched", "/v2/sandboxes/boxes", base + "/v2/sandboxes/boxes"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := resolveEndpoint(base, tt.path, spec) + if got != tt.want { + t.Errorf("resolveEndpoint(%q, %q) = %q, want %q", base, tt.path, got, tt.want) + } + }) + } +} + func TestIsHTTPMethod(t *testing.T) { for _, m := range []string{"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"} { if !isHTTPMethod(m) { diff --git a/internal/cmd/api/spec.go b/internal/cmd/api/spec.go index f3604b2..90c115f 100644 --- a/internal/cmd/api/spec.go +++ b/internal/cmd/api/spec.go @@ -86,13 +86,42 @@ func (s *OpenAPISpec) Endpoints() []Endpoint { return endpoints } +// resolveShorthand maps a shorthand path to an absolute spec path (prefer /api/v1/, then exact /, then a unique suffix match); "" if ambiguous. +func (s *OpenAPISpec) resolveShorthand(path string) string { + if candidate := "/api/v1/" + path; s.hasPath(candidate) { + return candidate + } + if candidate := "/" + path; s.hasPath(candidate) { + return candidate + } + suffix := "/" + path + var match string + for key := range s.Paths { + if strings.HasSuffix(key, suffix) { + if match != "" { + return "" // ambiguous + } + match = key + } + } + return match +} + +func (s *OpenAPISpec) hasPath(path string) bool { + _, ok := s.Paths[path] + return ok +} + // LookupEndpoint finds an endpoint by method and path, returning full detail. // The path argument can be shorthand ("sessions") or absolute ("/api/v1/sessions"). func (s *OpenAPISpec) LookupEndpoint(method, path string) (*EndpointDetail, error) { - // Normalize: if shorthand, prefix /api/v1/ normalized := path if !strings.HasPrefix(normalized, "/") { - normalized = "/api/v1/" + normalized + if resolved := s.resolveShorthand(normalized); resolved != "" { + normalized = resolved + } else { + normalized = "/api/v1/" + normalized + } } method = strings.ToUpper(method) @@ -266,6 +295,19 @@ func (s *OpenAPISpec) resolveComponentRef(ref string) any { return schema } +// cachedSpec returns the cached spec (nil on miss); it never fetches, so it is safe on the request hot path. +func cachedSpec(apiURL, cacheDir string) *OpenAPISpec { + data, err := cache.ReadIfFresh(cache.PathForKey(cacheDir, "openapi", apiURL), specCacheTTL) + if err != nil { + return nil + } + var spec OpenAPISpec + if err := json.Unmarshal(data, &spec); err != nil { + return nil + } + return &spec +} + // loadSpec loads the OpenAPI spec, using cache if available and not expired. func loadSpec(apiURL, cacheDir string, forceRefresh bool) (*OpenAPISpec, error) { cachePath := cache.PathForKey(cacheDir, "openapi", apiURL) From 316167347cfedf9f1d447faa34124d046c843097 Mon Sep 17 00:00:00 2001 From: Ramon Nogueira Date: Tue, 30 Jun 2026 15:01:58 -0400 Subject: [PATCH 3/9] chore(sdks): vendor langsmith-go/cli, build CLI against in-repo SDK [copybara] (#28676) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Imports the latest `main` from the upstream OSS repos via the Copybara vendor workflows: - `sdks/go` ← `langchain-ai/langsmith-go` (`./scripts/copybara copy.bara.sky vendor_go --folder-dir sdks/go`) - `sdks/cli` ← `langchain-ai/langsmith-cli` (`./scripts/copybara copy.bara.sky vendor_cli --folder-dir sdks/cli`) Also adds a reversible `copy.bara.sky` transform (`CLI_GO_MOD_REPLACE`) that injects `replace github.com/langchain-ai/langsmith-go => ../go` into the vendored CLI's `go.mod` on `vendor_cli`, so the CLI always builds against the in-repo `sdks/go` instead of the published module — the two vendors can no longer drift out of sync. The transform is reversed on `export_cli` so upstream `langsmith-cli` never carries the local path. No hand edits to vendored files. ## Test Plan - [x] `go build ./...` passes in `sdks/go` and `sdks/cli` - [x] `sdks/cli` builds against the in-repo `sdks/go` via the injected `replace` directive GitOrigin-RevId: c0b787a61bc1020db37497bdf78bd6857d673b2f --- .github/workflows/sync-rc-branches.yml | 65 ------ README.md | 8 + go.mod | 14 +- go.sum | 28 +-- internal/cmd/api/api.go | 9 +- internal/cmd/api/request.go | 4 +- internal/cmd/api/request_test.go | 22 +-- internal/cmd/api/resolve.go | 18 +- internal/cmd/api/resolve_test.go | 35 +--- internal/cmd/api/spec.go | 46 +---- internal/cmd/filters.go | 7 + internal/cmd/helpers.go | 239 +++++++++++++++++++++- internal/cmd/helpers_test.go | 44 +++++ internal/cmd/hub.go | 1 + internal/cmd/hub_list.go | 10 +- internal/cmd/hub_list_test.go | 15 +- internal/cmd/hub_repo_sdk.go | 1 + internal/cmd/message.go | 253 +++++++++++++++++++++--- internal/cmd/message_trajectory_test.go | 177 +++++++++++++++++ internal/cmd/run.go | 80 +++++--- internal/cmd/thread.go | 4 +- internal/cmd/trace.go | 41 ++-- internal/cmd/trace_stats.go | 16 +- internal/extract/extract.go | 15 +- internal/extract/extract_test.go | 6 +- 25 files changed, 857 insertions(+), 301 deletions(-) delete mode 100644 .github/workflows/sync-rc-branches.yml create mode 100644 internal/cmd/message_trajectory_test.go diff --git a/.github/workflows/sync-rc-branches.yml b/.github/workflows/sync-rc-branches.yml deleted file mode 100644 index fb6b10d..0000000 --- a/.github/workflows/sync-rc-branches.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Sync RC branches with upstream - -# Keeps the engine-facing RC branches in sync with their upstream integration -# branches while the v2 runs-query work is consumed pre-release: -# - langsmith-go rc/runs-v2-engine <- next (custom code lives off `next`) -# - langsmith-cli rc/runs-v2-engine <- main -# -# Merges (never rebases/force-pushes) so engine's go.mod pin to the RC HEAD -# stays valid. On conflict it leaves the last-good RC untouched and opens a -# single deduped issue for manual resolution. -# -# Requires a secret RC_SYNC_TOKEN with contents:write + issues:write on BOTH -# repos. The default GITHUB_TOKEN is repo-scoped and cannot push cross-repo. - -permissions: {} - -on: - schedule: - - cron: '23 */4 * * *' # every 4 hours - workflow_dispatch: {} - -concurrency: - group: sync-rc-runs-v2-engine - cancel-in-progress: false - -jobs: - sync: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - { repo: langchain-ai/langsmith-go, upstream: next, rc: rc/runs-v2-engine } - - { repo: langchain-ai/langsmith-cli, upstream: main, rc: rc/runs-v2-engine } - steps: - - name: Merge ${{ matrix.upstream }} into ${{ matrix.rc }} (${{ matrix.repo }}) - env: - GH_TOKEN: ${{ secrets.RC_SYNC_TOKEN }} - REPO: ${{ matrix.repo }} - UPSTREAM: ${{ matrix.upstream }} - RC: ${{ matrix.rc }} - run: | - set -euo pipefail - git config --global user.name "langsmith-rc-sync[bot]" - git config --global user.email "rc-sync@users.noreply.github.com" - - git clone --no-tags "https://x-access-token:${GH_TOKEN}@github.com/${REPO}.git" repo - cd repo - git fetch origin "$UPSTREAM" "$RC" - git checkout "$RC" - - if git merge --no-edit "origin/$UPSTREAM"; then - git push origin "HEAD:$RC" - echo "Synced $REPO: $UPSTREAM -> $RC" - else - git merge --abort - echo "::warning::Merge conflict syncing $UPSTREAM into $RC in $REPO" - TITLE="RC sync conflict: $UPSTREAM -> $RC" - BODY="Automated RC sync could not merge \`$UPSTREAM\` into \`$RC\`. Resolve manually: \`git checkout $RC && git merge origin/$UPSTREAM\`, then push." - EXISTING="$(gh issue list --repo "$REPO" --state open --search "$TITLE in:title" --json number --jq '.[0].number // empty')" - if [ -z "$EXISTING" ]; then - gh issue create --repo "$REPO" --title "$TITLE" --body "$BODY" - fi - exit 1 - fi diff --git a/README.md b/README.md index 8c5a9d7..06255cb 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,13 @@ langsmith run get --full langsmith run export llm_calls.jsonl --project my-app --run-type llm --full ``` +> **For agents querying runs:** prefer `--version v2` first (SmithDB-backed; faster on tenants that are rolled out). If the call fails with a 4xx (typically 403, 404, or 422), retry the **same command without** `--version` to fall back to v1. Example: +> +> ```bash +> langsmith run list --project my-app --version v2 \ +> || langsmith run list --project my-app +> ``` + ### `thread` — Query conversation threads A thread groups multiple root runs sharing a thread_id (multi-turn conversations). @@ -293,6 +300,7 @@ langsmith hub pull acme/my-skill:production --dir ./out # Discover, inspect, delete langsmith hub list --type skill --query foo +langsmith hub list --type skill --source external langsmith hub get acme/my-skill langsmith hub delete acme/my-skill --yes ``` diff --git a/go.mod b/go.mod index 30bdbf3..b5e57eb 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,14 @@ go 1.25.0 require ( github.com/google/uuid v1.6.0 github.com/itchyny/gojq v0.12.15 - github.com/langchain-ai/langsmith-go v0.10.0 + github.com/langchain-ai/langsmith-go v0.16.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pelletier/go-toml/v2 v2.2.4 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.11.1 github.com/xlab/treeprint v1.2.0 - golang.org/x/net v0.52.0 - golang.org/x/term v0.41.0 + golang.org/x/net v0.54.0 + golang.org/x/term v0.43.0 ) require ( @@ -24,12 +24,12 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect - github.com/klauspost/compress v1.18.4 // indirect + github.com/klauspost/compress v1.18.6 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/gjson v1.19.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect @@ -41,8 +41,8 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect - golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.35.0 // indirect + golang.org/x/sys v0.44.0 // indirect + golang.org/x/text v0.37.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/grpc v1.80.0 // indirect diff --git a/go.sum b/go.sum index 9711832..717fc12 100644 --- a/go.sum +++ b/go.sum @@ -25,14 +25,14 @@ github.com/itchyny/gojq v0.12.15 h1:WC1Nxbx4Ifw5U2oQWACYz32JK8G9qxNtHzrvW4KEcqI= github.com/itchyny/gojq v0.12.15/go.mod h1:uWAHCbCIla1jiNxmeT5/B5mOjSdfkCq6p8vxWg+BM10= github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= -github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= -github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/langchain-ai/langsmith-go v0.10.0 h1:crrioqgBBKY+rTYhFW0NujhxBUoq6iARv12bP9xR7mg= -github.com/langchain-ai/langsmith-go v0.10.0/go.mod h1:hUFPP9siu6sGSodcZo/c3uUG5xcXRdqZ8gN899R1OyE= +github.com/langchain-ai/langsmith-go v0.16.0 h1:1kqrrNdIdYvoLdWPFBOj+iTtP8fayAJgwZJW+/RaQVA= +github.com/langchain-ai/langsmith-go v0.16.0/go.mod h1:ghwFdWr6zS7s3rdMMtEQDU1xWzineFzXa1SEG2XA3XY= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -57,8 +57,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= -github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= +github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -88,14 +88,14 @@ go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpu go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= -golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= +golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= diff --git a/internal/cmd/api/api.go b/internal/cmd/api/api.go index d3a8131..2bbfe14 100644 --- a/internal/cmd/api/api.go +++ b/internal/cmd/api/api.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" - "github.com/langchain-ai/langsmith-cli/internal/cache" "github.com/langchain-ai/langsmith-cli/internal/cmdutil" "github.com/spf13/cobra" ) @@ -66,14 +65,8 @@ Make requests: return err } - // Resolve shorthand paths against the cached spec so non-/api/v1 surfaces (e.g. /v2/sandboxes/boxes) work without a leading slash. - var spec *OpenAPISpec - if !strings.HasPrefix(path, "/") && !strings.HasPrefix(path, "http://") && !strings.HasPrefix(path, "https://") { - spec = cachedSpec(c.APIURL(), cache.DefaultDir()) - } - w := cmd.OutOrStdout() - statusCode, err := runRequest(c, method, path, body, input, params, headers, include, spec, w) + statusCode, err := runRequest(c, method, path, body, input, params, headers, include, w) if err != nil { return err } diff --git a/internal/cmd/api/request.go b/internal/cmd/api/request.go index 5ff1238..efeef26 100644 --- a/internal/cmd/api/request.go +++ b/internal/cmd/api/request.go @@ -16,9 +16,9 @@ import ( // runRequest executes an HTTP request and writes the response to w. // Returns the HTTP status code and any transport-level error. -func runRequest(c *client.Client, method, path, body, input string, params map[string]any, headers []string, include bool, spec *OpenAPISpec, w io.Writer) (int, error) { +func runRequest(c *client.Client, method, path, body, input string, params map[string]any, headers []string, include bool, w io.Writer) (int, error) { apiURL := c.APIURL() - fullURL := resolveEndpoint(apiURL, path, spec) + fullURL := resolveEndpoint(apiURL, path) if len(params) > 0 && strings.EqualFold(method, "GET") { fullURL = addQueryParams(fullURL, params) } diff --git a/internal/cmd/api/request_test.go b/internal/cmd/api/request_test.go index 545c8d0..958a1ee 100644 --- a/internal/cmd/api/request_test.go +++ b/internal/cmd/api/request_test.go @@ -31,7 +31,7 @@ func TestRunRequest_GET(t *testing.T) { var out bytes.Buffer c := client.New("test-key", ts.URL) - code, err := runRequest(c, "GET", "sessions", "", "", nil, nil, false, nil, &out) + code, err := runRequest(c, "GET", "sessions", "", "", nil, nil, false, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -61,7 +61,7 @@ func TestRunRequest_POSTWithBody(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) - code, err := runRequest(c, "POST", "sessions", `{"name":"test"}`, "", nil, nil, false, nil, &out) + code, err := runRequest(c, "POST", "sessions", `{"name":"test"}`, "", nil, nil, false, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -94,7 +94,7 @@ func TestRunRequest_POSTWithFields(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) params := map[string]any{"name": "test", "limit": 10} - code, err := runRequest(c, "POST", "sessions", "", "", params, nil, false, nil, &out) + code, err := runRequest(c, "POST", "sessions", "", "", params, nil, false, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -126,7 +126,7 @@ func TestRunRequest_GETWithFieldsAddsQuery(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) params := map[string]any{"name": "test", "limit": 10} - _, err := runRequest(c, "GET", "sessions?existing=1", "", "", params, nil, false, nil, &out) + _, err := runRequest(c, "GET", "sessions?existing=1", "", "", params, nil, false, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -144,7 +144,7 @@ func TestRunRequest_ExtraHeaders(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) - _, err := runRequest(c, "GET", "sessions", "", "", nil, []string{"X-Custom:val"}, false, nil, &out) + _, err := runRequest(c, "GET", "sessions", "", "", nil, []string{"X-Custom:val"}, false, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -160,7 +160,7 @@ func TestRunRequest_Include(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) - _, err := runRequest(c, "GET", "sessions", "", "", nil, nil, true, nil, &out) + _, err := runRequest(c, "GET", "sessions", "", "", nil, nil, true, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -181,7 +181,7 @@ func TestRunRequest_4xxPrintsBody(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) - code, err := runRequest(c, "GET", "sessions", "", "", nil, nil, false, nil, &out) + code, err := runRequest(c, "GET", "sessions", "", "", nil, nil, false, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -207,7 +207,7 @@ func TestRunRequest_BodyFromFile(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) - code, err := runRequest(c, "POST", "sessions", "@"+f.Name(), "", nil, nil, false, nil, &out) + code, err := runRequest(c, "POST", "sessions", "@"+f.Name(), "", nil, nil, false, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -234,7 +234,7 @@ func TestRunRequest_FullURLDifferentHost(t *testing.T) { var out bytes.Buffer c := client.New("key", "https://different.host") - code, err := runRequest(c, "GET", ts.URL+"/custom/endpoint", "", "", nil, nil, false, nil, &out) + code, err := runRequest(c, "GET", ts.URL+"/custom/endpoint", "", "", nil, nil, false, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -256,7 +256,7 @@ func TestRunRequest_MultiValueHeaders(t *testing.T) { var out bytes.Buffer c := client.New("key", ts.URL) - _, err := runRequest(c, "GET", "sessions", "", "", nil, []string{"X-Multi:one", "X-Multi:two"}, false, nil, &out) + _, err := runRequest(c, "GET", "sessions", "", "", nil, []string{"X-Multi:one", "X-Multi:two"}, false, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -284,7 +284,7 @@ func TestRunRequest_PrefixConfusionAttack(t *testing.T) { c := client.New("secret-key", apiURL) var out bytes.Buffer - code, err := runRequest(c, "GET", ts.URL+"/steal", "", "", nil, nil, false, nil, &out) + code, err := runRequest(c, "GET", ts.URL+"/steal", "", "", nil, nil, false, &out) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/internal/cmd/api/resolve.go b/internal/cmd/api/resolve.go index 568467c..91203c6 100644 --- a/internal/cmd/api/resolve.go +++ b/internal/cmd/api/resolve.go @@ -7,9 +7,8 @@ import ( // resolveEndpoint resolves an endpoint argument to a full URL. // - Full URL (http:// or https://) → returned as-is. // - Absolute path (starts with /) → baseURL + path. -// - Shorthand (e.g. "sessions") → resolved against spec when available, -// falling back to baseURL + /api/v1/ + path. spec may be nil. -func resolveEndpoint(baseURL, path string, spec *OpenAPISpec) string { +// - Shorthand (e.g. "sessions") → baseURL + /api/v1/ + path. +func resolveEndpoint(baseURL, path string) string { baseURL = strings.TrimRight(baseURL, "/") if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { return path @@ -17,18 +16,7 @@ func resolveEndpoint(baseURL, path string, spec *OpenAPISpec) string { if strings.HasPrefix(path, "/") { return baseURL + path } - rel, query, hasQuery := strings.Cut(path, "?") - resolved := "" - if spec != nil { - resolved = spec.resolveShorthand(rel) - } - if resolved == "" { - resolved = "/api/v1/" + rel - } - if hasQuery { - resolved += "?" + query - } - return baseURL + resolved + return baseURL + "/api/v1/" + path } // isHTTPMethod returns true if s is an uppercase HTTP method name. diff --git a/internal/cmd/api/resolve_test.go b/internal/cmd/api/resolve_test.go index 171e2b5..172845d 100644 --- a/internal/cmd/api/resolve_test.go +++ b/internal/cmd/api/resolve_test.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "testing" ) @@ -23,7 +22,7 @@ func TestResolveEndpoint(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := resolveEndpoint(tt.baseURL, tt.path, nil) + got := resolveEndpoint(tt.baseURL, tt.path) if got != tt.want { t.Errorf("resolveEndpoint(%q, %q) = %q, want %q", tt.baseURL, tt.path, got, tt.want) } @@ -31,38 +30,6 @@ func TestResolveEndpoint(t *testing.T) { } } -func TestResolveEndpointWithSpec(t *testing.T) { - spec := &OpenAPISpec{ - Paths: map[string]map[string]json.RawMessage{ - "/api/v1/sessions": {"get": json.RawMessage(`{}`)}, - "/v2/sandboxes/boxes": {"post": json.RawMessage(`{}`)}, - "/commits": {"post": json.RawMessage(`{}`)}, - "/v2/sandboxes/widgets": {"get": json.RawMessage(`{}`)}, - }, - } - base := "https://api.smith.langchain.com" - tests := []struct { - name string - path string - want string - }{ - {"api/v1 shorthand still preferred", "sessions", base + "/api/v1/sessions"}, - {"v2 shorthand resolves via spec", "sandboxes/boxes", base + "/v2/sandboxes/boxes"}, - {"non-api-v1 root resolves", "commits", base + "/commits"}, - {"shorthand with query preserved", "sandboxes/boxes?foo=1", base + "/v2/sandboxes/boxes?foo=1"}, - {"unknown shorthand falls back to /api/v1", "nope/missing", base + "/api/v1/nope/missing"}, - {"absolute path untouched", "/v2/sandboxes/boxes", base + "/v2/sandboxes/boxes"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := resolveEndpoint(base, tt.path, spec) - if got != tt.want { - t.Errorf("resolveEndpoint(%q, %q) = %q, want %q", base, tt.path, got, tt.want) - } - }) - } -} - func TestIsHTTPMethod(t *testing.T) { for _, m := range []string{"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"} { if !isHTTPMethod(m) { diff --git a/internal/cmd/api/spec.go b/internal/cmd/api/spec.go index 90c115f..f3604b2 100644 --- a/internal/cmd/api/spec.go +++ b/internal/cmd/api/spec.go @@ -86,42 +86,13 @@ func (s *OpenAPISpec) Endpoints() []Endpoint { return endpoints } -// resolveShorthand maps a shorthand path to an absolute spec path (prefer /api/v1/, then exact /, then a unique suffix match); "" if ambiguous. -func (s *OpenAPISpec) resolveShorthand(path string) string { - if candidate := "/api/v1/" + path; s.hasPath(candidate) { - return candidate - } - if candidate := "/" + path; s.hasPath(candidate) { - return candidate - } - suffix := "/" + path - var match string - for key := range s.Paths { - if strings.HasSuffix(key, suffix) { - if match != "" { - return "" // ambiguous - } - match = key - } - } - return match -} - -func (s *OpenAPISpec) hasPath(path string) bool { - _, ok := s.Paths[path] - return ok -} - // LookupEndpoint finds an endpoint by method and path, returning full detail. // The path argument can be shorthand ("sessions") or absolute ("/api/v1/sessions"). func (s *OpenAPISpec) LookupEndpoint(method, path string) (*EndpointDetail, error) { + // Normalize: if shorthand, prefix /api/v1/ normalized := path if !strings.HasPrefix(normalized, "/") { - if resolved := s.resolveShorthand(normalized); resolved != "" { - normalized = resolved - } else { - normalized = "/api/v1/" + normalized - } + normalized = "/api/v1/" + normalized } method = strings.ToUpper(method) @@ -295,19 +266,6 @@ func (s *OpenAPISpec) resolveComponentRef(ref string) any { return schema } -// cachedSpec returns the cached spec (nil on miss); it never fetches, so it is safe on the request hot path. -func cachedSpec(apiURL, cacheDir string) *OpenAPISpec { - data, err := cache.ReadIfFresh(cache.PathForKey(cacheDir, "openapi", apiURL), specCacheTTL) - if err != nil { - return nil - } - var spec OpenAPISpec - if err := json.Unmarshal(data, &spec); err != nil { - return nil - } - return &spec -} - // loadSpec loads the OpenAPI spec, using cache if available and not expired. func loadSpec(apiURL, cacheDir string, forceRefresh bool) (*OpenAPISpec, error) { cachePath := cache.PathForKey(cacheDir, "openapi", apiURL) diff --git a/internal/cmd/filters.go b/internal/cmd/filters.go index a8ae94b..98ffb8c 100644 --- a/internal/cmd/filters.go +++ b/internal/cmd/filters.go @@ -15,6 +15,7 @@ type FilterFlags struct { TraceIDs string Limit int Project string + ProjectID string LastNMinutes int Since string Before string @@ -29,6 +30,7 @@ type FilterFlags struct { Tags string Metadata string RawFilter string + Version string } // addCommonFilterFlags attaches shared filter flags to a command. @@ -55,6 +57,11 @@ func addCommonFilterFlags(cmd *cobra.Command, f *FilterFlags, includeRunType boo } } +// addVersionFlag attaches the --version flag for selecting the runs query backend. +func addVersionFlag(cmd *cobra.Command, f *FilterFlags) { + cmd.Flags().StringVar(&f.Version, "version", "", `Query API version: "" (v1, default) or "v2" (SmithDB)`) +} + // resolveStartTime returns the start time for a query. // Priority: lastNMinutes > since > default (7 days ago). func resolveStartTime(since string, lastNMinutes int) time.Time { diff --git a/internal/cmd/helpers.go b/internal/cmd/helpers.go index 8eccdff..1089e79 100644 --- a/internal/cmd/helpers.go +++ b/internal/cmd/helpers.go @@ -2,7 +2,9 @@ package cmd import ( "context" + "encoding/json" "fmt" + "strings" "time" "github.com/langchain-ai/langsmith-cli/internal/client" @@ -13,15 +15,34 @@ import ( "github.com/google/uuid" ) -// queryRuns queries runs with the given params and optional session resolution. -// minTokens > 0 enables client-side filtering by total_tokens (not supported server-side). -func queryRuns(ctx context.Context, c *client.Client, params langsmith.RunQueryParams, projectName string, limit int, minTokens int) ([]langsmith.RunSchema, error) { - // Resolve project name → session ID - if projectName != "" { - sessionID, err := c.ResolveSessionID(ctx, projectName) - if err != nil { - return nil, err +// resolveSessionID resolves the target session (project) UUID for a command from +// either an explicit project ID or a project name. Precedence: +// +// --project-id (an explicit session UUID; takes precedence) → +// --project / $LANGSMITH_PROJECT (a project name, looked up via the API). +// +// projectID, when set, must be a well-formed UUID and is returned as-is without a +// name lookup (saving a Sessions.List round-trip). cmdName is used only to build +// the "required" error message. +func resolveSessionID(ctx context.Context, c *client.Client, projectName, projectID, cmdName string) (string, error) { + if projectID != "" { + if _, err := uuid.Parse(projectID); err != nil { + return "", fmt.Errorf("invalid --project-id %q: must be a project (session) UUID", projectID) } + return projectID, nil + } + name := ResolveProject(projectName) + if name == "" { + return "", fmt.Errorf("--project or --project-id is required for %s (or set LANGSMITH_PROJECT)", cmdName) + } + return c.ResolveSessionID(ctx, name) +} + +// queryRuns queries runs with the given params, scoped to sessionID when non-empty. +// The caller is responsible for resolving sessionID (see resolveSessionID). +// minTokens > 0 enables client-side filtering by total_tokens (not supported server-side). +func queryRuns(ctx context.Context, c *client.Client, params langsmith.RunQueryParams, sessionID string, limit int, minTokens int) ([]langsmith.RunSchema, error) { + if sessionID != "" { params.Session = langsmith.F([]string{sessionID}) } @@ -47,15 +68,213 @@ func queryRuns(ctx context.Context, c *client.Client, params langsmith.RunQueryP } // Check for next cursor - if resp.Cursors == nil || resp.Cursors["next"] == "" || remaining <= 0 { + if resp.Cursors.Next == "" || remaining <= 0 { + break + } + params.Cursor = langsmith.F(resp.Cursors.Next) + } + + return allRuns, nil +} + +// queryRunsV2 queries runs against the v2 (SmithDB) endpoint, resolves the +// project name to a session UUID, paginates on next_cursor, and normalizes +// each v2 Run back into v1's RunSchema so downstream rendering is unchanged. +// minTokens > 0 enables client-side filtering by total_tokens. +func queryRunsV2(ctx context.Context, c *client.Client, params langsmith.RunQueryV2Params, sessionID string, limit int, minTokens int) ([]langsmith.RunSchema, error) { + if sessionID != "" { + params.ProjectIDs = langsmith.F([]string{sessionID}) + } + + var allRuns []langsmith.RunSchema + + iter := c.SDK.Runs.QueryV2AutoPaging(ctx, params) + for iter.Next() { + if len(allRuns) >= limit { break } - params.Cursor = langsmith.F(resp.Cursors["next"]) + run := runV2ToSchema(iter.Current()) + if minTokens > 0 && run.TotalTokens < int64(minTokens) { + continue + } + allRuns = append(allRuns, run) + } + if err := iter.Err(); err != nil { + return nil, fmt.Errorf("querying runs (v2): %w", err) } return allRuns, nil } +// buildRunQueryV2Params translates FilterFlags into a v2 query body. The +// project name is left for queryRunsV2 to resolve into ProjectIDs. +func buildRunQueryV2Params(f *FilterFlags, isRoot bool, defaultLimit int) langsmith.RunQueryV2Params { + var params langsmith.RunQueryV2Params + + limit := defaultLimit + if f.Limit > 0 { + limit = f.Limit + } + params.PageSize = langsmith.F(int64(limit)) + + if isRoot { + params.IsRoot = langsmith.F(true) + } + + params.MinStartTime = langsmith.F(resolveStartTime(f.Since, f.LastNMinutes)) + if f.Before != "" { + t, err := time.Parse(time.RFC3339, f.Before) + if err != nil { + t, err = time.Parse("2006-01-02T15:04:05", f.Before) + if err != nil { + ExitErrorf("invalid --before timestamp: %s", f.Before) + } + } + params.MaxStartTime = langsmith.F(t.UTC()) + } + + if f.RunType != "" { + params.RunType = langsmith.F(langsmith.RunQueryV2ParamsRunType(strings.ToUpper(f.RunType))) + } + + if f.ErrorFlag { + params.HasError = langsmith.F(true) + } else if f.NoErrorFlag { + params.HasError = langsmith.F(false) + } + + if f.TraceIDs != "" { + ids := splitTrim(f.TraceIDs) + if len(ids) == 1 { + params.TraceID = langsmith.F(ids[0]) + } + } + + if s := buildFilterDSL(f); s != "" { + params.Filter = langsmith.F(s) + } + + return params +} + +// buildRunSelectV2 returns the v2 select-field set covering the same base +// fields used by the downstream RunSchema pipeline plus the optional groups +// requested by the include flags. +func buildRunSelectV2(includeIO, includeFeedback bool) []langsmith.RunQueryV2ParamsSelect { + fields := []langsmith.RunQueryV2ParamsSelect{ + langsmith.RunQueryV2ParamsSelectID, + langsmith.RunQueryV2ParamsSelectTraceID, + langsmith.RunQueryV2ParamsSelectName, + langsmith.RunQueryV2ParamsSelectRunType, + langsmith.RunQueryV2ParamsSelectStartTime, + langsmith.RunQueryV2ParamsSelectEndTime, + langsmith.RunQueryV2ParamsSelectParentRunIDs, + langsmith.RunQueryV2ParamsSelectProjectID, + langsmith.RunQueryV2ParamsSelectDottedOrder, + langsmith.RunQueryV2ParamsSelectIsRoot, + langsmith.RunQueryV2ParamsSelectExtra, + langsmith.RunQueryV2ParamsSelectMetadata, + langsmith.RunQueryV2ParamsSelectTags, + langsmith.RunQueryV2ParamsSelectPromptTokens, + langsmith.RunQueryV2ParamsSelectCompletionTokens, + langsmith.RunQueryV2ParamsSelectTotalTokens, + langsmith.RunQueryV2ParamsSelectPromptCost, + langsmith.RunQueryV2ParamsSelectCompletionCost, + langsmith.RunQueryV2ParamsSelectTotalCost, + langsmith.RunQueryV2ParamsSelectLatencySeconds, + langsmith.RunQueryV2ParamsSelectAppPath, + } + if includeIO { + fields = append(fields, + langsmith.RunQueryV2ParamsSelectInputs, + langsmith.RunQueryV2ParamsSelectOutputs, + langsmith.RunQueryV2ParamsSelectError, + ) + } + if includeFeedback { + fields = append(fields, langsmith.RunQueryV2ParamsSelectFeedbackStats) + } + return fields +} + +// runV2ToSchema converts a v2 Run into the legacy v1 RunSchema shape so the +// existing extract/output pipeline can consume it unchanged. +func runV2ToSchema(r langsmith.QueryRunResponse) langsmith.RunSchema { + extra := asMap(r.Extra) + if md := asMap(r.Metadata); md != nil { + if extra == nil { + extra = map[string]interface{}{} + } + if _, ok := extra["metadata"]; !ok { + extra["metadata"] = md + } + } + + out := langsmith.RunSchema{ + ID: r.ID, + TraceID: r.TraceID, + Name: r.Name, + RunType: langsmith.RunTypeEnum(strings.ToLower(string(r.RunType))), + StartTime: r.StartTime, + EndTime: r.EndTime, + FirstTokenTime: r.FirstTokenTime, + SessionID: r.ProjectID, + DottedOrder: r.DottedOrder, + Tags: r.Tags, + ThreadID: r.ThreadID, + AppPath: r.AppPath, + TotalTokens: r.TotalTokens, + PromptTokens: r.PromptTokens, + CompletionTokens: r.CompletionTokens, + TotalCost: r.TotalCost, + PromptCost: r.PromptCost, + CompletionCost: r.CompletionCost, + Error: r.Error, + InputsPreview: r.InputsPreview, + OutputsPreview: r.OutputsPreview, + ReferenceExampleID: r.ReferenceExampleID, + ReferenceDatasetID: r.ReferenceDatasetID, + PriceModelID: r.PriceModelID, + InDataset: r.IsInDataset, + Inputs: asMap(r.Inputs), + Outputs: asMap(r.Outputs), + Extra: extra, + FeedbackStats: convertFeedbackStats(r.FeedbackStats), + } + if len(r.ParentRunIDs) > 0 { + out.ParentRunIDs = r.ParentRunIDs + out.ParentRunID = r.ParentRunIDs[len(r.ParentRunIDs)-1] + } + return out +} + +// asMap returns v as a map[string]interface{} when it is one, else nil. +func asMap(v interface{}) map[string]interface{} { + m, ok := v.(map[string]interface{}) + if !ok { + return nil + } + return m +} + +// convertFeedbackStats round-trips the generated feedback-stats shape into the +// loosely-typed map the downstream pipeline expects. Returns nil on empty input +// or any marshalling error. +func convertFeedbackStats(stats map[string]langsmith.QueryRunResponseFeedbackStat) map[string]map[string]interface{} { + if len(stats) == 0 { + return nil + } + raw, err := json.Marshal(stats) + if err != nil { + return nil + } + var m map[string]map[string]interface{} + if err := json.Unmarshal(raw, &m); err != nil { + return nil + } + return m +} + // buildRunSelect returns the Select fields needed for the given include flags. // Returns nil when neither IO nor feedback is requested, letting the API use its defaults. // When set, includes all base/metadata fields so they aren't stripped from the response. diff --git a/internal/cmd/helpers_test.go b/internal/cmd/helpers_test.go index 631ecdc..0cc0bf0 100644 --- a/internal/cmd/helpers_test.go +++ b/internal/cmd/helpers_test.go @@ -1,6 +1,8 @@ package cmd import ( + "context" + "strings" "testing" "time" @@ -8,6 +10,48 @@ import ( langsmith "github.com/langchain-ai/langsmith-go" ) +// ---------- resolveSessionID ---------- +// +// These cases all resolve (or error) before any API call, so a nil client is +// safe. The project-name → session-ID lookup path is exercised via the +// command-level tests that stub the client. + +func TestResolveSessionID_ProjectIDTakesPrecedence(t *testing.T) { + // A valid --project-id is returned as-is, even when a project name is set + // via --project / $LANGSMITH_PROJECT — without any name lookup. + t.Setenv("LANGSMITH_PROJECT", "some-project-name") + const id = "3f2504e0-4f89-41d3-9a0c-0305e82c3301" + + got, err := resolveSessionID(context.Background(), nil, "another-name", id, "trace get") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != id { + t.Errorf("expected session id %q returned verbatim, got %q", id, got) + } +} + +func TestResolveSessionID_InvalidProjectID(t *testing.T) { + _, err := resolveSessionID(context.Background(), nil, "", "not-a-uuid", "trace get") + if err == nil { + t.Fatal("expected error for malformed --project-id") + } + if !strings.Contains(err.Error(), "--project-id") { + t.Errorf("error should mention --project-id, got %q", err.Error()) + } +} + +func TestResolveSessionID_NeitherProvided(t *testing.T) { + t.Setenv("LANGSMITH_PROJECT", "") + _, err := resolveSessionID(context.Background(), nil, "", "", "trace stats") + if err == nil { + t.Fatal("expected error when neither --project nor --project-id is provided") + } + if !strings.Contains(err.Error(), "trace stats") { + t.Errorf("error should name the command, got %q", err.Error()) + } +} + // ---------- formatTimedelta ---------- func TestFormatTimedelta_Milliseconds(t *testing.T) { diff --git a/internal/cmd/hub.go b/internal/cmd/hub.go index 97bb7e7..3fccbe8 100644 --- a/internal/cmd/hub.go +++ b/internal/cmd/hub.go @@ -54,6 +54,7 @@ type hubRepo struct { Owner *string `json:"owner"` RepoHandle string `json:"repo_handle"` RepoType string `json:"repo_type"` + Source *string `json:"source"` Description *string `json:"description"` Readme *string `json:"readme"` IsPublic bool `json:"is_public"` diff --git a/internal/cmd/hub_list.go b/internal/cmd/hub_list.go index 645869a..1b020e1 100644 --- a/internal/cmd/hub_list.go +++ b/internal/cmd/hub_list.go @@ -13,6 +13,7 @@ import ( func newHubListCmd() *cobra.Command { var ( repoType string + source string query string publicOnly bool limit int @@ -33,7 +34,13 @@ func newHubListCmd() *cobra.Command { if repoType != "agent" && repoType != "skill" { return fmt.Errorf("--type must be 'agent' or 'skill' when set (got %q)", repoType) } - params.RepoType = langsmith.F(langsmith.RepoListParamsRepoType(repoType)) + params.SingleRepoType = langsmith.F(langsmith.RepoListParamsRepoType(repoType)) + } + if source != "" { + if source != "internal" && source != "external" { + return fmt.Errorf("--source must be 'internal' or 'external' when set (got %q)", source) + } + params.Source = langsmith.F(langsmith.RepoListParamsSource(source)) } if query != "" { params.Query = langsmith.F(query) @@ -88,6 +95,7 @@ func newHubListCmd() *cobra.Command { } cmd.Flags().StringVar(&repoType, "type", "", "Filter by repo type: agent or skill") + cmd.Flags().StringVar(&source, "source", "", "Filter by source: internal or external") cmd.Flags().StringVar(&query, "query", "", "Filter by name substring") cmd.Flags().BoolVar(&publicOnly, "public", false, "Filter by public/private") cmd.Flags().IntVarP(&limit, "limit", "n", 100, "Maximum number of repos to return") diff --git a/internal/cmd/hub_list_test.go b/internal/cmd/hub_list_test.go index 3d740b6..92cbdfe 100644 --- a/internal/cmd/hub_list_test.go +++ b/internal/cmd/hub_list_test.go @@ -19,7 +19,7 @@ func TestHubList_QueryParams(t *testing.T) { captureStdout(t, func() { cmd := newHubCmd() - cmd.SetArgs([]string{"list", "--type", "skill", "--query", "foo", "--public", "--limit", "10", "--offset", "5"}) + cmd.SetArgs([]string{"list", "--type", "skill", "--source", "external", "--query", "foo", "--public", "--limit", "10", "--offset", "5"}) if err := cmd.Execute(); err != nil { t.Fatalf("execute: %v", err) } @@ -27,6 +27,7 @@ func TestHubList_QueryParams(t *testing.T) { checks := map[string]string{ "repo_type": "skill", + "source": "external", "query": "foo", "match_prefix": "true", "is_public": "true", @@ -53,6 +54,18 @@ func TestHubList_RejectsBadType(t *testing.T) { } } +func TestHubList_RejectsBadSource(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {}) + defer setupTestEnv(t, srv.URL)() + + cmd := newHubCmd() + cmd.SetArgs([]string{"list", "--source", "unknown"}) + err := cmd.Execute() + if err == nil || !strings.Contains(err.Error(), "internal") { + t.Errorf("expected error mentioning valid sources; got %v", err) + } +} + func TestHubList_RendersTable(t *testing.T) { srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/internal/cmd/hub_repo_sdk.go b/internal/cmd/hub_repo_sdk.go index c074c5b..f3b52fd 100644 --- a/internal/cmd/hub_repo_sdk.go +++ b/internal/cmd/hub_repo_sdk.go @@ -48,6 +48,7 @@ func sdkRepoToHubRepo(repo langsmith.RepoWithLookups) hubRepo { Owner: strPtrOrNil(repo.Owner), RepoHandle: repo.RepoHandle, RepoType: string(repo.RepoType), + Source: strPtrOrNil(string(repo.Source)), Description: strPtrOrNil(repo.Description), Readme: strPtrOrNil(repo.Readme), IsPublic: repo.IsPublic, diff --git a/internal/cmd/message.go b/internal/cmd/message.go index 11413ba..190e27d 100644 --- a/internal/cmd/message.go +++ b/internal/cmd/message.go @@ -2,8 +2,11 @@ package cmd import ( "context" + "encoding/json" "fmt" "os" + "regexp" + "strings" "time" langsmith "github.com/langchain-ai/langsmith-go" @@ -13,7 +16,28 @@ import ( "github.com/spf13/cobra" ) -// trajectoryStep is a compact single-step view of one message/tool-call in a trace. +// Char caps for the per-trace digest fields. Bounded so a single huge message +// or tool payload can't blow up the digest (and the context of any agent +// reading it). The full untruncated content is still available in `groups`. +const ( + digestTextMax = 600 + digestArgsMax = 200 + digestResultMax = 200 +) + +// errorishPattern is a heuristic flag for a tool result whose content looks +// like an *error response* (not merely text that mentions the word "error" — +// technical docs and search results say "error" constantly). It requires +// error-shaped context: an error/exception token with `:`/`=`, a quoted +// `"error":` key, an HTTP/status 4xx-5xx, a 4xx-5xx code next to a failure +// word, a traceback, or a common shell/tool failure string. Advisory, not +// authoritative — a reader still confirms by looking at the result. +var errorishPattern = regexp.MustCompile(`(?i)|traceback|command not found|permission denied|no such file|\b(error|exception)\b\s*[:=]|"error"\s*[:=]|\bhttp[ /]?[45]\d\d\b|\b[45]\d\d\b\s+\w*\s*(error|not found|forbidden|unauthorized|internal server)`) + +// trajectoryStep is a compact single-step view of one message/tool-call. It is +// deliberately THIN: the trajectory is scanned across *all* traces in a batch +// (triage), so per-trace content lives in the separate `digest` field, read +// per-trace — not crammed into the bulk-scanned trajectory. type trajectoryStep struct { Role string `json:"role"` ToolName string `json:"tool_name,omitempty"` @@ -24,10 +48,46 @@ type trajectoryStep struct { // traceTrajectory is the compact trajectory for a single trace. type traceTrajectory struct { - TraceID string `json:"trace_id"` - InputMessage string `json:"input_message,omitempty"` - OutputMessage string `json:"output_message,omitempty"` - Steps []trajectoryStep `json:"steps"` + TraceID string `json:"trace_id"` + Steps []trajectoryStep `json:"steps"` +} + +// digestToolArg is one tool call's name and its (bounded) JSON args. +type digestToolArg struct { + Name string `json:"name"` + Args string `json:"args"` +} + +// digestGroup is one normalized group in a trace's digest — enough to verdict +// the group without re-parsing the heterogeneously-shaped raw `groups`. +type digestGroup struct { + I int `json:"i"` + Kind string `json:"kind"` + Role string `json:"role,omitempty"` + Chars int `json:"chars"` + Text string `json:"text,omitempty"` // coerced content, ≤digestTextMax + Tools []string `json:"tools,omitempty"` // tool-call names in this group + ToolArgs []digestToolArg `json:"tool_args,omitempty"` // per call: name + ≤digestArgsMax JSON args + ToolResult []string `json:"tool_result,omitempty"` // per call: coerced result, ≤digestResultMax + Errorish bool `json:"errorish,omitempty"` // any tool result looks like an error +} + +// traceDigest is the detailed per-trace view a screener reads for the ONE trace +// it is judging. It is emitted as its own field (NOT folded into the trajectory) +// so this bulk detail is read per-trace, never scanned across the whole batch. +type traceDigest struct { + FirstHuman string `json:"first_human,omitempty"` + FinalAI string `json:"final_ai,omitempty"` + NGroups int `json:"n_groups"` + GroupIndex []digestGroup `json:"group_index"` +} + +// attachTrajectory attaches the thin trajectory and the per-trace digest to a +// trace map for JSON output. The trajectory is scanned across all traces; the +// digest is read per-trace. +func attachTrajectory(trace map[string]any, traj traceTrajectory, digest traceDigest) { + trace["trajectory"] = traj.Steps + trace["digest"] = digest } func newTraceMessagesCmd() *cobra.Command { @@ -73,12 +133,7 @@ Examples: c := MustGetClient() ctx := context.Background() - projectName := ResolveProject(ff.Project) - if projectName == "" { - ExitError("--project is required for trace messages (or set LANGSMITH_PROJECT)") - } - - sessionID, err := c.ResolveSessionID(ctx, projectName) + sessionID, err := resolveSessionID(ctx, c, ff.Project, ff.ProjectID, "trace messages") if err != nil { ExitErrorf("%v", err) } @@ -141,7 +196,7 @@ Examples: for _, t := range traces { trace, _ := t.(map[string]any) traj := buildTraceTrajectory(trace) - trace["trajectory"] = traj.Steps + attachTrajectory(trace, traj, buildTraceDigest(trace)) } nextCursor, _ := result["next_cursor"].(string) @@ -202,7 +257,7 @@ Examples: for _, t := range allTraces { trace, _ := t.(map[string]any) traj := buildTraceTrajectory(trace) - trace["trajectory"] = traj.Steps + attachTrajectory(trace, traj, buildTraceDigest(trace)) } combined := map[string]any{ @@ -220,7 +275,9 @@ Examples: } addCommonFilterFlags(cmd, &ff, true) + cmd.Flags().StringVar(&ff.ProjectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") + cmd.MarkFlagsMutuallyExclusive("project", "project-id") return cmd } @@ -361,11 +418,12 @@ func printTraceMessages(result map[string]any) { } } -// buildTraceTrajectory converts a raw trace map into a compact trajectory. +// buildTraceTrajectory converts a raw trace map into a compact, THIN trajectory +// (role/tool/tokens/latency/chars per step). It carries no message content — +// the trajectory is scanned across all traces for triage, so detail lives in +// buildTraceDigest and is read per-trace. func buildTraceTrajectory(trace map[string]any) traceTrajectory { traceID, _ := trace["trace_id"].(string) - inputsPreview, _ := trace["root_inputs_preview"].(string) - outputsPreview, _ := trace["root_outputs_preview"].(string) groups, _ := trace["groups"].([]any) var steps []trajectoryStep @@ -373,7 +431,6 @@ func buildTraceTrajectory(trace map[string]any) traceTrajectory { group, _ := g.(map[string]any) gType, _ := group["type"].(string) meta, _ := group["metadata"].(map[string]any) - tokens := trajTokens(meta) latency := trajLatency(meta) @@ -409,12 +466,85 @@ func buildTraceTrajectory(trace map[string]any) traceTrajectory { } } } + return traceTrajectory{TraceID: traceID, Steps: steps} +} + +// buildTraceDigest builds the detailed per-trace digest: the full first-human / +// final-AI messages plus a normalized per-group index (content coerced to +// strings, fields length-bounded). Read per-trace by a screener — kept OUT of +// the bulk-scanned trajectory on purpose. Mirrors the fetch-time digest the +// issues-agent screeners verdict from. +func buildTraceDigest(trace map[string]any) traceDigest { + inputsPreview, _ := trace["root_inputs_preview"].(string) + outputsPreview, _ := trace["root_outputs_preview"].(string) + groups, _ := trace["groups"].([]any) + + var firstHuman, finalAI string + index := make([]digestGroup, 0, len(groups)) + for i, g := range groups { + group, _ := g.(map[string]any) + gType, _ := group["type"].(string) + dg := digestGroup{I: i, Kind: gType} + if dg.Kind == "" { + dg.Kind = "message" + } + + var text string + switch gType { + case "message": + msg, _ := group["message"].(map[string]any) + dg.Role, _ = msg["role"].(string) + text = msgText(msg) + if dg.Role == "human" && firstHuman == "" { + firstHuman = text + } + if dg.Role == "ai" && text != "" { + finalAI = text + } + case "tool_interaction": + aiMsg, _ := group["aiMessage"].(map[string]any) + text = msgText(aiMsg) + if text != "" { + finalAI = text + } + toolCalls, _ := group["toolCalls"].([]any) + for _, tc := range toolCalls { + call, _ := tc.(map[string]any) + name, _ := call["name"].(string) + dg.Tools = append(dg.Tools, name) + dg.ToolArgs = append(dg.ToolArgs, digestToolArg{ + Name: name, + Args: truncateHard(marshalArgs(call["args"]), digestArgsMax), + }) + result, _ := call["result"].(map[string]any) + resultText := "" + if result != nil { + resultText = coerceContent(result["content"]) + } + dg.ToolResult = append(dg.ToolResult, truncateHard(resultText, digestResultMax)) + if resultText != "" && errorishPattern.MatchString(resultText) { + dg.Errorish = true + } + } + } + dg.Chars = len(text) + dg.Text = truncateHard(text, digestTextMax) + index = append(index, dg) + } - return traceTrajectory{ - TraceID: traceID, - InputMessage: inputsPreview, - OutputMessage: outputsPreview, - Steps: steps, + // Prefer the full message from groups; fall back to the (~150-char) preview + // only when groups don't carry it. + if firstHuman == "" { + firstHuman = inputsPreview + } + if finalAI == "" { + finalAI = outputsPreview + } + return traceDigest{ + FirstHuman: firstHuman, + FinalAI: finalAI, + NGroups: len(groups), + GroupIndex: index, } } @@ -464,6 +594,85 @@ func msgChars(msg map[string]any) int { return total } +// coerceContent flattens a message/tool `content` value — which may be a +// string, a list of content blocks, an object, or null — into a plain string. +// Content shape is heterogeneous across integrations; normalizing here lets a +// reader verdict off the trajectory without re-parsing each shape. +func coerceContent(v any) string { + switch c := v.(type) { + case nil: + return "" + case string: + return c + case []any: + var parts []string + for _, block := range c { + switch b := block.(type) { + case string: + parts = append(parts, b) + case map[string]any: + if t, ok := b["text"].(string); ok { + parts = append(parts, t) + } else if t, ok := b["content"].(string); ok { + parts = append(parts, t) + } else { + parts = append(parts, unrenderedContent(b)) + } + default: + parts = append(parts, fmt.Sprintf("%v", b)) + } + } + return strings.Join(parts, " ") + case map[string]any: + if t, ok := c["text"].(string); ok { + return t + } + if t, ok := c["content"].(string); ok { + return t + } + return unrenderedContent(c) + default: + return fmt.Sprintf("%v", c) + } +} + +// unrenderedContent represents a non-null content block that carries no +// plain-text `text`/`content` field (image, tool_use, or other structured +// blocks) as a non-empty string. Collapsing these to "" makes structured +// content indistinguishable from a genuinely empty turn, which misleads a +// downstream reader (e.g. the engine screener) into treating the content as +// absent. Prefer compact JSON (the downstream char caps truncate it); fall back +// to a typed sentinel if the block can't be marshaled. +func unrenderedContent(m map[string]any) string { + if b, err := json.Marshal(m); err == nil { + return string(b) + } + if t, ok := m["type"].(string); ok && t != "" { + return "[" + t + " content]" + } + return "[unrenderable content]" +} + +// msgText returns a message's content coerced to a plain string. +func msgText(msg map[string]any) string { + if msg == nil { + return "" + } + return coerceContent(msg["content"]) +} + +// marshalArgs renders tool-call args as compact JSON, or "" if absent/unmarshalable. +func marshalArgs(args any) string { + if args == nil { + return "" + } + b, err := json.Marshal(args) + if err != nil { + return "" + } + return string(b) +} + // printMessage prints a single message in a compact format. func printMessage(msg map[string]any) { if msg == nil { diff --git a/internal/cmd/message_trajectory_test.go b/internal/cmd/message_trajectory_test.go new file mode 100644 index 0000000..a791266 --- /dev/null +++ b/internal/cmd/message_trajectory_test.go @@ -0,0 +1,177 @@ +package cmd + +import ( + "strings" + "testing" +) + +// The trajectory must stay THIN — no message content / tool args / results. +// It is scanned across all traces, so per-trace detail belongs in the digest. +func TestBuildTraceTrajectory_IsThin(t *testing.T) { + trace := map[string]any{ + "trace_id": "t1", + "groups": []any{ + map[string]any{"type": "message", "message": map[string]any{"role": "human", "content": "x" + strings.Repeat("y", 5000)}}, + map[string]any{ + "type": "tool_interaction", + "aiMessage": map[string]any{"role": "ai", "content": "thinking"}, + "toolCalls": []any{map[string]any{"name": "search", "args": map[string]any{"q": "z"}, "result": map[string]any{"content": "404 not found"}}}, + }, + }, + } + traj := buildTraceTrajectory(trace) + if len(traj.Steps) != 3 { // human, ai, tool + t.Fatalf("got %d steps, want 3", len(traj.Steps)) + } + // Steps carry only role/tool_name/chars — no content fields exist on the struct. + if traj.Steps[2].Role != "tool" || traj.Steps[2].ToolName != "search" { + t.Errorf("tool step = %+v", traj.Steps[2]) + } + if traj.Steps[0].Chars != 5001 { + t.Errorf("human step chars = %d, want 5001 (count only, no content)", traj.Steps[0].Chars) + } +} + +// buildTraceDigest carries the detail: full first/final messages + a normalized +// per-group index with coerced content, tool args/results, and errorish flags. +func TestBuildTraceDigest_NormalizedGroups(t *testing.T) { + trace := map[string]any{ + "trace_id": "t1", + "root_inputs_preview": "human: how do I use MCP?", + "root_outputs_preview": "tool: clipped preview that isn't the answer", + "groups": []any{ + map[string]any{"type": "message", "message": map[string]any{"role": "human", "content": "how do I use MCP?"}}, + map[string]any{ + "type": "tool_interaction", + "aiMessage": map[string]any{"role": "ai", "content": []any{map[string]any{"type": "text", "text": "searching the docs"}}}, + "toolCalls": []any{map[string]any{"name": "search_docs", "args": map[string]any{"q": "mcp"}, "result": map[string]any{"content": "404 not found"}}}, + }, + map[string]any{"type": "message", "message": map[string]any{"role": "ai", "content": "Use create_agent with the mcp arg."}}, + }, + } + d := buildTraceDigest(trace) + + if d.FirstHuman != "how do I use MCP?" { + t.Errorf("first_human = %q, want full human message from groups", d.FirstHuman) + } + if d.FinalAI != "Use create_agent with the mcp arg." { + t.Errorf("final_ai = %q, want final ai message from groups", d.FinalAI) + } + if d.NGroups != 3 || len(d.GroupIndex) != 3 { + t.Fatalf("n_groups=%d group_index=%d, want 3/3", d.NGroups, len(d.GroupIndex)) + } + // tool_interaction group: content coerced from list-of-blocks, tool fields present. + tg := d.GroupIndex[1] + if tg.Kind != "tool_interaction" || tg.Text != "searching the docs" { + t.Errorf("group[1] kind/text = %q/%q", tg.Kind, tg.Text) + } + if len(tg.Tools) != 1 || tg.Tools[0] != "search_docs" { + t.Errorf("group[1] tools = %v", tg.Tools) + } + if len(tg.ToolArgs) != 1 || tg.ToolArgs[0].Name != "search_docs" || !strings.Contains(tg.ToolArgs[0].Args, `"q":"mcp"`) { + t.Errorf("group[1] tool_args = %+v", tg.ToolArgs) + } + if len(tg.ToolResult) != 1 || tg.ToolResult[0] != "404 not found" { + t.Errorf("group[1] tool_result = %v", tg.ToolResult) + } +} + +// Null/missing content coerces to empty (no panic); first/final fall back to previews. +func TestBuildTraceDigest_NullAndFallback(t *testing.T) { + trace := map[string]any{ + "trace_id": "t2", + "root_inputs_preview": "human: hi", + "root_outputs_preview": "the answer", + "groups": []any{ + map[string]any{ + "type": "tool_interaction", + "aiMessage": map[string]any{"role": "ai", "content": nil}, + "toolCalls": []any{map[string]any{"name": "read_url", "result": map[string]any{"content": nil}}}, + }, + }, + } + d := buildTraceDigest(trace) + if d.FirstHuman != "human: hi" || d.FinalAI != "the answer" { + t.Errorf("fallbacks: first=%q final=%q", d.FirstHuman, d.FinalAI) + } + g := d.GroupIndex[0] + if g.Text != "" || g.ToolResult[0] != "" || g.Errorish { + t.Errorf("null content should be empty/non-errorish, got %+v", g) + } +} + +// Non-null structured content (image / tool_use blocks, or a map with no +// text/content field) must NOT collapse to "" — that is indistinguishable from +// a genuinely empty turn and misleads a downstream reader into thinking the +// content is absent. Only nil stays empty. +func TestCoerceContent_UnrenderedStructuredContent(t *testing.T) { + if got := coerceContent(nil); got != "" { + t.Errorf("nil content = %q, want empty", got) + } + // A structured object with no text/content field (e.g. an image block). + img := map[string]any{"type": "image", "source": map[string]any{"url": "https://x/y.png"}} + if got := coerceContent(img); got == "" || !strings.Contains(got, "image") { + t.Errorf("image block coerced to %q, want non-empty mentioning its type", got) + } + // A list with a renderable text block and an unrenderable tool_use block: + // the tool_use must still contribute non-empty content, not be dropped. + blocks := []any{ + map[string]any{"type": "text", "text": "calling a tool"}, + map[string]any{"type": "tool_use", "name": "search", "input": map[string]any{"q": "mcp"}}, + } + if got := coerceContent(blocks); !strings.Contains(got, "calling a tool") || !strings.Contains(got, "tool_use") { + t.Errorf("block list coerced to %q, want both the text and the tool_use block", got) + } +} + +// errorish must flag error-SHAPED results, not text that merely mentions "error". +func TestBuildTraceDigest_ErrorishPrecision(t *testing.T) { + cases := []struct { + content string + want bool + }{ + {"Error: connection refused", true}, + {"404 not found", true}, + {`{"error": "rate limited"}`, true}, + {"HTTP 503 service unavailable", true}, + {"Traceback (most recent call last):", true}, + {"Evaluators help you catch error cases in your app.", false}, + {"This guide covers error handling best practices.", false}, + {`{"level": "error", "matched": 4}`, false}, + } + for _, c := range cases { + trace := map[string]any{"groups": []any{map[string]any{ + "type": "tool_interaction", + "aiMessage": map[string]any{"role": "ai", "content": ""}, + "toolCalls": []any{map[string]any{"name": "t", "result": map[string]any{"content": c.content}}}, + }}} + got := buildTraceDigest(trace).GroupIndex[0].Errorish + if got != c.want { + t.Errorf("errorish(%q) = %v, want %v", c.content, got, c.want) + } + } +} + +// Long content is bounded to the digest caps. +func TestBuildTraceDigest_Truncation(t *testing.T) { + big := strings.Repeat("x", 5000) + trace := map[string]any{"groups": []any{ + map[string]any{"type": "message", "message": map[string]any{"role": "human", "content": big}}, + map[string]any{ + "type": "tool_interaction", + "aiMessage": map[string]any{"role": "ai", "content": "ok"}, + "toolCalls": []any{map[string]any{"name": "f", "args": map[string]any{"k": big}, "result": map[string]any{"content": big}}}, + }, + }} + d := buildTraceDigest(trace) + if got := len(d.GroupIndex[0].Text); got != digestTextMax { + t.Errorf("group text len = %d, want %d", got, digestTextMax) + } + tg := d.GroupIndex[1] + if got := len(tg.ToolResult[0]); got != digestResultMax { + t.Errorf("tool_result len = %d, want %d", got, digestResultMax) + } + if got := len(tg.ToolArgs[0].Args); got != digestArgsMax { + t.Errorf("tool_args len = %d, want %d", got, digestArgsMax) + } +} diff --git a/internal/cmd/run.go b/internal/cmd/run.go index defa44a..a3aea47 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -60,16 +60,23 @@ func newRunListCmd() *cobra.Command { c := MustGetClient() ctx := context.Background() - projectName := ResolveProject(ff.Project) - if projectName == "" { - ExitError("--project is required for run list (or set LANGSMITH_PROJECT)") + sessionID, err := resolveSessionID(ctx, c, ff.Project, "", "run list") + if err != nil { + ExitErrorf("%v", err) } - params := BuildRunQueryParams(&ff, false, ff.Limit) - if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { - params.Select = langsmith.F(sel) + var runs []langsmith.RunSchema + if ff.Version == "v2" { + body := buildRunQueryV2Params(&ff, false, ff.Limit) + body.Selects = langsmith.F(buildRunSelectV2(includeIO, includeFeedback)) + runs, err = queryRunsV2(ctx, c, body, sessionID, ff.Limit, ff.MinTokens) + } else { + params := BuildRunQueryParams(&ff, false, ff.Limit) + if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { + params.Select = langsmith.F(sel) + } + runs, err = queryRuns(ctx, c, params, sessionID, ff.Limit, ff.MinTokens) } - runs, err := queryRuns(ctx, c, params, projectName, ff.Limit, ff.MinTokens) if err != nil { ExitErrorf("%v", err) } @@ -87,6 +94,7 @@ func newRunListCmd() *cobra.Command { } addCommonFilterFlags(cmd, &ff, true) + addVersionFlag(cmd, &ff) cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") @@ -101,6 +109,7 @@ func newRunGetCmd() *cobra.Command { project string since string lastNMinutes int + version string includeMetadata bool includeIO bool includeFeedback bool @@ -123,21 +132,31 @@ func newRunGetCmd() *cobra.Command { c := MustGetClient() ctx := context.Background() - projectName := ResolveProject(project) - if projectName == "" { - ExitError("--project is required for run get (or set LANGSMITH_PROJECT)") + sessionID, err := resolveSessionID(ctx, c, project, "", "run get") + if err != nil { + ExitErrorf("%v", err) } - params := langsmith.RunQueryParams{ - ID: langsmith.F([]string{runID}), - Limit: langsmith.F(int64(1)), - StartTime: langsmith.F(resolveStartTime(since, lastNMinutes)), - } - if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { - params.Select = langsmith.F(sel) + var runs []langsmith.RunSchema + if version == "v2" { + params := langsmith.RunQueryV2Params{ + IDs: langsmith.F([]string{runID}), + MinStartTime: langsmith.F(resolveStartTime(since, lastNMinutes)), + PageSize: langsmith.F(int64(1)), + Selects: langsmith.F(buildRunSelectV2(includeIO, includeFeedback)), + } + runs, err = queryRunsV2(ctx, c, params, sessionID, 1, 0) + } else { + params := langsmith.RunQueryParams{ + ID: langsmith.F([]string{runID}), + Limit: langsmith.F(int64(1)), + StartTime: langsmith.F(resolveStartTime(since, lastNMinutes)), + } + if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { + params.Select = langsmith.F(sel) + } + runs, err = queryRuns(ctx, c, params, sessionID, 1, 0) } - - runs, err := queryRuns(ctx, c, params, projectName, 1, 0) if err != nil { ExitErrorf("fetching run: %v", err) } @@ -159,6 +178,7 @@ func newRunGetCmd() *cobra.Command { cmd.Flags().StringVar(&project, "project", "", "Project name [env: LANGSMITH_PROJECT]") cmd.Flags().StringVar(&since, "since", "", "Only include runs after this timestamp, e.g. 2024-01-15T00:00:00Z (overrides 7-day default)") cmd.Flags().IntVar(&lastNMinutes, "last-n-minutes", 0, "Only include runs from the last N minutes, e.g. 60 (overrides 7-day default)") + cmd.Flags().StringVar(&version, "version", "", `Query API version: "" (v1, default) or "v2" (SmithDB)`) cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") @@ -196,16 +216,23 @@ func newRunExportCmd() *cobra.Command { c := MustGetClient() ctx := context.Background() - projectName := ResolveProject(ff.Project) - if projectName == "" { - ExitError("--project is required for run export (or set LANGSMITH_PROJECT)") + sessionID, err := resolveSessionID(ctx, c, ff.Project, "", "run export") + if err != nil { + ExitErrorf("%v", err) } - params := BuildRunQueryParams(&ff, false, ff.Limit) - if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { - params.Select = langsmith.F(sel) + var runs []langsmith.RunSchema + if ff.Version == "v2" { + body := buildRunQueryV2Params(&ff, false, ff.Limit) + body.Selects = langsmith.F(buildRunSelectV2(includeIO, includeFeedback)) + runs, err = queryRunsV2(ctx, c, body, sessionID, ff.Limit, ff.MinTokens) + } else { + params := BuildRunQueryParams(&ff, false, ff.Limit) + if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { + params.Select = langsmith.F(sel) + } + runs, err = queryRuns(ctx, c, params, sessionID, ff.Limit, ff.MinTokens) } - runs, err := queryRuns(ctx, c, params, projectName, ff.Limit, ff.MinTokens) if err != nil { ExitErrorf("%v", err) } @@ -216,6 +243,7 @@ func newRunExportCmd() *cobra.Command { } addCommonFilterFlags(cmd, &ff, true) + addVersionFlag(cmd, &ff) cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") diff --git a/internal/cmd/thread.go b/internal/cmd/thread.go index cde5413..57d4eef 100644 --- a/internal/cmd/thread.go +++ b/internal/cmd/thread.go @@ -105,10 +105,10 @@ func newThreadListCmd() *cobra.Command { threadsMap[tid] = append(threadsMap[tid], m) } } - if resp.Cursors == nil || resp.Cursors["next"] == "" { + if resp.Cursors.Next == "" { break } - cursor = resp.Cursors["next"] + cursor = resp.Cursors.Next } // Build thread summaries diff --git a/internal/cmd/trace.go b/internal/cmd/trace.go index c10e4a9..bc001d2 100644 --- a/internal/cmd/trace.go +++ b/internal/cmd/trace.go @@ -68,26 +68,22 @@ func newTraceListCmd() *cobra.Command { c := MustGetClient() ctx := context.Background() - projectName := ResolveProject(ff.Project) - if projectName == "" { - ExitError("--project is required for trace list (or set LANGSMITH_PROJECT)") + sessionID, err := resolveSessionID(ctx, c, ff.Project, ff.ProjectID, "trace list") + if err != nil { + ExitErrorf("%v", err) } params := BuildRunQueryParams(&ff, true, ff.Limit) if sel := buildRunSelect(includeIO, includeFeedback); sel != nil { params.Select = langsmith.F(sel) } - runs, err := queryRuns(ctx, c, params, projectName, ff.Limit, ff.MinTokens) + runs, err := queryRuns(ctx, c, params, sessionID, ff.Limit, ff.MinTokens) if err != nil { ExitErrorf("%v", err) } var flaggedByTrace map[string]string if includeFlagged { - sessionID, err := c.ResolveSessionID(ctx, projectName) - if err != nil { - ExitErrorf("%v", err) - } flagged := FetchFlaggedTraces(ctx, c, sessionID, ff.LastNMinutes) flaggedByTrace = make(map[string]string, len(flagged)) for _, ft := range flagged { @@ -103,7 +99,7 @@ func newTraceListCmd() *cobra.Command { allRuns, err := queryRuns(ctx, c, langsmith.RunQueryParams{ Trace: langsmith.F(run.TraceID), Order: langsmith.F(langsmith.RunQueryParamsOrderAsc), - }, projectName, 1000, 0) + }, sessionID, 1000, 0) if err != nil { ExitErrorf("%v", err) } @@ -124,7 +120,7 @@ func newTraceListCmd() *cobra.Command { var result []map[string]any for _, run := range runs { childParams.Trace = langsmith.F(run.TraceID) - allRuns, err := queryRuns(ctx, c, childParams, projectName, 1000, 0) + allRuns, err := queryRuns(ctx, c, childParams, sessionID, 1000, 0) if err != nil { ExitErrorf("%v", err) } @@ -147,6 +143,7 @@ func newTraceListCmd() *cobra.Command { } addCommonFilterFlags(cmd, &ff, false) + cmd.Flags().StringVar(&ff.ProjectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") @@ -154,6 +151,7 @@ func newTraceListCmd() *cobra.Command { cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().BoolVar(&showHierarchy, "show-hierarchy", false, "Fetch the full run tree for each trace") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") + cmd.MarkFlagsMutuallyExclusive("project", "project-id") return cmd } @@ -161,6 +159,7 @@ func newTraceListCmd() *cobra.Command { func newTraceGetCmd() *cobra.Command { var ( project string + projectID string since string lastNMinutes int includeMetadata bool @@ -185,9 +184,9 @@ func newTraceGetCmd() *cobra.Command { c := MustGetClient() ctx := context.Background() - projectName := ResolveProject(project) - if projectName == "" { - ExitError("--project is required for trace get (or set LANGSMITH_PROJECT)") + sessionID, err := resolveSessionID(ctx, c, project, projectID, "trace get") + if err != nil { + ExitErrorf("%v", err) } params := langsmith.RunQueryParams{ @@ -199,7 +198,7 @@ func newTraceGetCmd() *cobra.Command { params.Select = langsmith.F(sel) } - runs, err := queryRuns(ctx, c, params, projectName, 1000, 0) + runs, err := queryRuns(ctx, c, params, sessionID, 1000, 0) if err != nil { ExitErrorf("%v", err) } @@ -220,6 +219,7 @@ func newTraceGetCmd() *cobra.Command { } cmd.Flags().StringVar(&project, "project", "", "Project name [env: LANGSMITH_PROJECT]") + cmd.Flags().StringVar(&projectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") cmd.Flags().StringVar(&since, "since", "", "Only include runs after this timestamp, e.g. 2024-01-15T00:00:00Z (overrides 7-day default)") cmd.Flags().IntVar(&lastNMinutes, "last-n-minutes", 0, "Only include runs from the last N minutes, e.g. 60 (overrides 7-day default)") cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") @@ -227,6 +227,7 @@ func newTraceGetCmd() *cobra.Command { cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") + cmd.MarkFlagsMutuallyExclusive("project", "project-id") return cmd } @@ -264,9 +265,9 @@ func newTraceExportCmd() *cobra.Command { c := MustGetClient() ctx := context.Background() - projectName := ResolveProject(ff.Project) - if projectName == "" { - ExitError("--project is required for trace export (or set LANGSMITH_PROJECT)") + sessionID, err := resolveSessionID(ctx, c, ff.Project, ff.ProjectID, "trace export") + if err != nil { + ExitErrorf("%v", err) } params := BuildRunQueryParams(&ff, true, ff.Limit) @@ -274,7 +275,7 @@ func newTraceExportCmd() *cobra.Command { if sel != nil { params.Select = langsmith.F(sel) } - rootRuns, err := queryRuns(ctx, c, params, projectName, ff.Limit, ff.MinTokens) + rootRuns, err := queryRuns(ctx, c, params, sessionID, ff.Limit, ff.MinTokens) if err != nil { ExitErrorf("%v", err) } @@ -290,7 +291,7 @@ func newTraceExportCmd() *cobra.Command { if sel != nil { childParams.Select = langsmith.F(sel) } - allRuns, err := queryRuns(ctx, c, childParams, projectName, 1000, 0) + allRuns, err := queryRuns(ctx, c, childParams, sessionID, 1000, 0) if err != nil { ExitErrorf("%v", err) } @@ -330,12 +331,14 @@ func newTraceExportCmd() *cobra.Command { } addCommonFilterFlags(cmd, &ff, false) + cmd.Flags().StringVar(&ff.ProjectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVar(&filenamePattern, "filename-pattern", "{trace_id}.jsonl", "Filename pattern. Supports {trace_id} and {name} placeholders.") + cmd.MarkFlagsMutuallyExclusive("project", "project-id") return cmd } diff --git a/internal/cmd/trace_stats.go b/internal/cmd/trace_stats.go index 6e4701d..cc4ef93 100644 --- a/internal/cmd/trace_stats.go +++ b/internal/cmd/trace_stats.go @@ -31,6 +31,7 @@ type runStats struct { func newTraceStatsCmd() *cobra.Command { var ( project string + projectID string since string before string lastNMin int @@ -63,14 +64,9 @@ Examples: c := MustGetClient() ctx := context.Background() - projectName := ResolveProject(project) - if projectName == "" { - return fmt.Errorf("--project is required (or set LANGSMITH_PROJECT)") - } - - sessionID, err := c.ResolveSessionID(ctx, projectName) + sessionID, err := resolveSessionID(ctx, c, project, projectID, "trace stats") if err != nil { - return fmt.Errorf("resolving project %q: %w", projectName, err) + return err } primary, err := fetchRunStats(ctx, c, sessionID, since, before, lastNMin, filter) @@ -103,6 +99,7 @@ Examples: } cmd.Flags().StringVar(&project, "project", "", "Project name [env: LANGSMITH_PROJECT]") + cmd.Flags().StringVar(&projectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") cmd.Flags().StringVar(&since, "since", "", "Start of time window (RFC3339 or YYYY-MM-DD; default: 7 days ago)") cmd.Flags().StringVar(&before, "before", "", "End of time window (RFC3339 or YYYY-MM-DD; default: now)") cmd.Flags().IntVar(&lastNMin, "last-n-minutes", 0, "Shorthand: window = last N minutes (overrides --since)") @@ -111,6 +108,7 @@ Examples: cmd.Flags().IntVar(&cmpLastNMin, "compare-last-n-minutes", 0, "Shorthand: comparison window = N minutes before the primary window starts") cmd.Flags().StringVar(&filter, "filter", "", "LangSmith filter DSL (applied to both windows if comparing)") cmd.Flags().StringVar(&outputFile, "output", "", "Write JSON output to file instead of stdout") + cmd.MarkFlagsMutuallyExclusive("project", "project-id") return cmd } @@ -165,13 +163,13 @@ func fetchRunStats(ctx context.Context, c *client.Client, sessionID, since, befo } } -func toRunStats(runCount int64, latencyP50, latencyP99 float64, totalTokens, promptTokens, completionTokens int64, totalCost string, errorRate float64, feedbackStats map[string]interface{}) runStats { +func toRunStats(runCount int64, latencyP50, latencyP99 float64, totalTokens, promptTokens, completionTokens int64, totalCost float64, errorRate float64, feedbackStats map[string]interface{}) runStats { fs := make(map[string]any, len(feedbackStats)) for k, v := range feedbackStats { fs[k] = v } var cost any - if totalCost != "" { + if totalCost > 0 { cost = totalCost } return runStats{ diff --git a/internal/extract/extract.go b/internal/extract/extract.go index b92863d..5e63ed9 100644 --- a/internal/extract/extract.go +++ b/internal/extract/extract.go @@ -2,7 +2,6 @@ package extract import ( "fmt" - "strconv" "time" langsmith "github.com/langchain-ai/langsmith-go" @@ -48,16 +47,16 @@ func ExtractRun(run langsmith.RunSchema, includeMetadata, includeIO, includeFeed } var costs map[string]any - if run.PromptCost != "" || run.CompletionCost != "" || run.TotalCost != "" { + if run.PromptCost > 0 || run.CompletionCost > 0 || run.TotalCost > 0 { costs = map[string]any{} - if f, err := strconv.ParseFloat(run.PromptCost, 64); err == nil && f > 0 { - costs["prompt_cost"] = f + if run.PromptCost > 0 { + costs["prompt_cost"] = run.PromptCost } - if f, err := strconv.ParseFloat(run.CompletionCost, 64); err == nil && f > 0 { - costs["completion_cost"] = f + if run.CompletionCost > 0 { + costs["completion_cost"] = run.CompletionCost } - if f, err := strconv.ParseFloat(run.TotalCost, 64); err == nil && f > 0 { - costs["total_cost"] = f + if run.TotalCost > 0 { + costs["total_cost"] = run.TotalCost } } if len(costs) == 0 { diff --git a/internal/extract/extract_test.go b/internal/extract/extract_test.go index 5399bde..79c3974 100644 --- a/internal/extract/extract_test.go +++ b/internal/extract/extract_test.go @@ -63,9 +63,9 @@ func TestExtractRunWithMetadata(t *testing.T) { PromptTokens: 100, CompletionTokens: 50, TotalTokens: 150, - PromptCost: "0.001", - CompletionCost: "0.0005", - TotalCost: "0.0015", + PromptCost: 0.001, + CompletionCost: 0.0005, + TotalCost: 0.0015, Tags: []string{"production", "v2"}, Extra: map[string]any{"metadata": map[string]any{"model": "gpt-4"}}, } From 2a9c6e717696c4768764f9d2d97fb03055a76ee4 Mon Sep 17 00:00:00 2001 From: sineha-mani Date: Tue, 14 Jul 2026 09:45:00 -0700 Subject: [PATCH 4/9] fix(cli): updating the way we upload evaluators via cli (#30192) ## Description This PR changes only the Go CLI command: langsmith evaluator upload --replace - Updates `langsmith evaluator upload --replace` to patch the existing code evaluator rule instead of deleting it before creating the replacement. References [LSO-3365](https://linear.app/langchain/issue/LSO-3365/cli-evaluator-upload-replace-can-delete-the-existing-evaluator-before) ## Test Plan - [x] Unit tests Co-authored-by: Cursor GitOrigin-RevId: 6c4d578278fbc632a6658c83b578c1d051b477ae --- internal/cmd/evaluator.go | 60 ++++++++++--------- internal/cmd/evaluator_test.go | 105 +++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 29 deletions(-) diff --git a/internal/cmd/evaluator.go b/internal/cmd/evaluator.go index b936e72..3771e59 100644 --- a/internal/cmd/evaluator.go +++ b/internal/cmd/evaluator.go @@ -236,34 +236,6 @@ func newEvaluatorUploadCmd() *cobra.Command { projectID = sid } - // Check for existing evaluator - rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) - if err != nil { - ExitErrorf("checking existing evaluators: %v", err) - } - - existing := findEvaluator(*rules, name, datasetID, projectID) - if existing != nil { - if !replace { - output.OutputJSON(map[string]any{ - "error": fmt.Sprintf("Evaluator '%s' already exists (use --replace to overwrite)", name), - "id": existing.ID, - }, "") - return - } - if !yes { - fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) - var confirm string - _, _ = fmt.Scanln(&confirm) - if strings.ToLower(confirm) != "y" { - ExitError("aborted") - } - } - if err := c.RawDelete(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), nil); err != nil { - ExitErrorf("deleting existing evaluator: %v", err) - } - } - // Read and prepare function source source, err := os.ReadFile(evaluatorFile) if err != nil { @@ -310,8 +282,38 @@ func newEvaluatorUploadCmd() *cobra.Command { payload["session_id"] = projectID } + // Check for existing evaluator after the replacement payload is known + // to be valid, so --replace never removes a working evaluator first. + rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) + if err != nil { + ExitErrorf("checking existing evaluators: %v", err) + } + + existing := findEvaluator(*rules, name, datasetID, projectID) + if existing != nil { + if !replace { + output.OutputJSON(map[string]any{ + "error": fmt.Sprintf("Evaluator '%s' already exists (use --replace to overwrite)", name), + "id": existing.ID, + }, "") + return + } + if !yes { + fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) + var confirm string + _, _ = fmt.Scanln(&confirm) + if strings.ToLower(confirm) != "y" { + ExitError("aborted") + } + } + } + var result map[string]any - if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { + if existing != nil { + if err := c.RawPatch(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), payload, &result); err != nil { + ExitErrorf("replacing evaluator: %v", err) + } + } else if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { ExitErrorf("uploading evaluator: %v", err) } diff --git a/internal/cmd/evaluator_test.go b/internal/cmd/evaluator_test.go index 4936821..d7a05ef 100644 --- a/internal/cmd/evaluator_test.go +++ b/internal/cmd/evaluator_test.go @@ -3,6 +3,7 @@ package cmd import ( "encoding/json" "net/http" + "os" "strings" "testing" @@ -639,6 +640,110 @@ func TestEvaluatorListCmd_VerifiesAPIKeyHeader(t *testing.T) { } } +func TestEvaluatorUploadReplacePatchesExistingCodeEvaluator(t *testing.T) { + evaluatorFile := t.TempDir() + "/eval.py" + if err := os.WriteFile( + evaluatorFile, + []byte("def check_accuracy(run, example):\n return {\"score\": 1}\n"), + 0o600, + ); err != nil { + t.Fatal(err) + } + + var sawDelete bool + var patchBody map[string]any + ts := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/api/v1/sessions" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]map[string]any{ + {"id": "project-1", "name": "my-project"}, + }) + case r.URL.Path == "/api/v1/runs/rules" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]testRule{ + { + ID: "existing-rule", + DisplayName: "accuracy", + SamplingRate: 0.25, + IsEnabled: true, + SessionID: "project-1", + CodeEvaluators: []testCodeEval{ + {Code: "def perform_eval(run, example):\n return {\"score\": 0}", Language: "python"}, + }, + }, + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "PATCH": + if err := json.NewDecoder(r.Body).Decode(&patchBody); err != nil { + t.Fatalf("decoding patch body: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "existing-rule", + "display_name": "accuracy", + "session_id": "project-1", + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "DELETE": + sawDelete = true + http.Error(w, "delete should not be called", http.StatusInternalServerError) + default: + http.Error(w, "not found", http.StatusNotFound) + } + }) + + cleanup := setupTestEnv(t, ts.URL) + defer cleanup() + flagOutputFormat = "json" + + out := captureStdout(t, func() { + cmd := newEvaluatorUploadCmd() + _ = cmd.Flags().Set("name", "accuracy") + _ = cmd.Flags().Set("function", "check_accuracy") + _ = cmd.Flags().Set("project", "my-project") + _ = cmd.Flags().Set("sampling-rate", "0.5") + _ = cmd.Flags().Set("replace", "true") + _ = cmd.Flags().Set("yes", "true") + cmd.Run(cmd, []string{evaluatorFile}) + }) + + if sawDelete { + t.Fatal("upload --replace should patch the existing evaluator, not delete it") + } + if patchBody == nil { + t.Fatal("expected PATCH body") + } + if patchBody["display_name"] != "accuracy" { + t.Errorf("expected display_name=accuracy, got %v", patchBody["display_name"]) + } + if patchBody["session_id"] != "project-1" { + t.Errorf("expected session_id=project-1, got %v", patchBody["session_id"]) + } + if patchBody["sampling_rate"] != 0.5 { + t.Errorf("expected sampling_rate=0.5, got %v", patchBody["sampling_rate"]) + } + evaluators, ok := patchBody["code_evaluators"].([]any) + if !ok || len(evaluators) != 1 { + t.Fatalf("expected one code evaluator, got %#v", patchBody["code_evaluators"]) + } + codeEvaluator, ok := evaluators[0].(map[string]any) + if !ok { + t.Fatalf("expected code evaluator object, got %#v", evaluators[0]) + } + if codeEvaluator["language"] != "python" { + t.Errorf("expected language=python, got %v", codeEvaluator["language"]) + } + code, _ := codeEvaluator["code"].(string) + if !strings.Contains(code, "def perform_eval(") { + t.Errorf("expected uploaded code to be renamed to perform_eval, got:\n%s", code) + } + + var result map[string]any + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("failed to parse output JSON: %v\noutput: %s", err, out) + } + if result["id"] != "existing-rule" { + t.Errorf("expected output id=existing-rule, got %v", result["id"]) + } +} + // ==================== evaluator get ==================== func TestEvaluatorGetCmd_UseField(t *testing.T) { From 4714161dc51a40eb24fcdc3f429e9df3c16ce180 Mon Sep 17 00:00:00 2001 From: John Kennedy <65985482+jkennedyvz@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:00:44 -0700 Subject: [PATCH 5/9] chore: patch critical and high Dependabot alerts [INF-0000] (#30296) ## Summary - Patch the minimum fixed versions for repository-owned critical/high Dependabot findings across Go, Python, and frontend dependency manifests. - Align `smith-clio`'s LangChain/LangGraph constraints so the patched LangChain release can resolve. - Regenerate affected lockfiles and Go checksums. No matching open Linear tickets were found. ## Validation - `uv lock --check` passed for the root and `smith-clio` environments. - Frozen pnpm lockfile checks passed for `smith-frontend-perf` and `smith-polly`. - `git diff --check` passed. - Go tests could not run locally: the installed toolchain is Go 1.26.2 while cached artifacts were built with Go 1.25.6. `smith-go` dependency fetching was also blocked by Socket Firewall TLS verification for `golang.org/x/image@v0.41.0`. ## Test Plan - [x] Confirm Go tests and dependency checks pass in CI with the repository-supported toolchain. - [x] Confirm repository-owned critical/high alerts clear after merge. [no screenshot] GitOrigin-RevId: 3e790a2ba2b17429ec4a72bec08765a7882f4410 --- go.mod | 4 ++-- go.sum | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index b5e57eb..d595d6b 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.11.1 github.com/xlab/treeprint v1.2.0 - golang.org/x/net v0.54.0 + golang.org/x/net v0.55.0 golang.org/x/term v0.43.0 ) @@ -41,7 +41,7 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect - golang.org/x/sys v0.44.0 // indirect + golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect diff --git a/go.sum b/go.sum index 717fc12..ebe81f4 100644 --- a/go.sum +++ b/go.sum @@ -31,8 +31,6 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/langchain-ai/langsmith-go v0.16.0 h1:1kqrrNdIdYvoLdWPFBOj+iTtP8fayAJgwZJW+/RaQVA= -github.com/langchain-ai/langsmith-go v0.16.0/go.mod h1:ghwFdWr6zS7s3rdMMtEQDU1xWzineFzXa1SEG2XA3XY= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -88,10 +86,10 @@ go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpu go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= -golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= -golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= From 6eee49dec2e023610d66347a95050f73413a95a5 Mon Sep 17 00:00:00 2001 From: John Kennedy <65985482+jkennedyvz@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:17:09 -0700 Subject: [PATCH 6/9] chore: bump langsmith-go module versions (#30343) ## Description Bumps langsmith-go module requirements to v0.19.0 without importing the vendored generated SDK. This keeps the PR focused on Go module metadata only. ## Test Plan - [x] `go test ./...` in `cmd/langdev`, `sdks/cli`, and `sandbox-runner-controller`. Made by [Open SWE](https://openswe.vercel.app/agents/a9a13dbd-e06f-88b4-7dcc-42401154c3fd) --------- Co-authored-by: open-swe[bot] GitOrigin-RevId: f3178c2e6eedd22e1dd81d72bb767a759f50545a --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d595d6b..96ba171 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.25.0 require ( github.com/google/uuid v1.6.0 github.com/itchyny/gojq v0.12.15 - github.com/langchain-ai/langsmith-go v0.16.0 + github.com/langchain-ai/langsmith-go v0.19.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pelletier/go-toml/v2 v2.2.4 github.com/spf13/cobra v1.8.1 From 8a9d5b73322d41c21edb15e010fec3be69b4c236 Mon Sep 17 00:00:00 2001 From: sineha-mani Date: Tue, 14 Jul 2026 20:21:25 -0700 Subject: [PATCH 7/9] feat(cli): add LLM-as-judge evaluator creation via CLI (#28904) ## Description - Adds `langsmith evaluator create-llm` to create structured LLM-as-judge evaluator rules from the CLI (project or dataset target). - Supports inline JSON prompt + schema files, Prompt Hub refs (`--hub-ref`), and a serialized model config file (`--model-config`). - Adds `--trace-filter` to `evaluator upload` for parity with the API and `create-llm`. Screenshot 2026-06-30 at 2 15 41 PM References [LSE-2496](https://linear.app/langchain/issue/LSE-2496/feature-request-create-llm-as-judge-structured-evaluators-via-the-cli) ### Example Add model API keys to workspace **Secrets** first. Copy `structured.model` from an existing UI evaluator (or `GET /runs/rules`) into `model.json`. ```bash langsmith evaluator create-llm \ --name relevance \ --project my-app \ --prompt prompt.json \ --schema schema.json \ --model-config model.json \ --variable-mapping '{"input":"input.question","output":"output.answer"}' \ --trace-filter 'eq(is_root, true)' \ --sampling-rate 0.05 ``` ## Test Plan - [x] Added unit tests for prompt file parsing, model-config requirement, project/dataset targeting, and create-llm command registration - [x] Created an LLM evaluator against a test project locally and verified it appears in the UI --------- Co-authored-by: Cursor GitOrigin-RevId: 41b9fb8ed04acb65761c7bab9f1b60868950564c --- README.md | 12 ++ internal/cmd/evaluator.go | 132 +++++++++++++++++--- internal/cmd/evaluator_llm.go | 213 +++++++++++++++++++++++++++++++++ internal/cmd/evaluator_test.go | 176 ++++++++++++++++++++++++++- 4 files changed, 515 insertions(+), 18 deletions(-) create mode 100644 internal/cmd/evaluator_llm.go diff --git a/README.md b/README.md index 06255cb..9f7168a 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,18 @@ langsmith evaluator upload evals.py \ # Delete an evaluator langsmith evaluator delete accuracy --yes + +# Create an LLM-as-judge evaluator (--model-config is always required) +# model.json: copy the structured.model block from an existing evaluator or the UI. +langsmith evaluator create-llm \ + --name relevance --project my-app \ + --prompt prompt.json --schema schema.json --model-config model.json \ + --variable-mapping '{"input":"input.question","output":"output.answer"}' + +# Or reference an existing Prompt Hub commit (--hub-ref replaces --prompt and --schema) +langsmith evaluator create-llm \ + --name relevance --project my-app \ + --hub-ref my-org/relevance:latest --model-config model.json ``` ### `experiment` — Query experiment results diff --git a/internal/cmd/evaluator.go b/internal/cmd/evaluator.go index 3771e59..d48c1e5 100644 --- a/internal/cmd/evaluator.go +++ b/internal/cmd/evaluator.go @@ -20,9 +20,10 @@ func newEvaluatorCmd() *cobra.Command { Long: `Manage online and offline evaluator rules. Evaluators automatically score runs against a dataset (offline) or a project -(online). Two evaluator types are supported: +(online). Supported evaluator types: - upload Code evaluator from a Python or JavaScript/TypeScript file + upload Code evaluator from a Python or JavaScript/TypeScript file + create-llm LLM-as-judge evaluator (--model-config required; prompt via files or --hub-ref) Examples: langsmith evaluator list @@ -31,12 +32,14 @@ Examples: langsmith evaluator get accuracy --session-id langsmith evaluator upload eval.py --name accuracy --function check_accuracy --dataset my-eval-set langsmith evaluator upload eval.ts --name accuracy --function checkAccuracy --dataset my-eval-set + langsmith evaluator create-llm --name relevance --project my-app --prompt prompt.json --schema schema.json --model-config model.json langsmith evaluator delete accuracy --yes`, } cmd.AddCommand(newEvaluatorGetCmd()) cmd.AddCommand(newEvaluatorListCmd()) cmd.AddCommand(newEvaluatorUploadCmd()) + cmd.AddCommand(newEvaluatorCreateLLMCmd()) cmd.AddCommand(newEvaluatorDeleteCmd()) return cmd } @@ -196,6 +199,7 @@ func newEvaluatorUploadCmd() *cobra.Command { targetDataset string targetProject string samplingRate float64 + traceFilter string replace bool yes bool ) @@ -207,19 +211,15 @@ func newEvaluatorUploadCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { evaluatorFile := args[0] - if targetDataset == "" && targetProject == "" { - output.OutputJSON(map[string]any{ - "error": "Must specify --dataset or --project (global evaluators not supported)", - }, "") + if err := validateEvaluatorTargetFlags(targetDataset, targetProject); err != nil { + output.OutputJSON(map[string]any{"error": err.Error()}, "") return } c := MustGetClient() ctx := context.Background() - // Resolve targets var datasetID, projectID string - if targetDataset != "" { ds, err := resolveDataset(ctx, c, targetDataset) if err != nil { @@ -227,7 +227,6 @@ func newEvaluatorUploadCmd() *cobra.Command { } datasetID = ds.ID } - if targetProject != "" { sid, err := c.ResolveSessionID(ctx, targetProject) if err != nil { @@ -264,7 +263,6 @@ func newEvaluatorUploadCmd() *cobra.Command { sourceStr = renameJSFunction(sourceStr, funcName) } - // Build payload payload := map[string]any{ "display_name": name, "sampling_rate": samplingRate, @@ -274,16 +272,18 @@ func newEvaluatorUploadCmd() *cobra.Command { {"code": sourceStr, "language": language}, }, } - if datasetID != "" { payload["dataset_id"] = datasetID } if projectID != "" { payload["session_id"] = projectID } + if traceFilter != "" { + payload["trace_filter"] = traceFilter + } - // Check for existing evaluator after the replacement payload is known - // to be valid, so --replace never removes a working evaluator first. + // Prepare the new evaluator before touching any existing one. If this is + // a replacement, the old evaluator stays in place until the new version is ready. rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) if err != nil { ExitErrorf("checking existing evaluators: %v", err) @@ -317,16 +317,15 @@ func newEvaluatorUploadCmd() *cobra.Command { ExitErrorf("uploading evaluator: %v", err) } - target := "project" + targetLabel := "project" if datasetID != "" { - target = "dataset" + targetLabel = "dataset" } - output.OutputJSON(map[string]any{ "status": "uploaded", "id": result["id"], "name": name, - "target": target, + "target": targetLabel, }, "") }, } @@ -336,6 +335,7 @@ func newEvaluatorUploadCmd() *cobra.Command { cmd.Flags().StringVar(&targetDataset, "dataset", "", "Target dataset name (offline evaluator)") cmd.Flags().StringVar(&targetProject, "project", "", "Target project name (online evaluator)") cmd.Flags().Float64Var(&samplingRate, "sampling-rate", 1.0, "Fraction of runs to evaluate (0.0-1.0)") + cmd.Flags().StringVar(&traceFilter, "trace-filter", "", "Filter expression for which runs to evaluate") cmd.Flags().BoolVar(&replace, "replace", false, "Replace existing evaluator with same name") cmd.Flags().BoolVar(&yes, "yes", false, "Skip confirmation prompt when replacing") _ = cmd.MarkFlagRequired("name") @@ -344,6 +344,104 @@ func newEvaluatorUploadCmd() *cobra.Command { return cmd } +// Registers create-llm for LLM-as-judge evaluators. +func newEvaluatorCreateLLMCmd() *cobra.Command { + var ( + name string + targetDataset string + targetProject string + samplingRate float64 + traceFilter string + hubRef string + promptPath string + schemaPath string + modelConfigPath string + variableMapping string + replace bool + yes bool + ) + + cmd := &cobra.Command{ + Use: "create-llm", + Short: "Create an LLM-as-judge evaluator rule", + Long: `Create an LLM-as-judge run rule. --model-config is always required. + +Provide the judge prompt inline with --prompt and --schema, or point at Prompt Hub +with --hub-ref (which replaces --prompt and --schema). The model is configured separately. + +Examples: + langsmith evaluator create-llm --name relevance --project my-app \ + --prompt prompt.json --schema schema.json --model-config model.json + langsmith evaluator create-llm --name relevance --project my-app \ + --hub-ref my-org/relevance:latest --model-config model.json`, + Run: func(cmd *cobra.Command, args []string) { + c := MustGetClient() + ctx := context.Background() + + target, err := resolveLLMEvaluatorTarget(ctx, c, targetDataset, targetProject) + if err != nil { + ExitErrorf("%v", err) + } + mapping, err := parseVariableMapping(variableMapping) + if err != nil { + ExitErrorf("%v", err) + } + payload, err := buildLLMEvaluatorPayload( + name, target, samplingRate, traceFilter, hubRef, + promptPath, schemaPath, modelConfigPath, mapping, + ) + if err != nil { + ExitErrorf("%v", err) + } + existing, err := findLLMEvaluatorForCreate(ctx, c, name, target, replace, yes) + if err != nil { + if err.Error() == "aborted" { + ExitError("aborted") + } + if existing != nil { + output.OutputJSON(map[string]any{"error": err.Error(), "id": existing.ID}, "") + return + } + ExitErrorf("%v", err) + } + + var result map[string]any + if existing != nil { + if err := c.RawPatch(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), payload, &result); err != nil { + ExitErrorf("replacing LLM evaluator: %v", err) + } + } else if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { + ExitErrorf("creating LLM evaluator: %v", err) + } + targetLabel := "project" + if target.datasetID != "" { + targetLabel = "dataset" + } + output.OutputJSON(map[string]any{ + "status": "created", "type": "llm", + "id": result["id"], "name": name, "target": targetLabel, + }, "") + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Display name (required)") + cmd.Flags().StringVar(&targetDataset, "dataset", "", "Target dataset name") + cmd.Flags().StringVar(&targetProject, "project", "", "Target project name") + cmd.Flags().Float64Var(&samplingRate, "sampling-rate", 1.0, "Fraction of runs to evaluate (0.0-1.0)") + cmd.Flags().StringVar(&traceFilter, "trace-filter", "", "Filter expression for which runs to evaluate") + cmd.Flags().StringVar(&hubRef, "hub-ref", "", "Prompt Hub reference; replaces --prompt and --schema (e.g. my-org/prompt:latest)") + cmd.Flags().StringVar(&promptPath, "prompt", "", "Prompt JSON file ([[role,content],...] or [{role,content},...]); omit if --hub-ref is set") + cmd.Flags().StringVar(&schemaPath, "schema", "", "JSON schema file for structured output; omit if --hub-ref is set") + cmd.Flags().StringVar(&modelConfigPath, "model-config", "", "Serialized LangChain model JSON (required; copy from UI or GET /runs/rules)") + cmd.Flags().StringVar(&variableMapping, "variable-mapping", "", `Map prompt vars to trace paths (JSON or @file.json)`) + cmd.Flags().BoolVar(&replace, "replace", false, "Replace existing evaluator with same name") + cmd.Flags().BoolVar(&yes, "yes", false, "Skip confirmation when replacing") + _ = cmd.MarkFlagRequired("name") + _ = cmd.MarkFlagRequired("model-config") + + return cmd +} + func newEvaluatorDeleteCmd() *cobra.Command { var yes bool diff --git a/internal/cmd/evaluator_llm.go b/internal/cmd/evaluator_llm.go new file mode 100644 index 0000000..6b73443 --- /dev/null +++ b/internal/cmd/evaluator_llm.go @@ -0,0 +1,213 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/langchain-ai/langsmith-cli/internal/client" + langsmith "github.com/langchain-ai/langsmith-go" +) + +type llmEvaluatorTarget struct { + datasetID, projectID string +} + +func loadJSONFile(path string, dest any) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + if err := json.Unmarshal(data, dest); err != nil { + return fmt.Errorf("parsing JSON in %s: %w", path, err) + } + return nil +} + +// Accepts a plain schema file or one already saved from the LangSmith UI. +func loadEvaluatorSchema(path string) (map[string]any, error) { + var raw map[string]any + if err := loadJSONFile(path, &raw); err != nil { + return nil, err + } + if _, ok := raw["parameters"]; ok { + return raw, nil + } + return map[string]any{"name": "eval", "description": "", "parameters": raw}, nil +} + +func validatePromptMessagePair(i int, path string, msg []string) error { + if len(msg) != 2 || msg[0] == "" || msg[1] == "" { + return fmt.Errorf("prompt message %d in %s must be [role, content] with non-empty values", i, path) + } + return nil +} + +// Loads the judge's system/user messages from a prompt file. +func loadPromptMessagesFromJSON(path string, data []byte) ([][]string, error) { + var pairs [][]string + if err := json.Unmarshal(data, &pairs); err == nil && len(pairs) > 0 { + for i, msg := range pairs { + if err := validatePromptMessagePair(i, path, msg); err != nil { + return nil, err + } + } + return pairs, nil + } + var objects []struct { + Role, Content string + } + if err := json.Unmarshal(data, &objects); err != nil || len(objects) == 0 { + return nil, fmt.Errorf( + "prompt file %s: expected [[role,content],...] or [{role,content},...]", path, + ) + } + pairs = make([][]string, len(objects)) + for i, obj := range objects { + if err := validatePromptMessagePair(i, path, []string{obj.Role, obj.Content}); err != nil { + return nil, err + } + pairs[i] = []string{obj.Role, obj.Content} + } + return pairs, nil +} + +// Maps {{placeholders}} in the prompt to fields on each trace. +func parseVariableMapping(raw string) (map[string]string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } + if strings.HasPrefix(raw, "@") { + var mapping map[string]string + if err := loadJSONFile(raw[1:], &mapping); err != nil { + return nil, err + } + return mapping, nil + } + var mapping map[string]string + if err := json.Unmarshal([]byte(raw), &mapping); err != nil { + return nil, fmt.Errorf("parsing --variable-mapping: %w", err) + } + return mapping, nil +} + +func validateEvaluatorTargetFlags(dataset, project string) error { + if dataset == "" && project == "" { + return fmt.Errorf("must specify --dataset or --project (global evaluators not supported)") + } + if dataset != "" && project != "" { + return fmt.Errorf("cannot specify both --dataset and --project") + } + return nil +} + +// Finds the dataset or project this evaluator should run on. +func resolveLLMEvaluatorTarget(ctx context.Context, c *client.Client, dataset, project string) (llmEvaluatorTarget, error) { + if err := validateEvaluatorTargetFlags(dataset, project); err != nil { + return llmEvaluatorTarget{}, err + } + var target llmEvaluatorTarget + if dataset != "" { + ds, err := resolveDataset(ctx, c, dataset) + if err != nil { + return llmEvaluatorTarget{}, err + } + target.datasetID = ds.ID + } + if project != "" { + sid, err := c.ResolveSessionID(ctx, project) + if err != nil { + return llmEvaluatorTarget{}, err + } + target.projectID = sid + } + return target, nil +} + +// Finds an existing evaluator with the same name on this dataset or project. +// When the evaluator already exists and --replace is not set, returns the existing +// rule alongside the error so callers can reuse it without another list call. +func findLLMEvaluatorForCreate(ctx context.Context, c *client.Client, name string, target llmEvaluatorTarget, replace, yes bool) (*langsmith.Evaluator, error) { + rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) + if err != nil { + return nil, fmt.Errorf("checking existing evaluators: %w", err) + } + existing := findEvaluator(*rules, name, target.datasetID, target.projectID) + if existing == nil { + return nil, nil + } + if !replace { + return existing, fmt.Errorf("evaluator %q already exists (use --replace to overwrite)", name) + } + if !yes { + fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) + var confirm string + _, _ = fmt.Scanln(&confirm) + if strings.ToLower(confirm) != "y" { + return nil, fmt.Errorf("aborted") + } + } + return existing, nil +} + +// Packages prompt, schema, model, and targeting into the create-evaluator request. +func buildLLMEvaluatorPayload( + name string, + target llmEvaluatorTarget, + samplingRate float64, + traceFilter, hubRef, promptPath, schemaPath, modelConfigPath string, + variableMapping map[string]string, +) (map[string]any, error) { + if modelConfigPath == "" { + return nil, fmt.Errorf("--model-config is required") + } + var model map[string]any + if err := loadJSONFile(modelConfigPath, &model); err != nil { + return nil, err + } + + structured := map[string]any{"model": model} + if hubRef != "" { + structured["hub_ref"] = hubRef + } else { + if promptPath == "" || schemaPath == "" { + return nil, fmt.Errorf("--prompt and --schema are required unless --hub-ref is set") + } + data, err := os.ReadFile(promptPath) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", promptPath, err) + } + messages, err := loadPromptMessagesFromJSON(promptPath, data) + if err != nil { + return nil, err + } + schema, err := loadEvaluatorSchema(schemaPath) + if err != nil { + return nil, err + } + structured["prompt"] = messages + structured["schema"] = schema + structured["template_format"] = "mustache" + } + if len(variableMapping) > 0 { + structured["variable_mapping"] = variableMapping + } + payload := map[string]any{ + "display_name": name, "sampling_rate": samplingRate, "is_enabled": true, + "include_extended_stats": false, + "evaluators": []map[string]any{{"structured": structured}}, + } + if traceFilter != "" { + payload["trace_filter"] = traceFilter + } + if target.datasetID != "" { + payload["dataset_id"] = target.datasetID + } + if target.projectID != "" { + payload["session_id"] = target.projectID + } + return payload, nil +} diff --git a/internal/cmd/evaluator_test.go b/internal/cmd/evaluator_test.go index d7a05ef..ae5c482 100644 --- a/internal/cmd/evaluator_test.go +++ b/internal/cmd/evaluator_test.go @@ -367,11 +367,100 @@ func TestFindEvaluator_NameMatchButNoTarget(t *testing.T) { } } +func TestValidateEvaluatorTargetFlags(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + dataset string + project string + wantErr string + }{ + {name: "requires one target", wantErr: "must specify"}, + {name: "rejects both targets", dataset: "ds", project: "proj", wantErr: "cannot specify both"}, + {name: "dataset only", dataset: "ds"}, + {name: "project only", project: "proj"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := validateEvaluatorTargetFlags(tt.dataset, tt.project) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected %q error, got %v", tt.wantErr, err) + } + }) + } +} + +func TestLoadPromptMessagesFromJSON(t *testing.T) { + t.Parallel() + + t.Run("pair format", func(t *testing.T) { + t.Parallel() + msgs, err := loadPromptMessagesFromJSON("prompt.json", []byte( + `[["system","You are a judge."],["user","Q: {{input}}"]]`, + )) + if err != nil { + t.Fatal(err) + } + if len(msgs) != 2 || msgs[0][0] != "system" { + t.Fatalf("unexpected messages: %#v", msgs) + } + }) + + t.Run("role content objects", func(t *testing.T) { + t.Parallel() + msgs, err := loadPromptMessagesFromJSON("prompt.json", []byte( + `[{"role":"system","content":"You are a judge."},{"role":"user","content":"Q: {{input}}"}]`, + )) + if err != nil { + t.Fatal(err) + } + if len(msgs) != 2 || msgs[1][1] != "Q: {{input}}" { + t.Fatalf("unexpected messages: %#v", msgs) + } + }) + + t.Run("invalid format", func(t *testing.T) { + t.Parallel() + _, err := loadPromptMessagesFromJSON("prompt.json", []byte(`{"messages":[]}`)) + if err == nil || !strings.Contains(err.Error(), "expected [[role,content]") { + t.Fatalf("expected format hint, got %v", err) + } + }) + + t.Run("rejects single-element pair", func(t *testing.T) { + t.Parallel() + _, err := loadPromptMessagesFromJSON("prompt.json", []byte(`[["system"]]`)) + if err == nil || !strings.Contains(err.Error(), "must be [role, content]") { + t.Fatalf("expected length error, got %v", err) + } + }) +} + +func TestBuildLLMEvaluatorPayload_requiresModelConfig(t *testing.T) { + t.Parallel() + + _, err := buildLLMEvaluatorPayload( + "relevance", llmEvaluatorTarget{projectID: "proj-1"}, + 1.0, "", "", "prompt.json", "schema.json", "", nil, + ) + if err == nil || !strings.Contains(err.Error(), "--model-config is required") { + t.Fatalf("expected model-config error, got %v", err) + } +} + // ==================== Command structure tests ==================== func TestEvaluatorCmd_Subcommands(t *testing.T) { cmd := newEvaluatorCmd() - expected := map[string]bool{"get": false, "list": false, "upload": false, "delete": false} + expected := map[string]bool{"get": false, "list": false, "upload": false, "create-llm": false, "delete": false} for _, sub := range cmd.Commands() { if _, ok := expected[sub.Name()]; ok { expected[sub.Name()] = true @@ -445,6 +534,7 @@ func TestEvaluatorUploadCmd_Flags(t *testing.T) { "dataset": "", "project": "", "sampling-rate": "1", + "trace-filter": "", "replace": "false", "yes": "false", } @@ -744,6 +834,90 @@ func TestEvaluatorUploadReplacePatchesExistingCodeEvaluator(t *testing.T) { } } +func TestEvaluatorCreateLLMReplacePatchesExistingEvaluator(t *testing.T) { + modelConfigPath := t.TempDir() + "/model.json" + if err := os.WriteFile( + modelConfigPath, + []byte(`{"type":"chat","config":{"model":"test-model"}}`), + 0o600, + ); err != nil { + t.Fatal(err) + } + + var sawDelete bool + var patchBody map[string]any + ts := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/api/v1/sessions" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]map[string]any{ + {"id": "project-1", "name": "my-project"}, + }) + case r.URL.Path == "/api/v1/runs/rules" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]testRule{ + { + ID: "existing-rule", + DisplayName: "relevance", + SamplingRate: 0.25, + IsEnabled: true, + SessionID: "project-1", + Evaluators: []testLLMEval{ + {Structured: testLLMStructured{HubRef: "my-org/old:latest"}}, + }, + }, + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "PATCH": + if err := json.NewDecoder(r.Body).Decode(&patchBody); err != nil { + t.Fatalf("decoding patch body: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "existing-rule", + "display_name": "relevance", + "session_id": "project-1", + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "DELETE": + sawDelete = true + http.Error(w, "delete should not be called", http.StatusInternalServerError) + default: + http.Error(w, "not found", http.StatusNotFound) + } + }) + + cleanup := setupTestEnv(t, ts.URL) + defer cleanup() + flagOutputFormat = "json" + + out := captureStdout(t, func() { + cmd := newEvaluatorCreateLLMCmd() + _ = cmd.Flags().Set("name", "relevance") + _ = cmd.Flags().Set("project", "my-project") + _ = cmd.Flags().Set("hub-ref", "my-org/relevance:latest") + _ = cmd.Flags().Set("model-config", modelConfigPath) + _ = cmd.Flags().Set("sampling-rate", "0.5") + _ = cmd.Flags().Set("replace", "true") + _ = cmd.Flags().Set("yes", "true") + cmd.Run(cmd, nil) + }) + + if sawDelete { + t.Fatal("create-llm --replace should patch the existing evaluator, not delete it") + } + if patchBody == nil { + t.Fatal("expected PATCH body") + } + if patchBody["display_name"] != "relevance" || patchBody["evaluators"] == nil { + t.Fatalf("expected LLM evaluator replacement payload, got %#v", patchBody) + } + + var result map[string]any + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("failed to parse output JSON: %v\noutput: %s", err, out) + } + if result["id"] != "existing-rule" { + t.Errorf("expected output id=existing-rule, got %v", result["id"]) + } +} + // ==================== evaluator get ==================== func TestEvaluatorGetCmd_UseField(t *testing.T) { From 031485817e8f67efb3114156b30ec96df7ba1900 Mon Sep 17 00:00:00 2001 From: Ramon Petgrave <32398091+ramonpetgrave64@users.noreply.github.com> Date: Fri, 17 Jul 2026 05:27:42 +0700 Subject: [PATCH 8/9] chore: sync latest langsmith-go SDK [copybara][addresses ENT-1219] (#30615) ## Description Syncs the vendored Go SDK from `langchain-ai/langsmith-go` main using the repository's Copybara workflow, bringing it to v0.20.1. ## Test Plan - [x] `go -C sdks/go test ./...` - [x] Verify changed Go files pass `gofmt -l` Made by [Open SWE](https://openswe.vercel.app/agents/80e7d2a9-2816-8835-06d6-935f9fdfb4a4) ## References - Slack thread: https://langchain.slack.com/archives/C0B3HS27NTH/p1784219110832059?thread_ts=1784219110.832059&cid=C0B3HS27NTH --------- Co-authored-by: open-swe[bot] GitOrigin-RevId: 24e8b80797548cc311562992249982b2087de6ce --- .github/workflows/ci.yml | 3 + README.md | 28 ++- go.mod | 24 +-- go.sum | 48 ++--- internal/client/client.go | 14 +- internal/client/client_test.go | 59 +++++- internal/cmd/evaluator.go | 188 +++++-------------- internal/cmd/evaluator_llm.go | 213 --------------------- internal/cmd/evaluator_test.go | 281 +--------------------------- internal/cmd/helpers.go | 27 ++- internal/cmd/helpers_test.go | 66 +++++++ internal/cmd/install_method.go | 101 ++++++++++ internal/cmd/install_method_test.go | 233 +++++++++++++++++++++++ internal/cmd/root.go | 4 + internal/cmd/root_test.go | 55 ++++++ internal/cmd/run.go | 12 +- internal/cmd/sandbox_box.go | 2 +- internal/cmd/sandbox_test.go | 6 +- internal/cmd/thread.go | 4 +- internal/cmd/trace.go | 12 +- internal/cmd/update.go | 63 +++++-- internal/cmd/update_test.go | 6 +- internal/cmdutil/resolve.go | 4 + internal/cmdutil/resolve_test.go | 38 ++++ internal/extract/extract.go | 9 + internal/extract/extract_test.go | 31 +++ scripts/install.sh | 24 ++- 27 files changed, 819 insertions(+), 736 deletions(-) delete mode 100644 internal/cmd/evaluator_llm.go create mode 100644 internal/cmd/install_method.go create mode 100644 internal/cmd/install_method_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f0db1e..4ad39e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,9 @@ jobs: os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index 9f7168a..600dfb9 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,5 @@ # langsmith-cli -> [!IMPORTANT] -> **Alpha** — This CLI is under active development. Commands, flags, and output schemas may change between releases. Feedback and bug reports welcome via [GitHub Issues](https://github.com/langchain-ai/langsmith-cli/issues). - An agent-first CLI for querying and managing [LangSmith](https://smith.langchain.com) resources. Built for AI coding agents (deepagents, Claude Code, Cursor, etc.) and developers who need fast, scriptable access to projects, traces, runs, datasets, evaluators, experiments, and threads. @@ -268,18 +265,6 @@ langsmith evaluator upload evals.py \ # Delete an evaluator langsmith evaluator delete accuracy --yes - -# Create an LLM-as-judge evaluator (--model-config is always required) -# model.json: copy the structured.model block from an existing evaluator or the UI. -langsmith evaluator create-llm \ - --name relevance --project my-app \ - --prompt prompt.json --schema schema.json --model-config model.json \ - --variable-mapping '{"input":"input.question","output":"output.answer"}' - -# Or reference an existing Prompt Hub commit (--hub-ref replaces --prompt and --schema) -langsmith evaluator create-llm \ - --name relevance --project my-app \ - --hub-ref my-org/relevance:latest --model-config model.json ``` ### `experiment` — Query experiment results @@ -331,6 +316,19 @@ langsmith self-update --dry-run langsmith self-update ``` +If langsmith was installed through a package manager, `self-update` won't replace the +binary in place — it points you at the right command instead: + +| Installed via | Update with | +| --- | --- | +| Homebrew | `brew upgrade langchain-ai/tap/langsmith-cli` | +| Scoop | `scoop update langsmith-cli` | +| `go install` | `go install github.com/langchain-ai/langsmith-cli/cmd/langsmith@latest` | + +Installs from the `install.sh`/`install.ps1` scripts or a direct GitHub Releases download +are updated in place as usual. Pass `--force` to update in place regardless of how +langsmith was installed. + ### `trace setup` — Trace coding agents to LangSmith Configure Claude Code or Codex to send full-content traces (prompts, responses, tool diff --git a/go.mod b/go.mod index 96ba171..c75e138 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.25.0 require ( github.com/google/uuid v1.6.0 github.com/itchyny/gojq v0.12.15 - github.com/langchain-ai/langsmith-go v0.19.0 + github.com/langchain-ai/langsmith-go v0.18.2 github.com/olekukonko/tablewriter v0.0.5 github.com/pelletier/go-toml/v2 v2.2.4 github.com/spf13/cobra v1.8.1 @@ -21,10 +21,10 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect - github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/compress v1.19.0 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -34,18 +34,18 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/grpc v1.80.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.81.1 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ebe81f4..5a5e71a 100644 --- a/go.sum +++ b/go.sum @@ -17,16 +17,16 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/itchyny/gojq v0.12.15 h1:WC1Nxbx4Ifw5U2oQWACYz32JK8G9qxNtHzrvW4KEcqI= github.com/itchyny/gojq v0.12.15/go.mod h1:uWAHCbCIla1jiNxmeT5/B5mOjSdfkCq6p8vxWg+BM10= github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -68,20 +68,20 @@ github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -96,12 +96,12 @@ golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/client/client.go b/internal/client/client.go index 2da3119..4f13138 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -58,13 +58,17 @@ func NewWithOptions(options Options) *Client { normalized := NormalizeURL(options.APIURL) var opts []option.RequestOption - if options.ProfileName != "" && options.APIKey == "" { + // Auth precedence mirrors resolveClientOptions. A resolved profile is routed + // through WithProfile so an explicit selection replaces the config's + // current_profile (clearing any inherited tenant/base URL); WithProfile + // supplies the profile's own auth (API key or OAuth). A profile-less explicit + // key or bearer is applied directly. + switch { + case options.ProfileName != "": opts = append(opts, langsmith.WithProfile(options.ProfileName)) - } - if options.APIKey != "" { + case options.APIKey != "": opts = append(opts, option.WithAPIKey(options.APIKey)) - } - if options.OAuthAccessToken != "" && options.ProfileName == "" && options.APIKey == "" { + case options.OAuthAccessToken != "": opts = append(opts, option.WithHeader("authorization", "Bearer "+options.OAuthAccessToken)) } // Only set base URL if not the default (the SDK reads LANGSMITH_ENDPOINT too). diff --git a/internal/client/client_test.go b/internal/client/client_test.go index 241b72d..41aeb26 100644 --- a/internal/client/client_test.go +++ b/internal/client/client_test.go @@ -136,7 +136,52 @@ func TestNewWithOptions_ProfileNameDelegatesAuthToSDK(t *testing.T) { } } -func TestNewWithOptions_APIKeyOverridesProfileName(t *testing.T) { +func TestNewWithOptions_APIKeyProfileRoutesThroughSDK(t *testing.T) { + var gotAPIKey string + var gotAuth string + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAPIKey = r.Header.Get("X-API-Key") + gotAuth = r.Header.Get("Authorization") + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"version": "test"}) + })) + defer ts.Close() + + path := filepath.Join(t.TempDir(), "config.json") + t.Setenv("LANGSMITH_CONFIG_FILE", path) + t.Setenv("LANGSMITH_PROFILE", "") + t.Setenv("LANGSMITH_API_KEY", "") + if err := os.WriteFile(path, []byte(`{ + "profiles": { + "prod": { + "api_key": "prod-api-key" + } + } +} +`), 0600); err != nil { + t.Fatal(err) + } + + // An api-key profile carries both ProfileName and its own APIKey (as + // resolveClientOptions produces). It routes through WithProfile: the profile's + // key is sent, no bearer. + c := NewWithOptions(Options{ + APIKey: "prod-api-key", + ProfileName: "prod", + APIURL: ts.URL, + }) + if _, err := c.SDK.Info.List(context.Background()); err != nil { + t.Fatal(err) + } + if gotAPIKey != "prod-api-key" { + t.Fatalf("expected profile api key, got %q", gotAPIKey) + } + if gotAuth != "" { + t.Fatalf("expected no bearer for an api-key profile, got %q", gotAuth) + } +} + +func TestNewWithOptions_ProfileNameTakesPrecedenceOverExplicitAPIKey(t *testing.T) { var gotAPIKey string var gotAuth string ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -164,6 +209,10 @@ func TestNewWithOptions_APIKeyOverridesProfileName(t *testing.T) { t.Fatal(err) } + // resolveClientOptions never produces this combo, but NewWithOptions is + // exported: when a caller passes an explicit APIKey alongside a ProfileName, + // the profile wins (routes through WithProfile) and the explicit key is not + // sent for SDK calls. c := NewWithOptions(Options{ APIKey: "explicit-key", ProfileName: "prod", @@ -172,11 +221,11 @@ func TestNewWithOptions_APIKeyOverridesProfileName(t *testing.T) { if _, err := c.SDK.Info.List(context.Background()); err != nil { t.Fatal(err) } - if gotAPIKey != "explicit-key" { - t.Fatalf("expected explicit API key, got %q", gotAPIKey) + if gotAuth != "Bearer prod-access-token" { + t.Fatalf("expected profile OAuth bearer to win, got %q", gotAuth) } - if gotAuth != "" { - t.Fatalf("expected no profile bearer auth when API key wins, got %q", gotAuth) + if gotAPIKey != "" { + t.Fatalf("expected explicit API key not sent for SDK calls, got %q", gotAPIKey) } } diff --git a/internal/cmd/evaluator.go b/internal/cmd/evaluator.go index d48c1e5..b936e72 100644 --- a/internal/cmd/evaluator.go +++ b/internal/cmd/evaluator.go @@ -20,10 +20,9 @@ func newEvaluatorCmd() *cobra.Command { Long: `Manage online and offline evaluator rules. Evaluators automatically score runs against a dataset (offline) or a project -(online). Supported evaluator types: +(online). Two evaluator types are supported: - upload Code evaluator from a Python or JavaScript/TypeScript file - create-llm LLM-as-judge evaluator (--model-config required; prompt via files or --hub-ref) + upload Code evaluator from a Python or JavaScript/TypeScript file Examples: langsmith evaluator list @@ -32,14 +31,12 @@ Examples: langsmith evaluator get accuracy --session-id langsmith evaluator upload eval.py --name accuracy --function check_accuracy --dataset my-eval-set langsmith evaluator upload eval.ts --name accuracy --function checkAccuracy --dataset my-eval-set - langsmith evaluator create-llm --name relevance --project my-app --prompt prompt.json --schema schema.json --model-config model.json langsmith evaluator delete accuracy --yes`, } cmd.AddCommand(newEvaluatorGetCmd()) cmd.AddCommand(newEvaluatorListCmd()) cmd.AddCommand(newEvaluatorUploadCmd()) - cmd.AddCommand(newEvaluatorCreateLLMCmd()) cmd.AddCommand(newEvaluatorDeleteCmd()) return cmd } @@ -199,7 +196,6 @@ func newEvaluatorUploadCmd() *cobra.Command { targetDataset string targetProject string samplingRate float64 - traceFilter string replace bool yes bool ) @@ -211,15 +207,19 @@ func newEvaluatorUploadCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { evaluatorFile := args[0] - if err := validateEvaluatorTargetFlags(targetDataset, targetProject); err != nil { - output.OutputJSON(map[string]any{"error": err.Error()}, "") + if targetDataset == "" && targetProject == "" { + output.OutputJSON(map[string]any{ + "error": "Must specify --dataset or --project (global evaluators not supported)", + }, "") return } c := MustGetClient() ctx := context.Background() + // Resolve targets var datasetID, projectID string + if targetDataset != "" { ds, err := resolveDataset(ctx, c, targetDataset) if err != nil { @@ -227,6 +227,7 @@ func newEvaluatorUploadCmd() *cobra.Command { } datasetID = ds.ID } + if targetProject != "" { sid, err := c.ResolveSessionID(ctx, targetProject) if err != nil { @@ -235,6 +236,34 @@ func newEvaluatorUploadCmd() *cobra.Command { projectID = sid } + // Check for existing evaluator + rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) + if err != nil { + ExitErrorf("checking existing evaluators: %v", err) + } + + existing := findEvaluator(*rules, name, datasetID, projectID) + if existing != nil { + if !replace { + output.OutputJSON(map[string]any{ + "error": fmt.Sprintf("Evaluator '%s' already exists (use --replace to overwrite)", name), + "id": existing.ID, + }, "") + return + } + if !yes { + fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) + var confirm string + _, _ = fmt.Scanln(&confirm) + if strings.ToLower(confirm) != "y" { + ExitError("aborted") + } + } + if err := c.RawDelete(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), nil); err != nil { + ExitErrorf("deleting existing evaluator: %v", err) + } + } + // Read and prepare function source source, err := os.ReadFile(evaluatorFile) if err != nil { @@ -263,6 +292,7 @@ func newEvaluatorUploadCmd() *cobra.Command { sourceStr = renameJSFunction(sourceStr, funcName) } + // Build payload payload := map[string]any{ "display_name": name, "sampling_rate": samplingRate, @@ -272,60 +302,29 @@ func newEvaluatorUploadCmd() *cobra.Command { {"code": sourceStr, "language": language}, }, } + if datasetID != "" { payload["dataset_id"] = datasetID } if projectID != "" { payload["session_id"] = projectID } - if traceFilter != "" { - payload["trace_filter"] = traceFilter - } - - // Prepare the new evaluator before touching any existing one. If this is - // a replacement, the old evaluator stays in place until the new version is ready. - rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) - if err != nil { - ExitErrorf("checking existing evaluators: %v", err) - } - - existing := findEvaluator(*rules, name, datasetID, projectID) - if existing != nil { - if !replace { - output.OutputJSON(map[string]any{ - "error": fmt.Sprintf("Evaluator '%s' already exists (use --replace to overwrite)", name), - "id": existing.ID, - }, "") - return - } - if !yes { - fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) - var confirm string - _, _ = fmt.Scanln(&confirm) - if strings.ToLower(confirm) != "y" { - ExitError("aborted") - } - } - } var result map[string]any - if existing != nil { - if err := c.RawPatch(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), payload, &result); err != nil { - ExitErrorf("replacing evaluator: %v", err) - } - } else if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { + if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { ExitErrorf("uploading evaluator: %v", err) } - targetLabel := "project" + target := "project" if datasetID != "" { - targetLabel = "dataset" + target = "dataset" } + output.OutputJSON(map[string]any{ "status": "uploaded", "id": result["id"], "name": name, - "target": targetLabel, + "target": target, }, "") }, } @@ -335,7 +334,6 @@ func newEvaluatorUploadCmd() *cobra.Command { cmd.Flags().StringVar(&targetDataset, "dataset", "", "Target dataset name (offline evaluator)") cmd.Flags().StringVar(&targetProject, "project", "", "Target project name (online evaluator)") cmd.Flags().Float64Var(&samplingRate, "sampling-rate", 1.0, "Fraction of runs to evaluate (0.0-1.0)") - cmd.Flags().StringVar(&traceFilter, "trace-filter", "", "Filter expression for which runs to evaluate") cmd.Flags().BoolVar(&replace, "replace", false, "Replace existing evaluator with same name") cmd.Flags().BoolVar(&yes, "yes", false, "Skip confirmation prompt when replacing") _ = cmd.MarkFlagRequired("name") @@ -344,104 +342,6 @@ func newEvaluatorUploadCmd() *cobra.Command { return cmd } -// Registers create-llm for LLM-as-judge evaluators. -func newEvaluatorCreateLLMCmd() *cobra.Command { - var ( - name string - targetDataset string - targetProject string - samplingRate float64 - traceFilter string - hubRef string - promptPath string - schemaPath string - modelConfigPath string - variableMapping string - replace bool - yes bool - ) - - cmd := &cobra.Command{ - Use: "create-llm", - Short: "Create an LLM-as-judge evaluator rule", - Long: `Create an LLM-as-judge run rule. --model-config is always required. - -Provide the judge prompt inline with --prompt and --schema, or point at Prompt Hub -with --hub-ref (which replaces --prompt and --schema). The model is configured separately. - -Examples: - langsmith evaluator create-llm --name relevance --project my-app \ - --prompt prompt.json --schema schema.json --model-config model.json - langsmith evaluator create-llm --name relevance --project my-app \ - --hub-ref my-org/relevance:latest --model-config model.json`, - Run: func(cmd *cobra.Command, args []string) { - c := MustGetClient() - ctx := context.Background() - - target, err := resolveLLMEvaluatorTarget(ctx, c, targetDataset, targetProject) - if err != nil { - ExitErrorf("%v", err) - } - mapping, err := parseVariableMapping(variableMapping) - if err != nil { - ExitErrorf("%v", err) - } - payload, err := buildLLMEvaluatorPayload( - name, target, samplingRate, traceFilter, hubRef, - promptPath, schemaPath, modelConfigPath, mapping, - ) - if err != nil { - ExitErrorf("%v", err) - } - existing, err := findLLMEvaluatorForCreate(ctx, c, name, target, replace, yes) - if err != nil { - if err.Error() == "aborted" { - ExitError("aborted") - } - if existing != nil { - output.OutputJSON(map[string]any{"error": err.Error(), "id": existing.ID}, "") - return - } - ExitErrorf("%v", err) - } - - var result map[string]any - if existing != nil { - if err := c.RawPatch(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), payload, &result); err != nil { - ExitErrorf("replacing LLM evaluator: %v", err) - } - } else if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { - ExitErrorf("creating LLM evaluator: %v", err) - } - targetLabel := "project" - if target.datasetID != "" { - targetLabel = "dataset" - } - output.OutputJSON(map[string]any{ - "status": "created", "type": "llm", - "id": result["id"], "name": name, "target": targetLabel, - }, "") - }, - } - - cmd.Flags().StringVar(&name, "name", "", "Display name (required)") - cmd.Flags().StringVar(&targetDataset, "dataset", "", "Target dataset name") - cmd.Flags().StringVar(&targetProject, "project", "", "Target project name") - cmd.Flags().Float64Var(&samplingRate, "sampling-rate", 1.0, "Fraction of runs to evaluate (0.0-1.0)") - cmd.Flags().StringVar(&traceFilter, "trace-filter", "", "Filter expression for which runs to evaluate") - cmd.Flags().StringVar(&hubRef, "hub-ref", "", "Prompt Hub reference; replaces --prompt and --schema (e.g. my-org/prompt:latest)") - cmd.Flags().StringVar(&promptPath, "prompt", "", "Prompt JSON file ([[role,content],...] or [{role,content},...]); omit if --hub-ref is set") - cmd.Flags().StringVar(&schemaPath, "schema", "", "JSON schema file for structured output; omit if --hub-ref is set") - cmd.Flags().StringVar(&modelConfigPath, "model-config", "", "Serialized LangChain model JSON (required; copy from UI or GET /runs/rules)") - cmd.Flags().StringVar(&variableMapping, "variable-mapping", "", `Map prompt vars to trace paths (JSON or @file.json)`) - cmd.Flags().BoolVar(&replace, "replace", false, "Replace existing evaluator with same name") - cmd.Flags().BoolVar(&yes, "yes", false, "Skip confirmation when replacing") - _ = cmd.MarkFlagRequired("name") - _ = cmd.MarkFlagRequired("model-config") - - return cmd -} - func newEvaluatorDeleteCmd() *cobra.Command { var yes bool diff --git a/internal/cmd/evaluator_llm.go b/internal/cmd/evaluator_llm.go deleted file mode 100644 index 6b73443..0000000 --- a/internal/cmd/evaluator_llm.go +++ /dev/null @@ -1,213 +0,0 @@ -package cmd - -import ( - "context" - "encoding/json" - "fmt" - "os" - "strings" - - "github.com/langchain-ai/langsmith-cli/internal/client" - langsmith "github.com/langchain-ai/langsmith-go" -) - -type llmEvaluatorTarget struct { - datasetID, projectID string -} - -func loadJSONFile(path string, dest any) error { - data, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("reading %s: %w", path, err) - } - if err := json.Unmarshal(data, dest); err != nil { - return fmt.Errorf("parsing JSON in %s: %w", path, err) - } - return nil -} - -// Accepts a plain schema file or one already saved from the LangSmith UI. -func loadEvaluatorSchema(path string) (map[string]any, error) { - var raw map[string]any - if err := loadJSONFile(path, &raw); err != nil { - return nil, err - } - if _, ok := raw["parameters"]; ok { - return raw, nil - } - return map[string]any{"name": "eval", "description": "", "parameters": raw}, nil -} - -func validatePromptMessagePair(i int, path string, msg []string) error { - if len(msg) != 2 || msg[0] == "" || msg[1] == "" { - return fmt.Errorf("prompt message %d in %s must be [role, content] with non-empty values", i, path) - } - return nil -} - -// Loads the judge's system/user messages from a prompt file. -func loadPromptMessagesFromJSON(path string, data []byte) ([][]string, error) { - var pairs [][]string - if err := json.Unmarshal(data, &pairs); err == nil && len(pairs) > 0 { - for i, msg := range pairs { - if err := validatePromptMessagePair(i, path, msg); err != nil { - return nil, err - } - } - return pairs, nil - } - var objects []struct { - Role, Content string - } - if err := json.Unmarshal(data, &objects); err != nil || len(objects) == 0 { - return nil, fmt.Errorf( - "prompt file %s: expected [[role,content],...] or [{role,content},...]", path, - ) - } - pairs = make([][]string, len(objects)) - for i, obj := range objects { - if err := validatePromptMessagePair(i, path, []string{obj.Role, obj.Content}); err != nil { - return nil, err - } - pairs[i] = []string{obj.Role, obj.Content} - } - return pairs, nil -} - -// Maps {{placeholders}} in the prompt to fields on each trace. -func parseVariableMapping(raw string) (map[string]string, error) { - raw = strings.TrimSpace(raw) - if raw == "" { - return nil, nil - } - if strings.HasPrefix(raw, "@") { - var mapping map[string]string - if err := loadJSONFile(raw[1:], &mapping); err != nil { - return nil, err - } - return mapping, nil - } - var mapping map[string]string - if err := json.Unmarshal([]byte(raw), &mapping); err != nil { - return nil, fmt.Errorf("parsing --variable-mapping: %w", err) - } - return mapping, nil -} - -func validateEvaluatorTargetFlags(dataset, project string) error { - if dataset == "" && project == "" { - return fmt.Errorf("must specify --dataset or --project (global evaluators not supported)") - } - if dataset != "" && project != "" { - return fmt.Errorf("cannot specify both --dataset and --project") - } - return nil -} - -// Finds the dataset or project this evaluator should run on. -func resolveLLMEvaluatorTarget(ctx context.Context, c *client.Client, dataset, project string) (llmEvaluatorTarget, error) { - if err := validateEvaluatorTargetFlags(dataset, project); err != nil { - return llmEvaluatorTarget{}, err - } - var target llmEvaluatorTarget - if dataset != "" { - ds, err := resolveDataset(ctx, c, dataset) - if err != nil { - return llmEvaluatorTarget{}, err - } - target.datasetID = ds.ID - } - if project != "" { - sid, err := c.ResolveSessionID(ctx, project) - if err != nil { - return llmEvaluatorTarget{}, err - } - target.projectID = sid - } - return target, nil -} - -// Finds an existing evaluator with the same name on this dataset or project. -// When the evaluator already exists and --replace is not set, returns the existing -// rule alongside the error so callers can reuse it without another list call. -func findLLMEvaluatorForCreate(ctx context.Context, c *client.Client, name string, target llmEvaluatorTarget, replace, yes bool) (*langsmith.Evaluator, error) { - rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) - if err != nil { - return nil, fmt.Errorf("checking existing evaluators: %w", err) - } - existing := findEvaluator(*rules, name, target.datasetID, target.projectID) - if existing == nil { - return nil, nil - } - if !replace { - return existing, fmt.Errorf("evaluator %q already exists (use --replace to overwrite)", name) - } - if !yes { - fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) - var confirm string - _, _ = fmt.Scanln(&confirm) - if strings.ToLower(confirm) != "y" { - return nil, fmt.Errorf("aborted") - } - } - return existing, nil -} - -// Packages prompt, schema, model, and targeting into the create-evaluator request. -func buildLLMEvaluatorPayload( - name string, - target llmEvaluatorTarget, - samplingRate float64, - traceFilter, hubRef, promptPath, schemaPath, modelConfigPath string, - variableMapping map[string]string, -) (map[string]any, error) { - if modelConfigPath == "" { - return nil, fmt.Errorf("--model-config is required") - } - var model map[string]any - if err := loadJSONFile(modelConfigPath, &model); err != nil { - return nil, err - } - - structured := map[string]any{"model": model} - if hubRef != "" { - structured["hub_ref"] = hubRef - } else { - if promptPath == "" || schemaPath == "" { - return nil, fmt.Errorf("--prompt and --schema are required unless --hub-ref is set") - } - data, err := os.ReadFile(promptPath) - if err != nil { - return nil, fmt.Errorf("reading %s: %w", promptPath, err) - } - messages, err := loadPromptMessagesFromJSON(promptPath, data) - if err != nil { - return nil, err - } - schema, err := loadEvaluatorSchema(schemaPath) - if err != nil { - return nil, err - } - structured["prompt"] = messages - structured["schema"] = schema - structured["template_format"] = "mustache" - } - if len(variableMapping) > 0 { - structured["variable_mapping"] = variableMapping - } - payload := map[string]any{ - "display_name": name, "sampling_rate": samplingRate, "is_enabled": true, - "include_extended_stats": false, - "evaluators": []map[string]any{{"structured": structured}}, - } - if traceFilter != "" { - payload["trace_filter"] = traceFilter - } - if target.datasetID != "" { - payload["dataset_id"] = target.datasetID - } - if target.projectID != "" { - payload["session_id"] = target.projectID - } - return payload, nil -} diff --git a/internal/cmd/evaluator_test.go b/internal/cmd/evaluator_test.go index ae5c482..4936821 100644 --- a/internal/cmd/evaluator_test.go +++ b/internal/cmd/evaluator_test.go @@ -3,7 +3,6 @@ package cmd import ( "encoding/json" "net/http" - "os" "strings" "testing" @@ -367,100 +366,11 @@ func TestFindEvaluator_NameMatchButNoTarget(t *testing.T) { } } -func TestValidateEvaluatorTargetFlags(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - dataset string - project string - wantErr string - }{ - {name: "requires one target", wantErr: "must specify"}, - {name: "rejects both targets", dataset: "ds", project: "proj", wantErr: "cannot specify both"}, - {name: "dataset only", dataset: "ds"}, - {name: "project only", project: "proj"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - err := validateEvaluatorTargetFlags(tt.dataset, tt.project) - if tt.wantErr == "" { - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - return - } - if err == nil || !strings.Contains(err.Error(), tt.wantErr) { - t.Fatalf("expected %q error, got %v", tt.wantErr, err) - } - }) - } -} - -func TestLoadPromptMessagesFromJSON(t *testing.T) { - t.Parallel() - - t.Run("pair format", func(t *testing.T) { - t.Parallel() - msgs, err := loadPromptMessagesFromJSON("prompt.json", []byte( - `[["system","You are a judge."],["user","Q: {{input}}"]]`, - )) - if err != nil { - t.Fatal(err) - } - if len(msgs) != 2 || msgs[0][0] != "system" { - t.Fatalf("unexpected messages: %#v", msgs) - } - }) - - t.Run("role content objects", func(t *testing.T) { - t.Parallel() - msgs, err := loadPromptMessagesFromJSON("prompt.json", []byte( - `[{"role":"system","content":"You are a judge."},{"role":"user","content":"Q: {{input}}"}]`, - )) - if err != nil { - t.Fatal(err) - } - if len(msgs) != 2 || msgs[1][1] != "Q: {{input}}" { - t.Fatalf("unexpected messages: %#v", msgs) - } - }) - - t.Run("invalid format", func(t *testing.T) { - t.Parallel() - _, err := loadPromptMessagesFromJSON("prompt.json", []byte(`{"messages":[]}`)) - if err == nil || !strings.Contains(err.Error(), "expected [[role,content]") { - t.Fatalf("expected format hint, got %v", err) - } - }) - - t.Run("rejects single-element pair", func(t *testing.T) { - t.Parallel() - _, err := loadPromptMessagesFromJSON("prompt.json", []byte(`[["system"]]`)) - if err == nil || !strings.Contains(err.Error(), "must be [role, content]") { - t.Fatalf("expected length error, got %v", err) - } - }) -} - -func TestBuildLLMEvaluatorPayload_requiresModelConfig(t *testing.T) { - t.Parallel() - - _, err := buildLLMEvaluatorPayload( - "relevance", llmEvaluatorTarget{projectID: "proj-1"}, - 1.0, "", "", "prompt.json", "schema.json", "", nil, - ) - if err == nil || !strings.Contains(err.Error(), "--model-config is required") { - t.Fatalf("expected model-config error, got %v", err) - } -} - // ==================== Command structure tests ==================== func TestEvaluatorCmd_Subcommands(t *testing.T) { cmd := newEvaluatorCmd() - expected := map[string]bool{"get": false, "list": false, "upload": false, "create-llm": false, "delete": false} + expected := map[string]bool{"get": false, "list": false, "upload": false, "delete": false} for _, sub := range cmd.Commands() { if _, ok := expected[sub.Name()]; ok { expected[sub.Name()] = true @@ -534,7 +444,6 @@ func TestEvaluatorUploadCmd_Flags(t *testing.T) { "dataset": "", "project": "", "sampling-rate": "1", - "trace-filter": "", "replace": "false", "yes": "false", } @@ -730,194 +639,6 @@ func TestEvaluatorListCmd_VerifiesAPIKeyHeader(t *testing.T) { } } -func TestEvaluatorUploadReplacePatchesExistingCodeEvaluator(t *testing.T) { - evaluatorFile := t.TempDir() + "/eval.py" - if err := os.WriteFile( - evaluatorFile, - []byte("def check_accuracy(run, example):\n return {\"score\": 1}\n"), - 0o600, - ); err != nil { - t.Fatal(err) - } - - var sawDelete bool - var patchBody map[string]any - ts := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - switch { - case r.URL.Path == "/api/v1/sessions" && r.Method == "GET": - _ = json.NewEncoder(w).Encode([]map[string]any{ - {"id": "project-1", "name": "my-project"}, - }) - case r.URL.Path == "/api/v1/runs/rules" && r.Method == "GET": - _ = json.NewEncoder(w).Encode([]testRule{ - { - ID: "existing-rule", - DisplayName: "accuracy", - SamplingRate: 0.25, - IsEnabled: true, - SessionID: "project-1", - CodeEvaluators: []testCodeEval{ - {Code: "def perform_eval(run, example):\n return {\"score\": 0}", Language: "python"}, - }, - }, - }) - case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "PATCH": - if err := json.NewDecoder(r.Body).Decode(&patchBody); err != nil { - t.Fatalf("decoding patch body: %v", err) - } - _ = json.NewEncoder(w).Encode(map[string]any{ - "id": "existing-rule", - "display_name": "accuracy", - "session_id": "project-1", - }) - case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "DELETE": - sawDelete = true - http.Error(w, "delete should not be called", http.StatusInternalServerError) - default: - http.Error(w, "not found", http.StatusNotFound) - } - }) - - cleanup := setupTestEnv(t, ts.URL) - defer cleanup() - flagOutputFormat = "json" - - out := captureStdout(t, func() { - cmd := newEvaluatorUploadCmd() - _ = cmd.Flags().Set("name", "accuracy") - _ = cmd.Flags().Set("function", "check_accuracy") - _ = cmd.Flags().Set("project", "my-project") - _ = cmd.Flags().Set("sampling-rate", "0.5") - _ = cmd.Flags().Set("replace", "true") - _ = cmd.Flags().Set("yes", "true") - cmd.Run(cmd, []string{evaluatorFile}) - }) - - if sawDelete { - t.Fatal("upload --replace should patch the existing evaluator, not delete it") - } - if patchBody == nil { - t.Fatal("expected PATCH body") - } - if patchBody["display_name"] != "accuracy" { - t.Errorf("expected display_name=accuracy, got %v", patchBody["display_name"]) - } - if patchBody["session_id"] != "project-1" { - t.Errorf("expected session_id=project-1, got %v", patchBody["session_id"]) - } - if patchBody["sampling_rate"] != 0.5 { - t.Errorf("expected sampling_rate=0.5, got %v", patchBody["sampling_rate"]) - } - evaluators, ok := patchBody["code_evaluators"].([]any) - if !ok || len(evaluators) != 1 { - t.Fatalf("expected one code evaluator, got %#v", patchBody["code_evaluators"]) - } - codeEvaluator, ok := evaluators[0].(map[string]any) - if !ok { - t.Fatalf("expected code evaluator object, got %#v", evaluators[0]) - } - if codeEvaluator["language"] != "python" { - t.Errorf("expected language=python, got %v", codeEvaluator["language"]) - } - code, _ := codeEvaluator["code"].(string) - if !strings.Contains(code, "def perform_eval(") { - t.Errorf("expected uploaded code to be renamed to perform_eval, got:\n%s", code) - } - - var result map[string]any - if err := json.Unmarshal([]byte(out), &result); err != nil { - t.Fatalf("failed to parse output JSON: %v\noutput: %s", err, out) - } - if result["id"] != "existing-rule" { - t.Errorf("expected output id=existing-rule, got %v", result["id"]) - } -} - -func TestEvaluatorCreateLLMReplacePatchesExistingEvaluator(t *testing.T) { - modelConfigPath := t.TempDir() + "/model.json" - if err := os.WriteFile( - modelConfigPath, - []byte(`{"type":"chat","config":{"model":"test-model"}}`), - 0o600, - ); err != nil { - t.Fatal(err) - } - - var sawDelete bool - var patchBody map[string]any - ts := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - switch { - case r.URL.Path == "/api/v1/sessions" && r.Method == "GET": - _ = json.NewEncoder(w).Encode([]map[string]any{ - {"id": "project-1", "name": "my-project"}, - }) - case r.URL.Path == "/api/v1/runs/rules" && r.Method == "GET": - _ = json.NewEncoder(w).Encode([]testRule{ - { - ID: "existing-rule", - DisplayName: "relevance", - SamplingRate: 0.25, - IsEnabled: true, - SessionID: "project-1", - Evaluators: []testLLMEval{ - {Structured: testLLMStructured{HubRef: "my-org/old:latest"}}, - }, - }, - }) - case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "PATCH": - if err := json.NewDecoder(r.Body).Decode(&patchBody); err != nil { - t.Fatalf("decoding patch body: %v", err) - } - _ = json.NewEncoder(w).Encode(map[string]any{ - "id": "existing-rule", - "display_name": "relevance", - "session_id": "project-1", - }) - case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "DELETE": - sawDelete = true - http.Error(w, "delete should not be called", http.StatusInternalServerError) - default: - http.Error(w, "not found", http.StatusNotFound) - } - }) - - cleanup := setupTestEnv(t, ts.URL) - defer cleanup() - flagOutputFormat = "json" - - out := captureStdout(t, func() { - cmd := newEvaluatorCreateLLMCmd() - _ = cmd.Flags().Set("name", "relevance") - _ = cmd.Flags().Set("project", "my-project") - _ = cmd.Flags().Set("hub-ref", "my-org/relevance:latest") - _ = cmd.Flags().Set("model-config", modelConfigPath) - _ = cmd.Flags().Set("sampling-rate", "0.5") - _ = cmd.Flags().Set("replace", "true") - _ = cmd.Flags().Set("yes", "true") - cmd.Run(cmd, nil) - }) - - if sawDelete { - t.Fatal("create-llm --replace should patch the existing evaluator, not delete it") - } - if patchBody == nil { - t.Fatal("expected PATCH body") - } - if patchBody["display_name"] != "relevance" || patchBody["evaluators"] == nil { - t.Fatalf("expected LLM evaluator replacement payload, got %#v", patchBody) - } - - var result map[string]any - if err := json.Unmarshal([]byte(out), &result); err != nil { - t.Fatalf("failed to parse output JSON: %v\noutput: %s", err, out) - } - if result["id"] != "existing-rule" { - t.Errorf("expected output id=existing-rule, got %v", result["id"]) - } -} - // ==================== evaluator get ==================== func TestEvaluatorGetCmd_UseField(t *testing.T) { diff --git a/internal/cmd/helpers.go b/internal/cmd/helpers.go index 1089e79..282872d 100644 --- a/internal/cmd/helpers.go +++ b/internal/cmd/helpers.go @@ -183,12 +183,14 @@ func buildRunSelectV2(includeIO, includeFeedback bool) []langsmith.RunQueryV2Par langsmith.RunQueryV2ParamsSelectTotalCost, langsmith.RunQueryV2ParamsSelectLatencySeconds, langsmith.RunQueryV2ParamsSelectAppPath, + langsmith.RunQueryV2ParamsSelectFirstTokenTime, } if includeIO { fields = append(fields, langsmith.RunQueryV2ParamsSelectInputs, langsmith.RunQueryV2ParamsSelectOutputs, langsmith.RunQueryV2ParamsSelectError, + langsmith.RunQueryV2ParamsSelectEvents, ) } if includeFeedback { @@ -199,7 +201,7 @@ func buildRunSelectV2(includeIO, includeFeedback bool) []langsmith.RunQueryV2Par // runV2ToSchema converts a v2 Run into the legacy v1 RunSchema shape so the // existing extract/output pipeline can consume it unchanged. -func runV2ToSchema(r langsmith.QueryRunResponse) langsmith.RunSchema { +func runV2ToSchema(r langsmith.Run) langsmith.RunSchema { extra := asMap(r.Extra) if md := asMap(r.Metadata); md != nil { if extra == nil { @@ -240,6 +242,7 @@ func runV2ToSchema(r langsmith.QueryRunResponse) langsmith.RunSchema { Outputs: asMap(r.Outputs), Extra: extra, FeedbackStats: convertFeedbackStats(r.FeedbackStats), + Events: convertEvents(r.Events), } if len(r.ParentRunIDs) > 0 { out.ParentRunIDs = r.ParentRunIDs @@ -260,7 +263,7 @@ func asMap(v interface{}) map[string]interface{} { // convertFeedbackStats round-trips the generated feedback-stats shape into the // loosely-typed map the downstream pipeline expects. Returns nil on empty input // or any marshalling error. -func convertFeedbackStats(stats map[string]langsmith.QueryRunResponseFeedbackStat) map[string]map[string]interface{} { +func convertFeedbackStats(stats map[string]langsmith.RunFeedbackStat) map[string]map[string]interface{} { if len(stats) == 0 { return nil } @@ -275,6 +278,24 @@ func convertFeedbackStats(stats map[string]langsmith.QueryRunResponseFeedbackSta return m } +// convertEvents round-trips the generated v2 event shape into the +// loosely-typed maps the legacy RunSchema.Events field expects. Returns nil on +// empty input or any marshalling error. +func convertEvents(events []langsmith.RunEvent) []map[string]interface{} { + if len(events) == 0 { + return nil + } + raw, err := json.Marshal(events) + if err != nil { + return nil + } + var m []map[string]interface{} + if err := json.Unmarshal(raw, &m); err != nil { + return nil + } + return m +} + // buildRunSelect returns the Select fields needed for the given include flags. // Returns nil when neither IO nor feedback is requested, letting the API use its defaults. // When set, includes all base/metadata fields so they aren't stripped from the response. @@ -295,6 +316,7 @@ func buildRunSelect(includeIO, includeFeedback bool) []langsmith.RunQueryParamsS langsmith.RunQueryParamsSelectStatus, // Metadata fields langsmith.RunQueryParamsSelectExtra, + langsmith.RunQueryParamsSelectFirstTokenTime, langsmith.RunQueryParamsSelectPromptTokens, langsmith.RunQueryParamsSelectCompletionTokens, langsmith.RunQueryParamsSelectTotalTokens, @@ -309,6 +331,7 @@ func buildRunSelect(includeIO, includeFeedback bool) []langsmith.RunQueryParamsS langsmith.RunQueryParamsSelectInputs, langsmith.RunQueryParamsSelectOutputs, langsmith.RunQueryParamsSelectError, + langsmith.RunQueryParamsSelectEvents, ) } diff --git a/internal/cmd/helpers_test.go b/internal/cmd/helpers_test.go index 0cc0bf0..3744954 100644 --- a/internal/cmd/helpers_test.go +++ b/internal/cmd/helpers_test.go @@ -404,6 +404,19 @@ func TestBuildRunSelect_IncludesMetadataFields(t *testing.T) { } } +func TestBuildRunSelect_IncludesFirstTokenTimeAndEvents(t *testing.T) { + // first_token_time and events are native Run fields that --full is + // documented to return; a prior version of this select list silently + // dropped both even when --include-io/--include-metadata were requested. + has := selectSet(buildRunSelect(true, true)) + if !has[langsmith.RunQueryParamsSelectFirstTokenTime] { + t.Error("missing first_token_time field") + } + if !has[langsmith.RunQueryParamsSelectEvents] { + t.Error("missing events field (should be requested alongside inputs/outputs/error)") + } +} + // selectSet converts a slice to a set for easy lookup. func selectSet(sel []langsmith.RunQueryParamsSelect) map[langsmith.RunQueryParamsSelect]bool { m := make(map[langsmith.RunQueryParamsSelect]bool, len(sel)) @@ -484,3 +497,56 @@ func TestRunTreeData_AllFields(t *testing.T) { t.Error("unexpected RunTreeData field values") } } + +// ---------- buildRunSelectV2 ---------- + +func TestBuildRunSelectV2_IncludesFirstTokenTimeAndEvents(t *testing.T) { + // Mirrors TestBuildRunSelect_IncludesFirstTokenTimeAndEvents for the v2 + // select builder: first_token_time is a base field always requested, + // events is requested alongside inputs/outputs/error under --include-io. + base := buildRunSelectV2(false, false) + baseHas := map[langsmith.RunQueryV2ParamsSelect]bool{} + for _, f := range base { + baseHas[f] = true + } + if !baseHas[langsmith.RunQueryV2ParamsSelectFirstTokenTime] { + t.Error("missing first_token_time field in base v2 select set") + } + if baseHas[langsmith.RunQueryV2ParamsSelectEvents] { + t.Error("events should not be requested without --include-io") + } + + withIO := buildRunSelectV2(true, false) + ioHas := map[langsmith.RunQueryV2ParamsSelect]bool{} + for _, f := range withIO { + ioHas[f] = true + } + if !ioHas[langsmith.RunQueryV2ParamsSelectEvents] { + t.Error("missing events field when --include-io requested") + } +} + +// ---------- runV2ToSchema ---------- + +func TestRunV2ToSchema_MapsFirstTokenTimeAndEvents(t *testing.T) { + firstToken := time.Date(2026, 1, 15, 10, 30, 1, 0, time.UTC) + v2 := langsmith.Run{ + ID: "run-123", + FirstTokenTime: firstToken, + Events: []langsmith.RunEvent{ + {Name: "new_token", Kwargs: map[string]interface{}{"token": "hi"}}, + }, + } + + out := runV2ToSchema(v2) + + if !out.FirstTokenTime.Equal(firstToken) { + t.Errorf("expected FirstTokenTime=%v, got %v", firstToken, out.FirstTokenTime) + } + if len(out.Events) != 1 { + t.Fatalf("expected 1 event, got %d", len(out.Events)) + } + if out.Events[0]["name"] != "new_token" { + t.Errorf("expected event name=new_token, got %v", out.Events[0]["name"]) + } +} diff --git a/internal/cmd/install_method.go b/internal/cmd/install_method.go new file mode 100644 index 0000000..987aa39 --- /dev/null +++ b/internal/cmd/install_method.go @@ -0,0 +1,101 @@ +package cmd + +import ( + "path/filepath" + "strings" +) + +// installMethod identifies how the langsmith binary was installed, which +// determines whether self-update may replace the binary in place. +type installMethod int + +const ( + // methodManaged means self-update owns the binary: install.sh, install.ps1, + // or a manual download from GitHub Releases. In-place replacement is correct. + methodManaged installMethod = iota + // methodHomebrew means the binary lives in a Homebrew Cellar. + methodHomebrew + // methodScoop means the binary was installed by Scoop. + methodScoop + // methodGo means the binary was installed via `go install`. + methodGo + // methodDev means a local/development build with no release version. + methodDev +) + +// externalManagers describes installs owned by a package manager: how to name +// the manager in output and the command the user should run to update. Only the +// methods self-update must defer to appear here — methodManaged and methodDev are +// handled by self-update itself and intentionally have no entry, so membership in +// this map is the single source of truth for "must defer to a package manager". +var externalManagers = map[installMethod]struct { + label string // machine-readable name, for JSON output + display string // human-readable name, for the pretty message + command string // the command the user should run to update +}{ + methodHomebrew: {"homebrew", "Homebrew", "brew upgrade langchain-ai/tap/langsmith-cli"}, + methodScoop: {"scoop", "Scoop", "scoop update langsmith-cli"}, + methodGo: {"go", "`go install`", "go install github.com/langchain-ai/langsmith-cli/cmd/langsmith@latest"}, +} + +// detectInstallMethod classifies the install method from the resolved executable +// path, the build version, and the relevant environment values. It is pure: all +// I/O (resolving the path, reading env) happens in the caller. +// +// - execPath must already be resolved through filepath.EvalSymlinks so that +// Homebrew's bin symlinks point at the Cellar. +// - goos is runtime.GOOS. +// - gobin/gopath/home come from $GOBIN, $GOPATH, and the user's home dir. +func detectInstallMethod(execPath, version, goos, gobin, gopath, home string) installMethod { + // Normalize both separators to "/" so a single "/segment/" check works for + // Windows paths regardless of the OS this code runs on (matters for tests). + lower := strings.ToLower(strings.ReplaceAll(execPath, `\`, "/")) + + if strings.Contains(lower, "/cellar/") { + return methodHomebrew + } + if strings.Contains(lower, "/scoop/") { + return methodScoop + } + + dir := filepath.Dir(execPath) + for _, bin := range goBinDirs(gobin, gopath, home) { + if bin != "" && filepath.Clean(dir) == filepath.Clean(bin) { + return methodGo + } + } + + if version == "dev" { + return methodDev + } + return methodManaged +} + +// goBinDirs returns the candidate directories where `go install` places binaries, +// in precedence order: $GOBIN, then $GOPATH/bin, then $HOME/go/bin (Go's default +// GOPATH when unset). +func goBinDirs(gobin, gopath, home string) []string { + if gobin != "" { + return []string{gobin} + } + if gopath != "" { + // GOPATH may contain multiple entries; go install uses the first. + first := filepath.SplitList(gopath) + if len(first) > 0 && first[0] != "" { + return []string{filepath.Join(first[0], "bin")} + } + } + if home != "" { + return []string{filepath.Join(home, "go", "bin")} + } + return nil +} + +// shouldDeferToManager reports whether self-update must defer to a package +// manager instead of replacing the binary in place. --force overrides it. +// Membership in externalManagers is the source of truth; methodManaged and +// methodDev are not externally managed and so are never deferred. +func shouldDeferToManager(method installMethod, force bool) bool { + _, external := externalManagers[method] + return external && !force +} diff --git a/internal/cmd/install_method_test.go b/internal/cmd/install_method_test.go new file mode 100644 index 0000000..78aa832 --- /dev/null +++ b/internal/cmd/install_method_test.go @@ -0,0 +1,233 @@ +package cmd + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestDetectInstallMethod(t *testing.T) { + tests := []struct { + name string + execPath string + version string + goos string + gobin string + gopath string + home string + want installMethod + }{ + { + name: "homebrew apple silicon cellar", + execPath: "/opt/homebrew/Cellar/langsmith-cli/0.2.38/bin/langsmith", + version: "0.2.38", + goos: "darwin", + home: "/Users/me", + want: methodHomebrew, + }, + { + name: "homebrew intel cellar", + execPath: "/usr/local/Cellar/langsmith-cli/0.2.38/bin/langsmith", + version: "0.2.38", + goos: "darwin", + home: "/Users/me", + want: methodHomebrew, + }, + { + name: "homebrew linux cellar", + execPath: "/home/linuxbrew/.linuxbrew/Cellar/langsmith-cli/0.2.38/bin/langsmith", + version: "0.2.38", + goos: "linux", + home: "/home/me", + want: methodHomebrew, + }, + { + name: "scoop apps", + execPath: `C:\Users\me\scoop\apps\langsmith-cli\current\langsmith.exe`, + version: "0.2.38", + goos: "windows", + home: `C:\Users\me`, + want: methodScoop, + }, + { + name: "scoop shims", + execPath: `C:\Users\me\scoop\shims\langsmith.exe`, + version: "0.2.38", + goos: "windows", + home: `C:\Users\me`, + want: methodScoop, + }, + { + name: "go install via GOBIN", + execPath: "/Users/me/dev/gobin/langsmith", + version: "0.2.38", + goos: "darwin", + gobin: "/Users/me/dev/gobin", + home: "/Users/me", + want: methodGo, + }, + { + name: "go install via GOPATH bin", + execPath: "/Users/me/gopath/bin/langsmith", + version: "0.2.38", + goos: "darwin", + gopath: "/Users/me/gopath", + home: "/Users/me", + want: methodGo, + }, + { + name: "go install via default HOME/go/bin", + execPath: "/Users/me/go/bin/langsmith", + version: "0.2.38", + goos: "darwin", + home: "/Users/me", + want: methodGo, + }, + { + name: "go install with dev version still detected as go", + execPath: "/Users/me/go/bin/langsmith", + version: "dev", + goos: "darwin", + home: "/Users/me", + want: methodGo, + }, + { + name: "managed install.sh usr local bin", + execPath: "/usr/local/bin/langsmith", + version: "0.2.38", + goos: "darwin", + home: "/Users/me", + want: methodManaged, + }, + { + name: "managed install.sh local bin", + execPath: "/Users/me/.local/bin/langsmith", + version: "0.2.38", + goos: "linux", + home: "/Users/me", + want: methodManaged, + }, + { + name: "dev build outside package dirs", + execPath: "/Users/me/repos/langsmith-cli/bin/langsmith", + version: "dev", + goos: "darwin", + home: "/Users/me", + want: methodDev, + }, + { + name: "GOBIN takes precedence over GOPATH", + execPath: "/Users/me/gopath/bin/langsmith", + version: "0.2.38", + goos: "darwin", + gobin: "/Users/me/gobin", + gopath: "/Users/me/gopath", + home: "/Users/me", + // exec is in GOPATH/bin, but GOBIN is set and differs → not the go bin dir + want: methodManaged, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := detectInstallMethod(tt.execPath, tt.version, tt.goos, tt.gobin, tt.gopath, tt.home) + if got != tt.want { + t.Errorf("detectInstallMethod() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestExternalManagers(t *testing.T) { + want := map[installMethod]struct{ label, command string }{ + methodHomebrew: {"homebrew", "brew upgrade langchain-ai/tap/langsmith-cli"}, + methodScoop: {"scoop", "scoop update langsmith-cli"}, + methodGo: {"go", "go install github.com/langchain-ai/langsmith-cli/cmd/langsmith@latest"}, + } + for method, exp := range want { + mgr, ok := externalManagers[method] + if !ok { + t.Errorf("externalManagers missing entry for %v", method) + continue + } + if mgr.label != exp.label { + t.Errorf("externalManagers[%v].label = %q, want %q", method, mgr.label, exp.label) + } + if mgr.command != exp.command { + t.Errorf("externalManagers[%v].command = %q, want %q", method, mgr.command, exp.command) + } + if mgr.display == "" { + t.Errorf("externalManagers[%v].display is empty", method) + } + } + // managed and dev are handled by self-update itself and must not be listed. + for _, method := range []installMethod{methodManaged, methodDev} { + if _, ok := externalManagers[method]; ok { + t.Errorf("externalManagers should not contain %v", method) + } + } +} + +func TestShouldDeferToManager(t *testing.T) { + tests := []struct { + method installMethod + force bool + want bool + }{ + {methodHomebrew, false, true}, + {methodHomebrew, true, false}, // --force overrides the defer + {methodScoop, false, true}, + {methodScoop, true, false}, + {methodGo, false, true}, + {methodGo, true, false}, + {methodManaged, false, false}, + {methodManaged, true, false}, + {methodDev, false, false}, + {methodDev, true, false}, + } + for _, tt := range tests { + if got := shouldDeferToManager(tt.method, tt.force); got != tt.want { + t.Errorf("shouldDeferToManager(%v, force=%v) = %v, want %v", tt.method, tt.force, got, tt.want) + } + } +} + +func TestReportManagedExternally_JSON(t *testing.T) { + oldFmt := flagOutputFormat + flagOutputFormat = "json" + defer func() { flagOutputFormat = oldFmt }() + + output := captureStdout(t, func() { + if err := reportManagedExternally(methodHomebrew, "json"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + var result map[string]string + if err := json.Unmarshal([]byte(output), &result); err != nil { + t.Fatalf("invalid JSON output: %v, output: %q", err, output) + } + if result["status"] != "managed-externally" { + t.Errorf("expected status managed-externally, got %q", result["status"]) + } + if result["install_method"] != "homebrew" { + t.Errorf("expected install_method homebrew, got %q", result["install_method"]) + } + if result["update_command"] != "brew upgrade langchain-ai/tap/langsmith-cli" { + t.Errorf("unexpected update_command: %q", result["update_command"]) + } +} + +func TestReportManagedExternally_Pretty(t *testing.T) { + output := captureStdout(t, func() { + if err := reportManagedExternally(methodScoop, "pretty"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + if !strings.Contains(output, "scoop update langsmith-cli") { + t.Errorf("expected pretty output to contain the scoop command, got %q", output) + } + if !strings.Contains(output, "--force") { + t.Errorf("expected pretty output to mention --force, got %q", output) + } +} diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 30f514c..9c8992b 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -198,6 +198,10 @@ func resolveClientOptions(refreshOAuth bool) (client.Options, error) { opts.OAuthAccessToken = profile.AccessToken() case hasProfile && profile.APIKey != "": opts.APIKey = profile.APIKey + // Route the resolved profile through the SDK (WithProfile) so an explicit + // selection replaces the config's current_profile instead of inheriting + // its tenant/base URL. APIKey is kept for the raw-HTTP helpers. + opts.ProfileName = profileName } if cfgErr != nil && opts.APIKey == "" { return opts, cfgErr diff --git a/internal/cmd/root_test.go b/internal/cmd/root_test.go index 0e8431e..d3734c4 100644 --- a/internal/cmd/root_test.go +++ b/internal/cmd/root_test.go @@ -313,6 +313,61 @@ func TestResolveClientOptionsRefreshesProfileWithoutAccessToken(t *testing.T) { } } +func TestResolveClientOptions_SetsProfileNameForAPIKeyProfile(t *testing.T) { + oldKey := flagAPIKey + oldURL := flagAPIURL + oldProfile := flagProfile + oldWS := flagWorkspaceID + defer func() { + flagAPIKey = oldKey + flagAPIURL = oldURL + flagProfile = oldProfile + flagWorkspaceID = oldWS + }() + flagAPIKey = "" + flagAPIURL = "" + flagWorkspaceID = "" + flagProfile = "aws" + + path := filepath.Join(t.TempDir(), "config.json") + t.Setenv("LANGSMITH_CONFIG_FILE", path) + t.Setenv("LANGSMITH_API_KEY", "") + t.Setenv("LANGSMITH_ENDPOINT", "") + t.Setenv("LANGSMITH_WORKSPACE_ID", "") + t.Setenv("LANGSMITH_TENANT_ID", "") + t.Setenv("LANGSMITH_PROFILE", "") + if err := os.WriteFile(path, []byte(`{ + "current_profile": "prod", + "profiles": { + "prod": { + "api_key": "prod-api-key", + "workspace_id": "prod-workspace-id" + }, + "aws": { + "api_key": "aws-api-key" + } + } +} +`), 0600); err != nil { + t.Fatal(err) + } + + opts, err := resolveClientOptions(false) + if err != nil { + t.Fatalf("resolveClientOptions returned error: %v", err) + } + if opts.APIKey != "aws-api-key" { + t.Fatalf("expected profile api key aws-api-key, got %q", opts.APIKey) + } + // The resolved profile must reach the SDK via WithProfile so an explicit + // --profile replaces current_profile and clears its inherited tenant. Without + // ProfileName set, NewWithOptions passes only WithAPIKey and current_profile's + // tenant leaks (403 cross-workspace). + if opts.ProfileName != "aws" { + t.Fatalf("expected ProfileName=aws, got %q", opts.ProfileName) + } +} + func TestGetOAuthAccessToken_ProfileFallback(t *testing.T) { oldKey := flagAPIKey oldURL := flagAPIURL diff --git a/internal/cmd/run.go b/internal/cmd/run.go index a3aea47..5b5d358 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -95,8 +95,8 @@ func newRunListCmd() *cobra.Command { addCommonFilterFlags(cmd, &ff, true) addVersionFlag(cmd, &ff) - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") @@ -179,8 +179,8 @@ func newRunGetCmd() *cobra.Command { cmd.Flags().StringVar(&since, "since", "", "Only include runs after this timestamp, e.g. 2024-01-15T00:00:00Z (overrides 7-day default)") cmd.Flags().IntVar(&lastNMinutes, "last-n-minutes", 0, "Only include runs from the last N minutes, e.g. 60 (overrides 7-day default)") cmd.Flags().StringVar(&version, "version", "", `Query API version: "" (v1, default) or "v2" (SmithDB)`) - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") @@ -244,8 +244,8 @@ func newRunExportCmd() *cobra.Command { addCommonFilterFlags(cmd, &ff, true) addVersionFlag(cmd, &ff) - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") diff --git a/internal/cmd/sandbox_box.go b/internal/cmd/sandbox_box.go index 3738387..049cec3 100644 --- a/internal/cmd/sandbox_box.go +++ b/internal/cmd/sandbox_box.go @@ -16,7 +16,7 @@ import ( const defaultBoxPollInterval = 2 * time.Second -func waitForBoxReady(ctx context.Context, c *client.Client, name string) (*langsmith.SandboxBoxGetStatusResponse, error) { +func waitForBoxReady(ctx context.Context, c *client.Client, name string) (*langsmith.SandboxStatusResponse, error) { for { resp, err := c.SDK.Sandboxes.Boxes.GetStatus(ctx, name) if err != nil { diff --git a/internal/cmd/sandbox_test.go b/internal/cmd/sandbox_test.go index 30030d3..1c7f3d3 100644 --- a/internal/cmd/sandbox_test.go +++ b/internal/cmd/sandbox_test.go @@ -288,7 +288,7 @@ func TestSandboxBoxDetailRenderSupportsSDKResponseTypes(t *testing.T) { }{ { name: "new response", - model: langsmith.SandboxBoxNewResponse{ + model: langsmith.SandboxResponse{ ID: "box-new", Name: "new-vm", Status: "running", @@ -305,7 +305,7 @@ func TestSandboxBoxDetailRenderSupportsSDKResponseTypes(t *testing.T) { }, { name: "get response", - model: langsmith.SandboxBoxGetResponse{ + model: langsmith.SandboxResponse{ ID: "box-get", Name: "get-vm", Status: "running", @@ -322,7 +322,7 @@ func TestSandboxBoxDetailRenderSupportsSDKResponseTypes(t *testing.T) { }, { name: "update response", - model: langsmith.SandboxBoxUpdateResponse{ + model: langsmith.SandboxResponse{ ID: "box-update", Name: "update-vm", Status: "stopped", diff --git a/internal/cmd/thread.go b/internal/cmd/thread.go index 57d4eef..92ab76c 100644 --- a/internal/cmd/thread.go +++ b/internal/cmd/thread.go @@ -267,8 +267,8 @@ func newThreadGetCmd() *cobra.Command { } cmd.Flags().StringVar(&project, "project", "", "Project name [env: LANGSMITH_PROJECT]") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().IntVarP(&limit, "limit", "n", 0, "Maximum number of runs (turns) to return") diff --git a/internal/cmd/trace.go b/internal/cmd/trace.go index bc001d2..ef78232 100644 --- a/internal/cmd/trace.go +++ b/internal/cmd/trace.go @@ -144,8 +144,8 @@ func newTraceListCmd() *cobra.Command { addCommonFilterFlags(cmd, &ff, false) cmd.Flags().StringVar(&ff.ProjectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&includeFlagged, "include-flagged", false, "Add flagged_comment field populated from user-flagged trace feedback") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") @@ -222,8 +222,8 @@ func newTraceGetCmd() *cobra.Command { cmd.Flags().StringVar(&projectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") cmd.Flags().StringVar(&since, "since", "", "Only include runs after this timestamp, e.g. 2024-01-15T00:00:00Z (overrides 7-day default)") cmd.Flags().IntVar(&lastNMinutes, "last-n-minutes", 0, "Only include runs from the last N minutes, e.g. 60 (overrides 7-day default)") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") @@ -332,8 +332,8 @@ func newTraceExportCmd() *cobra.Command { addCommonFilterFlags(cmd, &ff, false) cmd.Flags().StringVar(&ff.ProjectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVar(&filenamePattern, "filename-pattern", "{trace_id}.jsonl", diff --git a/internal/cmd/update.go b/internal/cmd/update.go index 19ba244..d84deb1 100644 --- a/internal/cmd/update.go +++ b/internal/cmd/update.go @@ -28,26 +28,48 @@ var githubDownloadBaseURL = "https://github.com" func newUpdateCmd(rawVersion string) *cobra.Command { var dryRun bool + var force bool cmd := &cobra.Command{ Use: "self-update", Short: "Update langsmith to the latest version", Long: "Check for and install the latest version of the langsmith CLI.", RunE: func(cmd *cobra.Command, args []string) error { - return runUpdate(cmd.Context(), rawVersion, dryRun) + return runUpdate(cmd.Context(), rawVersion, dryRun, force) }, } cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Check for updates without installing") + cmd.Flags().BoolVar(&force, "force", false, "Replace the binary in place even if a package manager (brew, scoop, go) manages this install") return cmd } -func runUpdate(ctx context.Context, currentVersion string, dryRun bool) error { - if currentVersion == "dev" { +func runUpdate(ctx context.Context, currentVersion string, dryRun, force bool) error { + // Resolve the current binary path first: it drives both install-method + // detection and the eventual in-place replacement. + execPath, err := os.Executable() + if err != nil { + return fmt.Errorf("resolving executable path: %w", err) + } + execPath, err = filepath.EvalSymlinks(execPath) + if err != nil { + return fmt.Errorf("resolving symlinks: %w", err) + } + + home, _ := os.UserHomeDir() + method := detectInstallMethod(execPath, currentVersion, runtime.GOOS, os.Getenv("GOBIN"), os.Getenv("GOPATH"), home) + + if method == methodDev { return fmt.Errorf("cannot update a development build; install from a release") } + // For package-manager-owned installs, refuse to touch the binary and point + // the user at the right command instead. --force overrides this. + if shouldDeferToManager(method, force) { + return reportManagedExternally(method, GetFormat()) + } + latest, err := fetchLatestVersion(ctx) if err != nil { return fmt.Errorf("checking for updates: %w", err) @@ -83,16 +105,6 @@ func runUpdate(ctx context.Context, currentVersion string, dryRun bool) error { return nil } - // Resolve current binary path - execPath, err := os.Executable() - if err != nil { - return fmt.Errorf("resolving executable path: %w", err) - } - execPath, err = filepath.EvalSymlinks(execPath) - if err != nil { - return fmt.Errorf("resolving symlinks: %w", err) - } - archiveName := buildArchiveName() archiveURL := fmt.Sprintf("%s/langchain-ai/langsmith-cli/releases/download/v%s/%s", githubDownloadBaseURL, latest, archiveName) checksumURL := fmt.Sprintf("%s/langchain-ai/langsmith-cli/releases/download/v%s/checksums.txt", githubDownloadBaseURL, latest) @@ -146,6 +158,31 @@ func runUpdate(ctx context.Context, currentVersion string, dryRun bool) error { return nil } +// reportManagedExternally informs the user that their install is managed by a +// package manager and prints the command to update it, without modifying the +// binary. It returns nil (a successful, informational outcome). +func reportManagedExternally(method installMethod, format string) error { + mgr, ok := externalManagers[method] + if !ok { + // Programming error: only externally-managed methods should reach here + // (methodManaged updates in place, methodDev errors out earlier). + return fmt.Errorf("internal error: install method %d is not externally managed", method) + } + + if format == "pretty" { + fmt.Printf("langsmith was installed with %s, which manages updates for you.\n"+ + "To update, run:\n\n %s\n\n(Use --force to update in place anyway.)\n", mgr.display, mgr.command) + } else { + out, _ := json.Marshal(map[string]string{ + "status": "managed-externally", + "install_method": mgr.label, + "update_command": mgr.command, + }) + fmt.Println(string(out)) + } + return nil +} + // fetchLatestVersion queries the GitHub Releases API for the latest release tag. func fetchLatestVersion(ctx context.Context) (string, error) { url := fmt.Sprintf("%s/repos/langchain-ai/langsmith-cli/releases/latest", githubReleasesBaseURL) diff --git a/internal/cmd/update_test.go b/internal/cmd/update_test.go index 7580434..8e52280 100644 --- a/internal/cmd/update_test.go +++ b/internal/cmd/update_test.go @@ -203,7 +203,7 @@ func TestRunUpdate_AlreadyUpToDate(t *testing.T) { defer func() { flagOutputFormat = oldFmt }() output := captureStdout(t, func() { - err := runUpdate(context.Background(), "0.1.7", false) + err := runUpdate(context.Background(), "0.1.7", false, false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -235,7 +235,7 @@ func TestRunUpdate_DryRun(t *testing.T) { defer func() { flagOutputFormat = oldFmt }() output := captureStdout(t, func() { - err := runUpdate(context.Background(), "0.1.7", true) + err := runUpdate(context.Background(), "0.1.7", true, false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -254,7 +254,7 @@ func TestRunUpdate_DryRun(t *testing.T) { } func TestRunUpdate_DevBuild(t *testing.T) { - err := runUpdate(context.Background(), "dev", false) + err := runUpdate(context.Background(), "dev", false, false) if err == nil { t.Error("expected error for dev build") } diff --git a/internal/cmdutil/resolve.go b/internal/cmdutil/resolve.go index 0efa9de..7ea4f83 100644 --- a/internal/cmdutil/resolve.go +++ b/internal/cmdutil/resolve.go @@ -187,6 +187,10 @@ func ResolveClientOptions(cmd *cobra.Command, refreshOAuth bool) (client.Options opts.OAuthAccessToken = profile.AccessToken() case hasProfile && profile.APIKey != "": opts.APIKey = profile.APIKey + // Route the resolved profile through the SDK (WithProfile) so an explicit + // selection replaces the config's current_profile instead of inheriting + // its tenant/base URL. APIKey is kept for the raw-HTTP helpers. + opts.ProfileName = profileName } if cfgErr != nil && opts.APIKey == "" { return opts, cfgErr diff --git a/internal/cmdutil/resolve_test.go b/internal/cmdutil/resolve_test.go index ba7ec5a..7dacf5e 100644 --- a/internal/cmdutil/resolve_test.go +++ b/internal/cmdutil/resolve_test.go @@ -205,6 +205,44 @@ func TestResolveClientOptions_ProfileFlagSetsProfileName(t *testing.T) { } } +func TestResolveClientOptions_APIKeyProfileSetsProfileName(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + t.Setenv("LANGSMITH_CONFIG_FILE", path) + t.Setenv("LANGSMITH_API_KEY", "") + t.Setenv("LANGSMITH_ENDPOINT", "") + if err := os.WriteFile(path, []byte(`{ + "current_profile": "prod", + "profiles": { + "prod": { + "api_key": "prod-api-key", + "workspace_id": "ws-prod" + }, + "aws": { + "api_key": "aws-api-key" + } + } +} +`), 0600); err != nil { + t.Fatal(err) + } + + cmd := newTestCmd() + _ = cmd.PersistentFlags().Set("profile", "aws") + opts, err := ResolveClientOptions(cmd, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.APIKey != "aws-api-key" { + t.Fatalf("expected profile api key, got %q", opts.APIKey) + } + // An api-key profile must route through WithProfile too, so it replaces + // current_profile and clears the inherited tenant. This resolver (used by the + // api/sandbox/ssh subcommands) previously omitted ProfileName here. + if opts.ProfileName != "aws" { + t.Fatalf("expected ProfileName=aws, got %q", opts.ProfileName) + } +} + func TestResolveClientOptions_WorkspaceFlagOverridesEnvAndProfile(t *testing.T) { path := filepath.Join(t.TempDir(), "config.json") t.Setenv("LANGSMITH_CONFIG_FILE", path) diff --git a/internal/extract/extract.go b/internal/extract/extract.go index 5e63ed9..c0dc1e4 100644 --- a/internal/extract/extract.go +++ b/internal/extract/extract.go @@ -70,6 +70,7 @@ func ExtractRun(run langsmith.RunSchema, includeMetadata, includeIO, includeFeed result["status"] = run.Status result["duration_ms"] = durationMs + result["first_token_time"] = formatTimeNullable(run.FirstTokenTime) result["custom_metadata"] = customMetadata result["token_usage"] = tokenUsage result["costs"] = costs @@ -80,6 +81,7 @@ func ExtractRun(run langsmith.RunSchema, includeMetadata, includeIO, includeFeed result["inputs"] = nilIfEmptyMap(run.Inputs) result["outputs"] = nilIfEmptyMap(run.Outputs) result["error"] = nilIfEmpty(run.Error) + result["events"] = nilIfEmptyEvents(run.Events) } if includeFeedback { @@ -129,6 +131,13 @@ func nilIfEmptyMap(m map[string]any) any { return m } +func nilIfEmptyEvents(events []map[string]any) any { + if len(events) == 0 { + return nil + } + return events +} + // FormatDurationHuman formats milliseconds as human-readable string. func FormatDurationHuman(ms int64) string { if ms < 1000 { diff --git a/internal/extract/extract_test.go b/internal/extract/extract_test.go index 79c3974..20060c7 100644 --- a/internal/extract/extract_test.go +++ b/internal/extract/extract_test.go @@ -51,6 +51,7 @@ func TestExtractRunBase(t *testing.T) { func TestExtractRunWithMetadata(t *testing.T) { start := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC) end := time.Date(2024, 1, 15, 10, 30, 5, 0, time.UTC) + firstToken := time.Date(2024, 1, 15, 10, 30, 1, 0, time.UTC) run := langsmith.RunSchema{ ID: "run-123", @@ -59,6 +60,7 @@ func TestExtractRunWithMetadata(t *testing.T) { RunType: "llm", StartTime: start, EndTime: end, + FirstTokenTime: firstToken, Status: "success", PromptTokens: 100, CompletionTokens: 50, @@ -76,6 +78,10 @@ func TestExtractRunWithMetadata(t *testing.T) { t.Errorf("expected status=success, got %v", result["status"]) } + if result["first_token_time"] != firstToken.Format(time.RFC3339Nano) { + t.Errorf("expected first_token_time=%s, got %v", firstToken.Format(time.RFC3339Nano), result["first_token_time"]) + } + durationMs, ok := result["duration_ms"].(int64) if !ok || durationMs != 5000 { t.Errorf("expected duration_ms=5000, got %v", result["duration_ms"]) @@ -115,6 +121,7 @@ func TestExtractRunWithIO(t *testing.T) { Inputs: map[string]any{"query": "hello"}, Outputs: map[string]any{"response": "world"}, Error: "some error", + Events: []map[string]interface{}{{"name": "new_token", "kwargs": map[string]interface{}{"token": "hi"}}}, } result := ExtractRun(run, false, true, false) @@ -132,6 +139,30 @@ func TestExtractRunWithIO(t *testing.T) { if result["error"] != "some error" { t.Errorf("expected error='some error', got %v", result["error"]) } + + events, ok := result["events"].([]map[string]interface{}) + if !ok { + t.Fatalf("expected events to be []map[string]interface{}, got %T", result["events"]) + } + if len(events) != 1 || events[0]["name"] != "new_token" { + t.Errorf("expected one new_token event, got %v", events) + } +} + +func TestExtractRunWithIOEmptyEvents(t *testing.T) { + run := langsmith.RunSchema{ + ID: "run-123", + TraceID: "trace-456", + Name: "ChatOpenAI", + RunType: "llm", + StartTime: time.Now(), + } + + result := ExtractRun(run, false, true, false) + + if result["events"] != nil { + t.Errorf("expected events=nil for run with no events, got %v", result["events"]) + } } func TestExtractRunNilParent(t *testing.T) { diff --git a/scripts/install.sh b/scripts/install.sh index 57dc130..284ff6c 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -39,9 +39,29 @@ fi # Get version if [ -z "$VERSION" ]; then - VERSION="$(curl -sSL "https://api.github.com/repos/${REPO}/releases/latest" | grep '"tag_name"' | sed -E 's/.*"tag_name": *"([^"]+)".*/\1/')" + API_URL="https://api.github.com/repos/${REPO}/releases/latest" + + fetch_latest() { + if [ -n "$GITHUB_TOKEN" ]; then + curl -fsSL -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GITHUB_TOKEN" "$API_URL" + else + curl -fsSL -H "Accept: application/vnd.github+json" "$API_URL" + fi + } + + VERSION="" + i=1 + while [ "$i" -le 3 ]; do + VERSION="$(fetch_latest | grep '"tag_name"' | sed -E 's/.*"tag_name": *"([^"]+)".*/\1/')" + [ -n "$VERSION" ] && break + [ "$i" -lt 3 ] && sleep "$i" + i=$((i + 1)) + done + if [ -z "$VERSION" ]; then - echo "Failed to determine latest version" >&2 + echo "Failed to determine latest version (GitHub API unreachable or rate-limited)." >&2 + echo "Set GITHUB_TOKEN to raise the rate limit, or set VERSION to install a specific release." >&2 exit 1 fi fi From 522cf7aaba271c72f597e0e6e0174c8308b33c22 Mon Sep 17 00:00:00 2001 From: Ramon Petgrave <32398091+ramonpetgrave64@users.noreply.github.com> Date: Fri, 17 Jul 2026 05:51:02 +0700 Subject: [PATCH 9/9] Revert "chore: sync latest langsmith-go SDK [copybara][addresses ENT-1219]" (#30694) Reverts langchain-ai/langchainplus#30615 GitOrigin-RevId: e5f86b6b5935a2db90f13584577f83c717df5efd --- .github/workflows/ci.yml | 3 - README.md | 28 +-- go.mod | 24 +-- go.sum | 48 ++--- internal/client/client.go | 14 +- internal/client/client_test.go | 59 +----- internal/cmd/evaluator.go | 188 ++++++++++++++----- internal/cmd/evaluator_llm.go | 213 +++++++++++++++++++++ internal/cmd/evaluator_test.go | 281 +++++++++++++++++++++++++++- internal/cmd/helpers.go | 27 +-- internal/cmd/helpers_test.go | 66 ------- internal/cmd/install_method.go | 101 ---------- internal/cmd/install_method_test.go | 233 ----------------------- internal/cmd/root.go | 4 - internal/cmd/root_test.go | 55 ------ internal/cmd/run.go | 12 +- internal/cmd/sandbox_box.go | 2 +- internal/cmd/sandbox_test.go | 6 +- internal/cmd/thread.go | 4 +- internal/cmd/trace.go | 12 +- internal/cmd/update.go | 63 ++----- internal/cmd/update_test.go | 6 +- internal/cmdutil/resolve.go | 4 - internal/cmdutil/resolve_test.go | 38 ---- internal/extract/extract.go | 9 - internal/extract/extract_test.go | 31 --- scripts/install.sh | 24 +-- 27 files changed, 736 insertions(+), 819 deletions(-) create mode 100644 internal/cmd/evaluator_llm.go delete mode 100644 internal/cmd/install_method.go delete mode 100644 internal/cmd/install_method_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ad39e9..1f0db1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,9 +52,6 @@ jobs: os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index 600dfb9..9f7168a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # langsmith-cli +> [!IMPORTANT] +> **Alpha** — This CLI is under active development. Commands, flags, and output schemas may change between releases. Feedback and bug reports welcome via [GitHub Issues](https://github.com/langchain-ai/langsmith-cli/issues). + An agent-first CLI for querying and managing [LangSmith](https://smith.langchain.com) resources. Built for AI coding agents (deepagents, Claude Code, Cursor, etc.) and developers who need fast, scriptable access to projects, traces, runs, datasets, evaluators, experiments, and threads. @@ -265,6 +268,18 @@ langsmith evaluator upload evals.py \ # Delete an evaluator langsmith evaluator delete accuracy --yes + +# Create an LLM-as-judge evaluator (--model-config is always required) +# model.json: copy the structured.model block from an existing evaluator or the UI. +langsmith evaluator create-llm \ + --name relevance --project my-app \ + --prompt prompt.json --schema schema.json --model-config model.json \ + --variable-mapping '{"input":"input.question","output":"output.answer"}' + +# Or reference an existing Prompt Hub commit (--hub-ref replaces --prompt and --schema) +langsmith evaluator create-llm \ + --name relevance --project my-app \ + --hub-ref my-org/relevance:latest --model-config model.json ``` ### `experiment` — Query experiment results @@ -316,19 +331,6 @@ langsmith self-update --dry-run langsmith self-update ``` -If langsmith was installed through a package manager, `self-update` won't replace the -binary in place — it points you at the right command instead: - -| Installed via | Update with | -| --- | --- | -| Homebrew | `brew upgrade langchain-ai/tap/langsmith-cli` | -| Scoop | `scoop update langsmith-cli` | -| `go install` | `go install github.com/langchain-ai/langsmith-cli/cmd/langsmith@latest` | - -Installs from the `install.sh`/`install.ps1` scripts or a direct GitHub Releases download -are updated in place as usual. Pass `--force` to update in place regardless of how -langsmith was installed. - ### `trace setup` — Trace coding agents to LangSmith Configure Claude Code or Codex to send full-content traces (prompts, responses, tool diff --git a/go.mod b/go.mod index c75e138..96ba171 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.25.0 require ( github.com/google/uuid v1.6.0 github.com/itchyny/gojq v0.12.15 - github.com/langchain-ai/langsmith-go v0.18.2 + github.com/langchain-ai/langsmith-go v0.19.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pelletier/go-toml/v2 v2.2.4 github.com/spf13/cobra v1.8.1 @@ -21,10 +21,10 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect - github.com/klauspost/compress v1.19.0 // indirect + github.com/klauspost/compress v1.18.6 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -34,18 +34,18 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel v1.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 // indirect - go.opentelemetry.io/otel/metric v1.44.0 // indirect - go.opentelemetry.io/otel/sdk v1.44.0 // indirect - go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.81.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/grpc v1.80.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 5a5e71a..ebe81f4 100644 --- a/go.sum +++ b/go.sum @@ -17,16 +17,16 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/itchyny/gojq v0.12.15 h1:WC1Nxbx4Ifw5U2oQWACYz32JK8G9qxNtHzrvW4KEcqI= github.com/itchyny/gojq v0.12.15/go.mod h1:uWAHCbCIla1jiNxmeT5/B5mOjSdfkCq6p8vxWg+BM10= github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= -github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= -github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -68,20 +68,20 @@ github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -96,12 +96,12 @@ golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= -google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/client/client.go b/internal/client/client.go index 4f13138..2da3119 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -58,17 +58,13 @@ func NewWithOptions(options Options) *Client { normalized := NormalizeURL(options.APIURL) var opts []option.RequestOption - // Auth precedence mirrors resolveClientOptions. A resolved profile is routed - // through WithProfile so an explicit selection replaces the config's - // current_profile (clearing any inherited tenant/base URL); WithProfile - // supplies the profile's own auth (API key or OAuth). A profile-less explicit - // key or bearer is applied directly. - switch { - case options.ProfileName != "": + if options.ProfileName != "" && options.APIKey == "" { opts = append(opts, langsmith.WithProfile(options.ProfileName)) - case options.APIKey != "": + } + if options.APIKey != "" { opts = append(opts, option.WithAPIKey(options.APIKey)) - case options.OAuthAccessToken != "": + } + if options.OAuthAccessToken != "" && options.ProfileName == "" && options.APIKey == "" { opts = append(opts, option.WithHeader("authorization", "Bearer "+options.OAuthAccessToken)) } // Only set base URL if not the default (the SDK reads LANGSMITH_ENDPOINT too). diff --git a/internal/client/client_test.go b/internal/client/client_test.go index 41aeb26..241b72d 100644 --- a/internal/client/client_test.go +++ b/internal/client/client_test.go @@ -136,52 +136,7 @@ func TestNewWithOptions_ProfileNameDelegatesAuthToSDK(t *testing.T) { } } -func TestNewWithOptions_APIKeyProfileRoutesThroughSDK(t *testing.T) { - var gotAPIKey string - var gotAuth string - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotAPIKey = r.Header.Get("X-API-Key") - gotAuth = r.Header.Get("Authorization") - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]any{"version": "test"}) - })) - defer ts.Close() - - path := filepath.Join(t.TempDir(), "config.json") - t.Setenv("LANGSMITH_CONFIG_FILE", path) - t.Setenv("LANGSMITH_PROFILE", "") - t.Setenv("LANGSMITH_API_KEY", "") - if err := os.WriteFile(path, []byte(`{ - "profiles": { - "prod": { - "api_key": "prod-api-key" - } - } -} -`), 0600); err != nil { - t.Fatal(err) - } - - // An api-key profile carries both ProfileName and its own APIKey (as - // resolveClientOptions produces). It routes through WithProfile: the profile's - // key is sent, no bearer. - c := NewWithOptions(Options{ - APIKey: "prod-api-key", - ProfileName: "prod", - APIURL: ts.URL, - }) - if _, err := c.SDK.Info.List(context.Background()); err != nil { - t.Fatal(err) - } - if gotAPIKey != "prod-api-key" { - t.Fatalf("expected profile api key, got %q", gotAPIKey) - } - if gotAuth != "" { - t.Fatalf("expected no bearer for an api-key profile, got %q", gotAuth) - } -} - -func TestNewWithOptions_ProfileNameTakesPrecedenceOverExplicitAPIKey(t *testing.T) { +func TestNewWithOptions_APIKeyOverridesProfileName(t *testing.T) { var gotAPIKey string var gotAuth string ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -209,10 +164,6 @@ func TestNewWithOptions_ProfileNameTakesPrecedenceOverExplicitAPIKey(t *testing. t.Fatal(err) } - // resolveClientOptions never produces this combo, but NewWithOptions is - // exported: when a caller passes an explicit APIKey alongside a ProfileName, - // the profile wins (routes through WithProfile) and the explicit key is not - // sent for SDK calls. c := NewWithOptions(Options{ APIKey: "explicit-key", ProfileName: "prod", @@ -221,11 +172,11 @@ func TestNewWithOptions_ProfileNameTakesPrecedenceOverExplicitAPIKey(t *testing. if _, err := c.SDK.Info.List(context.Background()); err != nil { t.Fatal(err) } - if gotAuth != "Bearer prod-access-token" { - t.Fatalf("expected profile OAuth bearer to win, got %q", gotAuth) + if gotAPIKey != "explicit-key" { + t.Fatalf("expected explicit API key, got %q", gotAPIKey) } - if gotAPIKey != "" { - t.Fatalf("expected explicit API key not sent for SDK calls, got %q", gotAPIKey) + if gotAuth != "" { + t.Fatalf("expected no profile bearer auth when API key wins, got %q", gotAuth) } } diff --git a/internal/cmd/evaluator.go b/internal/cmd/evaluator.go index b936e72..d48c1e5 100644 --- a/internal/cmd/evaluator.go +++ b/internal/cmd/evaluator.go @@ -20,9 +20,10 @@ func newEvaluatorCmd() *cobra.Command { Long: `Manage online and offline evaluator rules. Evaluators automatically score runs against a dataset (offline) or a project -(online). Two evaluator types are supported: +(online). Supported evaluator types: - upload Code evaluator from a Python or JavaScript/TypeScript file + upload Code evaluator from a Python or JavaScript/TypeScript file + create-llm LLM-as-judge evaluator (--model-config required; prompt via files or --hub-ref) Examples: langsmith evaluator list @@ -31,12 +32,14 @@ Examples: langsmith evaluator get accuracy --session-id langsmith evaluator upload eval.py --name accuracy --function check_accuracy --dataset my-eval-set langsmith evaluator upload eval.ts --name accuracy --function checkAccuracy --dataset my-eval-set + langsmith evaluator create-llm --name relevance --project my-app --prompt prompt.json --schema schema.json --model-config model.json langsmith evaluator delete accuracy --yes`, } cmd.AddCommand(newEvaluatorGetCmd()) cmd.AddCommand(newEvaluatorListCmd()) cmd.AddCommand(newEvaluatorUploadCmd()) + cmd.AddCommand(newEvaluatorCreateLLMCmd()) cmd.AddCommand(newEvaluatorDeleteCmd()) return cmd } @@ -196,6 +199,7 @@ func newEvaluatorUploadCmd() *cobra.Command { targetDataset string targetProject string samplingRate float64 + traceFilter string replace bool yes bool ) @@ -207,19 +211,15 @@ func newEvaluatorUploadCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { evaluatorFile := args[0] - if targetDataset == "" && targetProject == "" { - output.OutputJSON(map[string]any{ - "error": "Must specify --dataset or --project (global evaluators not supported)", - }, "") + if err := validateEvaluatorTargetFlags(targetDataset, targetProject); err != nil { + output.OutputJSON(map[string]any{"error": err.Error()}, "") return } c := MustGetClient() ctx := context.Background() - // Resolve targets var datasetID, projectID string - if targetDataset != "" { ds, err := resolveDataset(ctx, c, targetDataset) if err != nil { @@ -227,7 +227,6 @@ func newEvaluatorUploadCmd() *cobra.Command { } datasetID = ds.ID } - if targetProject != "" { sid, err := c.ResolveSessionID(ctx, targetProject) if err != nil { @@ -236,34 +235,6 @@ func newEvaluatorUploadCmd() *cobra.Command { projectID = sid } - // Check for existing evaluator - rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) - if err != nil { - ExitErrorf("checking existing evaluators: %v", err) - } - - existing := findEvaluator(*rules, name, datasetID, projectID) - if existing != nil { - if !replace { - output.OutputJSON(map[string]any{ - "error": fmt.Sprintf("Evaluator '%s' already exists (use --replace to overwrite)", name), - "id": existing.ID, - }, "") - return - } - if !yes { - fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) - var confirm string - _, _ = fmt.Scanln(&confirm) - if strings.ToLower(confirm) != "y" { - ExitError("aborted") - } - } - if err := c.RawDelete(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), nil); err != nil { - ExitErrorf("deleting existing evaluator: %v", err) - } - } - // Read and prepare function source source, err := os.ReadFile(evaluatorFile) if err != nil { @@ -292,7 +263,6 @@ func newEvaluatorUploadCmd() *cobra.Command { sourceStr = renameJSFunction(sourceStr, funcName) } - // Build payload payload := map[string]any{ "display_name": name, "sampling_rate": samplingRate, @@ -302,29 +272,60 @@ func newEvaluatorUploadCmd() *cobra.Command { {"code": sourceStr, "language": language}, }, } - if datasetID != "" { payload["dataset_id"] = datasetID } if projectID != "" { payload["session_id"] = projectID } + if traceFilter != "" { + payload["trace_filter"] = traceFilter + } + + // Prepare the new evaluator before touching any existing one. If this is + // a replacement, the old evaluator stays in place until the new version is ready. + rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) + if err != nil { + ExitErrorf("checking existing evaluators: %v", err) + } + + existing := findEvaluator(*rules, name, datasetID, projectID) + if existing != nil { + if !replace { + output.OutputJSON(map[string]any{ + "error": fmt.Sprintf("Evaluator '%s' already exists (use --replace to overwrite)", name), + "id": existing.ID, + }, "") + return + } + if !yes { + fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) + var confirm string + _, _ = fmt.Scanln(&confirm) + if strings.ToLower(confirm) != "y" { + ExitError("aborted") + } + } + } var result map[string]any - if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { + if existing != nil { + if err := c.RawPatch(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), payload, &result); err != nil { + ExitErrorf("replacing evaluator: %v", err) + } + } else if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { ExitErrorf("uploading evaluator: %v", err) } - target := "project" + targetLabel := "project" if datasetID != "" { - target = "dataset" + targetLabel = "dataset" } - output.OutputJSON(map[string]any{ "status": "uploaded", "id": result["id"], "name": name, - "target": target, + "target": targetLabel, }, "") }, } @@ -334,6 +335,7 @@ func newEvaluatorUploadCmd() *cobra.Command { cmd.Flags().StringVar(&targetDataset, "dataset", "", "Target dataset name (offline evaluator)") cmd.Flags().StringVar(&targetProject, "project", "", "Target project name (online evaluator)") cmd.Flags().Float64Var(&samplingRate, "sampling-rate", 1.0, "Fraction of runs to evaluate (0.0-1.0)") + cmd.Flags().StringVar(&traceFilter, "trace-filter", "", "Filter expression for which runs to evaluate") cmd.Flags().BoolVar(&replace, "replace", false, "Replace existing evaluator with same name") cmd.Flags().BoolVar(&yes, "yes", false, "Skip confirmation prompt when replacing") _ = cmd.MarkFlagRequired("name") @@ -342,6 +344,104 @@ func newEvaluatorUploadCmd() *cobra.Command { return cmd } +// Registers create-llm for LLM-as-judge evaluators. +func newEvaluatorCreateLLMCmd() *cobra.Command { + var ( + name string + targetDataset string + targetProject string + samplingRate float64 + traceFilter string + hubRef string + promptPath string + schemaPath string + modelConfigPath string + variableMapping string + replace bool + yes bool + ) + + cmd := &cobra.Command{ + Use: "create-llm", + Short: "Create an LLM-as-judge evaluator rule", + Long: `Create an LLM-as-judge run rule. --model-config is always required. + +Provide the judge prompt inline with --prompt and --schema, or point at Prompt Hub +with --hub-ref (which replaces --prompt and --schema). The model is configured separately. + +Examples: + langsmith evaluator create-llm --name relevance --project my-app \ + --prompt prompt.json --schema schema.json --model-config model.json + langsmith evaluator create-llm --name relevance --project my-app \ + --hub-ref my-org/relevance:latest --model-config model.json`, + Run: func(cmd *cobra.Command, args []string) { + c := MustGetClient() + ctx := context.Background() + + target, err := resolveLLMEvaluatorTarget(ctx, c, targetDataset, targetProject) + if err != nil { + ExitErrorf("%v", err) + } + mapping, err := parseVariableMapping(variableMapping) + if err != nil { + ExitErrorf("%v", err) + } + payload, err := buildLLMEvaluatorPayload( + name, target, samplingRate, traceFilter, hubRef, + promptPath, schemaPath, modelConfigPath, mapping, + ) + if err != nil { + ExitErrorf("%v", err) + } + existing, err := findLLMEvaluatorForCreate(ctx, c, name, target, replace, yes) + if err != nil { + if err.Error() == "aborted" { + ExitError("aborted") + } + if existing != nil { + output.OutputJSON(map[string]any{"error": err.Error(), "id": existing.ID}, "") + return + } + ExitErrorf("%v", err) + } + + var result map[string]any + if existing != nil { + if err := c.RawPatch(ctx, fmt.Sprintf("/runs/rules/%s", existing.ID), payload, &result); err != nil { + ExitErrorf("replacing LLM evaluator: %v", err) + } + } else if err := c.RawPost(ctx, "/runs/rules", payload, &result); err != nil { + ExitErrorf("creating LLM evaluator: %v", err) + } + targetLabel := "project" + if target.datasetID != "" { + targetLabel = "dataset" + } + output.OutputJSON(map[string]any{ + "status": "created", "type": "llm", + "id": result["id"], "name": name, "target": targetLabel, + }, "") + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Display name (required)") + cmd.Flags().StringVar(&targetDataset, "dataset", "", "Target dataset name") + cmd.Flags().StringVar(&targetProject, "project", "", "Target project name") + cmd.Flags().Float64Var(&samplingRate, "sampling-rate", 1.0, "Fraction of runs to evaluate (0.0-1.0)") + cmd.Flags().StringVar(&traceFilter, "trace-filter", "", "Filter expression for which runs to evaluate") + cmd.Flags().StringVar(&hubRef, "hub-ref", "", "Prompt Hub reference; replaces --prompt and --schema (e.g. my-org/prompt:latest)") + cmd.Flags().StringVar(&promptPath, "prompt", "", "Prompt JSON file ([[role,content],...] or [{role,content},...]); omit if --hub-ref is set") + cmd.Flags().StringVar(&schemaPath, "schema", "", "JSON schema file for structured output; omit if --hub-ref is set") + cmd.Flags().StringVar(&modelConfigPath, "model-config", "", "Serialized LangChain model JSON (required; copy from UI or GET /runs/rules)") + cmd.Flags().StringVar(&variableMapping, "variable-mapping", "", `Map prompt vars to trace paths (JSON or @file.json)`) + cmd.Flags().BoolVar(&replace, "replace", false, "Replace existing evaluator with same name") + cmd.Flags().BoolVar(&yes, "yes", false, "Skip confirmation when replacing") + _ = cmd.MarkFlagRequired("name") + _ = cmd.MarkFlagRequired("model-config") + + return cmd +} + func newEvaluatorDeleteCmd() *cobra.Command { var yes bool diff --git a/internal/cmd/evaluator_llm.go b/internal/cmd/evaluator_llm.go new file mode 100644 index 0000000..6b73443 --- /dev/null +++ b/internal/cmd/evaluator_llm.go @@ -0,0 +1,213 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/langchain-ai/langsmith-cli/internal/client" + langsmith "github.com/langchain-ai/langsmith-go" +) + +type llmEvaluatorTarget struct { + datasetID, projectID string +} + +func loadJSONFile(path string, dest any) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + if err := json.Unmarshal(data, dest); err != nil { + return fmt.Errorf("parsing JSON in %s: %w", path, err) + } + return nil +} + +// Accepts a plain schema file or one already saved from the LangSmith UI. +func loadEvaluatorSchema(path string) (map[string]any, error) { + var raw map[string]any + if err := loadJSONFile(path, &raw); err != nil { + return nil, err + } + if _, ok := raw["parameters"]; ok { + return raw, nil + } + return map[string]any{"name": "eval", "description": "", "parameters": raw}, nil +} + +func validatePromptMessagePair(i int, path string, msg []string) error { + if len(msg) != 2 || msg[0] == "" || msg[1] == "" { + return fmt.Errorf("prompt message %d in %s must be [role, content] with non-empty values", i, path) + } + return nil +} + +// Loads the judge's system/user messages from a prompt file. +func loadPromptMessagesFromJSON(path string, data []byte) ([][]string, error) { + var pairs [][]string + if err := json.Unmarshal(data, &pairs); err == nil && len(pairs) > 0 { + for i, msg := range pairs { + if err := validatePromptMessagePair(i, path, msg); err != nil { + return nil, err + } + } + return pairs, nil + } + var objects []struct { + Role, Content string + } + if err := json.Unmarshal(data, &objects); err != nil || len(objects) == 0 { + return nil, fmt.Errorf( + "prompt file %s: expected [[role,content],...] or [{role,content},...]", path, + ) + } + pairs = make([][]string, len(objects)) + for i, obj := range objects { + if err := validatePromptMessagePair(i, path, []string{obj.Role, obj.Content}); err != nil { + return nil, err + } + pairs[i] = []string{obj.Role, obj.Content} + } + return pairs, nil +} + +// Maps {{placeholders}} in the prompt to fields on each trace. +func parseVariableMapping(raw string) (map[string]string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } + if strings.HasPrefix(raw, "@") { + var mapping map[string]string + if err := loadJSONFile(raw[1:], &mapping); err != nil { + return nil, err + } + return mapping, nil + } + var mapping map[string]string + if err := json.Unmarshal([]byte(raw), &mapping); err != nil { + return nil, fmt.Errorf("parsing --variable-mapping: %w", err) + } + return mapping, nil +} + +func validateEvaluatorTargetFlags(dataset, project string) error { + if dataset == "" && project == "" { + return fmt.Errorf("must specify --dataset or --project (global evaluators not supported)") + } + if dataset != "" && project != "" { + return fmt.Errorf("cannot specify both --dataset and --project") + } + return nil +} + +// Finds the dataset or project this evaluator should run on. +func resolveLLMEvaluatorTarget(ctx context.Context, c *client.Client, dataset, project string) (llmEvaluatorTarget, error) { + if err := validateEvaluatorTargetFlags(dataset, project); err != nil { + return llmEvaluatorTarget{}, err + } + var target llmEvaluatorTarget + if dataset != "" { + ds, err := resolveDataset(ctx, c, dataset) + if err != nil { + return llmEvaluatorTarget{}, err + } + target.datasetID = ds.ID + } + if project != "" { + sid, err := c.ResolveSessionID(ctx, project) + if err != nil { + return llmEvaluatorTarget{}, err + } + target.projectID = sid + } + return target, nil +} + +// Finds an existing evaluator with the same name on this dataset or project. +// When the evaluator already exists and --replace is not set, returns the existing +// rule alongside the error so callers can reuse it without another list call. +func findLLMEvaluatorForCreate(ctx context.Context, c *client.Client, name string, target llmEvaluatorTarget, replace, yes bool) (*langsmith.Evaluator, error) { + rules, err := c.SDK.Evaluators.List(ctx, langsmith.EvaluatorListParams{}) + if err != nil { + return nil, fmt.Errorf("checking existing evaluators: %w", err) + } + existing := findEvaluator(*rules, name, target.datasetID, target.projectID) + if existing == nil { + return nil, nil + } + if !replace { + return existing, fmt.Errorf("evaluator %q already exists (use --replace to overwrite)", name) + } + if !yes { + fmt.Fprintf(os.Stderr, "Replace existing evaluator '%s'? [y/N] ", name) + var confirm string + _, _ = fmt.Scanln(&confirm) + if strings.ToLower(confirm) != "y" { + return nil, fmt.Errorf("aborted") + } + } + return existing, nil +} + +// Packages prompt, schema, model, and targeting into the create-evaluator request. +func buildLLMEvaluatorPayload( + name string, + target llmEvaluatorTarget, + samplingRate float64, + traceFilter, hubRef, promptPath, schemaPath, modelConfigPath string, + variableMapping map[string]string, +) (map[string]any, error) { + if modelConfigPath == "" { + return nil, fmt.Errorf("--model-config is required") + } + var model map[string]any + if err := loadJSONFile(modelConfigPath, &model); err != nil { + return nil, err + } + + structured := map[string]any{"model": model} + if hubRef != "" { + structured["hub_ref"] = hubRef + } else { + if promptPath == "" || schemaPath == "" { + return nil, fmt.Errorf("--prompt and --schema are required unless --hub-ref is set") + } + data, err := os.ReadFile(promptPath) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", promptPath, err) + } + messages, err := loadPromptMessagesFromJSON(promptPath, data) + if err != nil { + return nil, err + } + schema, err := loadEvaluatorSchema(schemaPath) + if err != nil { + return nil, err + } + structured["prompt"] = messages + structured["schema"] = schema + structured["template_format"] = "mustache" + } + if len(variableMapping) > 0 { + structured["variable_mapping"] = variableMapping + } + payload := map[string]any{ + "display_name": name, "sampling_rate": samplingRate, "is_enabled": true, + "include_extended_stats": false, + "evaluators": []map[string]any{{"structured": structured}}, + } + if traceFilter != "" { + payload["trace_filter"] = traceFilter + } + if target.datasetID != "" { + payload["dataset_id"] = target.datasetID + } + if target.projectID != "" { + payload["session_id"] = target.projectID + } + return payload, nil +} diff --git a/internal/cmd/evaluator_test.go b/internal/cmd/evaluator_test.go index 4936821..ae5c482 100644 --- a/internal/cmd/evaluator_test.go +++ b/internal/cmd/evaluator_test.go @@ -3,6 +3,7 @@ package cmd import ( "encoding/json" "net/http" + "os" "strings" "testing" @@ -366,11 +367,100 @@ func TestFindEvaluator_NameMatchButNoTarget(t *testing.T) { } } +func TestValidateEvaluatorTargetFlags(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + dataset string + project string + wantErr string + }{ + {name: "requires one target", wantErr: "must specify"}, + {name: "rejects both targets", dataset: "ds", project: "proj", wantErr: "cannot specify both"}, + {name: "dataset only", dataset: "ds"}, + {name: "project only", project: "proj"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := validateEvaluatorTargetFlags(tt.dataset, tt.project) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected %q error, got %v", tt.wantErr, err) + } + }) + } +} + +func TestLoadPromptMessagesFromJSON(t *testing.T) { + t.Parallel() + + t.Run("pair format", func(t *testing.T) { + t.Parallel() + msgs, err := loadPromptMessagesFromJSON("prompt.json", []byte( + `[["system","You are a judge."],["user","Q: {{input}}"]]`, + )) + if err != nil { + t.Fatal(err) + } + if len(msgs) != 2 || msgs[0][0] != "system" { + t.Fatalf("unexpected messages: %#v", msgs) + } + }) + + t.Run("role content objects", func(t *testing.T) { + t.Parallel() + msgs, err := loadPromptMessagesFromJSON("prompt.json", []byte( + `[{"role":"system","content":"You are a judge."},{"role":"user","content":"Q: {{input}}"}]`, + )) + if err != nil { + t.Fatal(err) + } + if len(msgs) != 2 || msgs[1][1] != "Q: {{input}}" { + t.Fatalf("unexpected messages: %#v", msgs) + } + }) + + t.Run("invalid format", func(t *testing.T) { + t.Parallel() + _, err := loadPromptMessagesFromJSON("prompt.json", []byte(`{"messages":[]}`)) + if err == nil || !strings.Contains(err.Error(), "expected [[role,content]") { + t.Fatalf("expected format hint, got %v", err) + } + }) + + t.Run("rejects single-element pair", func(t *testing.T) { + t.Parallel() + _, err := loadPromptMessagesFromJSON("prompt.json", []byte(`[["system"]]`)) + if err == nil || !strings.Contains(err.Error(), "must be [role, content]") { + t.Fatalf("expected length error, got %v", err) + } + }) +} + +func TestBuildLLMEvaluatorPayload_requiresModelConfig(t *testing.T) { + t.Parallel() + + _, err := buildLLMEvaluatorPayload( + "relevance", llmEvaluatorTarget{projectID: "proj-1"}, + 1.0, "", "", "prompt.json", "schema.json", "", nil, + ) + if err == nil || !strings.Contains(err.Error(), "--model-config is required") { + t.Fatalf("expected model-config error, got %v", err) + } +} + // ==================== Command structure tests ==================== func TestEvaluatorCmd_Subcommands(t *testing.T) { cmd := newEvaluatorCmd() - expected := map[string]bool{"get": false, "list": false, "upload": false, "delete": false} + expected := map[string]bool{"get": false, "list": false, "upload": false, "create-llm": false, "delete": false} for _, sub := range cmd.Commands() { if _, ok := expected[sub.Name()]; ok { expected[sub.Name()] = true @@ -444,6 +534,7 @@ func TestEvaluatorUploadCmd_Flags(t *testing.T) { "dataset": "", "project": "", "sampling-rate": "1", + "trace-filter": "", "replace": "false", "yes": "false", } @@ -639,6 +730,194 @@ func TestEvaluatorListCmd_VerifiesAPIKeyHeader(t *testing.T) { } } +func TestEvaluatorUploadReplacePatchesExistingCodeEvaluator(t *testing.T) { + evaluatorFile := t.TempDir() + "/eval.py" + if err := os.WriteFile( + evaluatorFile, + []byte("def check_accuracy(run, example):\n return {\"score\": 1}\n"), + 0o600, + ); err != nil { + t.Fatal(err) + } + + var sawDelete bool + var patchBody map[string]any + ts := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/api/v1/sessions" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]map[string]any{ + {"id": "project-1", "name": "my-project"}, + }) + case r.URL.Path == "/api/v1/runs/rules" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]testRule{ + { + ID: "existing-rule", + DisplayName: "accuracy", + SamplingRate: 0.25, + IsEnabled: true, + SessionID: "project-1", + CodeEvaluators: []testCodeEval{ + {Code: "def perform_eval(run, example):\n return {\"score\": 0}", Language: "python"}, + }, + }, + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "PATCH": + if err := json.NewDecoder(r.Body).Decode(&patchBody); err != nil { + t.Fatalf("decoding patch body: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "existing-rule", + "display_name": "accuracy", + "session_id": "project-1", + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "DELETE": + sawDelete = true + http.Error(w, "delete should not be called", http.StatusInternalServerError) + default: + http.Error(w, "not found", http.StatusNotFound) + } + }) + + cleanup := setupTestEnv(t, ts.URL) + defer cleanup() + flagOutputFormat = "json" + + out := captureStdout(t, func() { + cmd := newEvaluatorUploadCmd() + _ = cmd.Flags().Set("name", "accuracy") + _ = cmd.Flags().Set("function", "check_accuracy") + _ = cmd.Flags().Set("project", "my-project") + _ = cmd.Flags().Set("sampling-rate", "0.5") + _ = cmd.Flags().Set("replace", "true") + _ = cmd.Flags().Set("yes", "true") + cmd.Run(cmd, []string{evaluatorFile}) + }) + + if sawDelete { + t.Fatal("upload --replace should patch the existing evaluator, not delete it") + } + if patchBody == nil { + t.Fatal("expected PATCH body") + } + if patchBody["display_name"] != "accuracy" { + t.Errorf("expected display_name=accuracy, got %v", patchBody["display_name"]) + } + if patchBody["session_id"] != "project-1" { + t.Errorf("expected session_id=project-1, got %v", patchBody["session_id"]) + } + if patchBody["sampling_rate"] != 0.5 { + t.Errorf("expected sampling_rate=0.5, got %v", patchBody["sampling_rate"]) + } + evaluators, ok := patchBody["code_evaluators"].([]any) + if !ok || len(evaluators) != 1 { + t.Fatalf("expected one code evaluator, got %#v", patchBody["code_evaluators"]) + } + codeEvaluator, ok := evaluators[0].(map[string]any) + if !ok { + t.Fatalf("expected code evaluator object, got %#v", evaluators[0]) + } + if codeEvaluator["language"] != "python" { + t.Errorf("expected language=python, got %v", codeEvaluator["language"]) + } + code, _ := codeEvaluator["code"].(string) + if !strings.Contains(code, "def perform_eval(") { + t.Errorf("expected uploaded code to be renamed to perform_eval, got:\n%s", code) + } + + var result map[string]any + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("failed to parse output JSON: %v\noutput: %s", err, out) + } + if result["id"] != "existing-rule" { + t.Errorf("expected output id=existing-rule, got %v", result["id"]) + } +} + +func TestEvaluatorCreateLLMReplacePatchesExistingEvaluator(t *testing.T) { + modelConfigPath := t.TempDir() + "/model.json" + if err := os.WriteFile( + modelConfigPath, + []byte(`{"type":"chat","config":{"model":"test-model"}}`), + 0o600, + ); err != nil { + t.Fatal(err) + } + + var sawDelete bool + var patchBody map[string]any + ts := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/api/v1/sessions" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]map[string]any{ + {"id": "project-1", "name": "my-project"}, + }) + case r.URL.Path == "/api/v1/runs/rules" && r.Method == "GET": + _ = json.NewEncoder(w).Encode([]testRule{ + { + ID: "existing-rule", + DisplayName: "relevance", + SamplingRate: 0.25, + IsEnabled: true, + SessionID: "project-1", + Evaluators: []testLLMEval{ + {Structured: testLLMStructured{HubRef: "my-org/old:latest"}}, + }, + }, + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "PATCH": + if err := json.NewDecoder(r.Body).Decode(&patchBody); err != nil { + t.Fatalf("decoding patch body: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "existing-rule", + "display_name": "relevance", + "session_id": "project-1", + }) + case r.URL.Path == "/runs/rules/existing-rule" && r.Method == "DELETE": + sawDelete = true + http.Error(w, "delete should not be called", http.StatusInternalServerError) + default: + http.Error(w, "not found", http.StatusNotFound) + } + }) + + cleanup := setupTestEnv(t, ts.URL) + defer cleanup() + flagOutputFormat = "json" + + out := captureStdout(t, func() { + cmd := newEvaluatorCreateLLMCmd() + _ = cmd.Flags().Set("name", "relevance") + _ = cmd.Flags().Set("project", "my-project") + _ = cmd.Flags().Set("hub-ref", "my-org/relevance:latest") + _ = cmd.Flags().Set("model-config", modelConfigPath) + _ = cmd.Flags().Set("sampling-rate", "0.5") + _ = cmd.Flags().Set("replace", "true") + _ = cmd.Flags().Set("yes", "true") + cmd.Run(cmd, nil) + }) + + if sawDelete { + t.Fatal("create-llm --replace should patch the existing evaluator, not delete it") + } + if patchBody == nil { + t.Fatal("expected PATCH body") + } + if patchBody["display_name"] != "relevance" || patchBody["evaluators"] == nil { + t.Fatalf("expected LLM evaluator replacement payload, got %#v", patchBody) + } + + var result map[string]any + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("failed to parse output JSON: %v\noutput: %s", err, out) + } + if result["id"] != "existing-rule" { + t.Errorf("expected output id=existing-rule, got %v", result["id"]) + } +} + // ==================== evaluator get ==================== func TestEvaluatorGetCmd_UseField(t *testing.T) { diff --git a/internal/cmd/helpers.go b/internal/cmd/helpers.go index 282872d..1089e79 100644 --- a/internal/cmd/helpers.go +++ b/internal/cmd/helpers.go @@ -183,14 +183,12 @@ func buildRunSelectV2(includeIO, includeFeedback bool) []langsmith.RunQueryV2Par langsmith.RunQueryV2ParamsSelectTotalCost, langsmith.RunQueryV2ParamsSelectLatencySeconds, langsmith.RunQueryV2ParamsSelectAppPath, - langsmith.RunQueryV2ParamsSelectFirstTokenTime, } if includeIO { fields = append(fields, langsmith.RunQueryV2ParamsSelectInputs, langsmith.RunQueryV2ParamsSelectOutputs, langsmith.RunQueryV2ParamsSelectError, - langsmith.RunQueryV2ParamsSelectEvents, ) } if includeFeedback { @@ -201,7 +199,7 @@ func buildRunSelectV2(includeIO, includeFeedback bool) []langsmith.RunQueryV2Par // runV2ToSchema converts a v2 Run into the legacy v1 RunSchema shape so the // existing extract/output pipeline can consume it unchanged. -func runV2ToSchema(r langsmith.Run) langsmith.RunSchema { +func runV2ToSchema(r langsmith.QueryRunResponse) langsmith.RunSchema { extra := asMap(r.Extra) if md := asMap(r.Metadata); md != nil { if extra == nil { @@ -242,7 +240,6 @@ func runV2ToSchema(r langsmith.Run) langsmith.RunSchema { Outputs: asMap(r.Outputs), Extra: extra, FeedbackStats: convertFeedbackStats(r.FeedbackStats), - Events: convertEvents(r.Events), } if len(r.ParentRunIDs) > 0 { out.ParentRunIDs = r.ParentRunIDs @@ -263,7 +260,7 @@ func asMap(v interface{}) map[string]interface{} { // convertFeedbackStats round-trips the generated feedback-stats shape into the // loosely-typed map the downstream pipeline expects. Returns nil on empty input // or any marshalling error. -func convertFeedbackStats(stats map[string]langsmith.RunFeedbackStat) map[string]map[string]interface{} { +func convertFeedbackStats(stats map[string]langsmith.QueryRunResponseFeedbackStat) map[string]map[string]interface{} { if len(stats) == 0 { return nil } @@ -278,24 +275,6 @@ func convertFeedbackStats(stats map[string]langsmith.RunFeedbackStat) map[string return m } -// convertEvents round-trips the generated v2 event shape into the -// loosely-typed maps the legacy RunSchema.Events field expects. Returns nil on -// empty input or any marshalling error. -func convertEvents(events []langsmith.RunEvent) []map[string]interface{} { - if len(events) == 0 { - return nil - } - raw, err := json.Marshal(events) - if err != nil { - return nil - } - var m []map[string]interface{} - if err := json.Unmarshal(raw, &m); err != nil { - return nil - } - return m -} - // buildRunSelect returns the Select fields needed for the given include flags. // Returns nil when neither IO nor feedback is requested, letting the API use its defaults. // When set, includes all base/metadata fields so they aren't stripped from the response. @@ -316,7 +295,6 @@ func buildRunSelect(includeIO, includeFeedback bool) []langsmith.RunQueryParamsS langsmith.RunQueryParamsSelectStatus, // Metadata fields langsmith.RunQueryParamsSelectExtra, - langsmith.RunQueryParamsSelectFirstTokenTime, langsmith.RunQueryParamsSelectPromptTokens, langsmith.RunQueryParamsSelectCompletionTokens, langsmith.RunQueryParamsSelectTotalTokens, @@ -331,7 +309,6 @@ func buildRunSelect(includeIO, includeFeedback bool) []langsmith.RunQueryParamsS langsmith.RunQueryParamsSelectInputs, langsmith.RunQueryParamsSelectOutputs, langsmith.RunQueryParamsSelectError, - langsmith.RunQueryParamsSelectEvents, ) } diff --git a/internal/cmd/helpers_test.go b/internal/cmd/helpers_test.go index 3744954..0cc0bf0 100644 --- a/internal/cmd/helpers_test.go +++ b/internal/cmd/helpers_test.go @@ -404,19 +404,6 @@ func TestBuildRunSelect_IncludesMetadataFields(t *testing.T) { } } -func TestBuildRunSelect_IncludesFirstTokenTimeAndEvents(t *testing.T) { - // first_token_time and events are native Run fields that --full is - // documented to return; a prior version of this select list silently - // dropped both even when --include-io/--include-metadata were requested. - has := selectSet(buildRunSelect(true, true)) - if !has[langsmith.RunQueryParamsSelectFirstTokenTime] { - t.Error("missing first_token_time field") - } - if !has[langsmith.RunQueryParamsSelectEvents] { - t.Error("missing events field (should be requested alongside inputs/outputs/error)") - } -} - // selectSet converts a slice to a set for easy lookup. func selectSet(sel []langsmith.RunQueryParamsSelect) map[langsmith.RunQueryParamsSelect]bool { m := make(map[langsmith.RunQueryParamsSelect]bool, len(sel)) @@ -497,56 +484,3 @@ func TestRunTreeData_AllFields(t *testing.T) { t.Error("unexpected RunTreeData field values") } } - -// ---------- buildRunSelectV2 ---------- - -func TestBuildRunSelectV2_IncludesFirstTokenTimeAndEvents(t *testing.T) { - // Mirrors TestBuildRunSelect_IncludesFirstTokenTimeAndEvents for the v2 - // select builder: first_token_time is a base field always requested, - // events is requested alongside inputs/outputs/error under --include-io. - base := buildRunSelectV2(false, false) - baseHas := map[langsmith.RunQueryV2ParamsSelect]bool{} - for _, f := range base { - baseHas[f] = true - } - if !baseHas[langsmith.RunQueryV2ParamsSelectFirstTokenTime] { - t.Error("missing first_token_time field in base v2 select set") - } - if baseHas[langsmith.RunQueryV2ParamsSelectEvents] { - t.Error("events should not be requested without --include-io") - } - - withIO := buildRunSelectV2(true, false) - ioHas := map[langsmith.RunQueryV2ParamsSelect]bool{} - for _, f := range withIO { - ioHas[f] = true - } - if !ioHas[langsmith.RunQueryV2ParamsSelectEvents] { - t.Error("missing events field when --include-io requested") - } -} - -// ---------- runV2ToSchema ---------- - -func TestRunV2ToSchema_MapsFirstTokenTimeAndEvents(t *testing.T) { - firstToken := time.Date(2026, 1, 15, 10, 30, 1, 0, time.UTC) - v2 := langsmith.Run{ - ID: "run-123", - FirstTokenTime: firstToken, - Events: []langsmith.RunEvent{ - {Name: "new_token", Kwargs: map[string]interface{}{"token": "hi"}}, - }, - } - - out := runV2ToSchema(v2) - - if !out.FirstTokenTime.Equal(firstToken) { - t.Errorf("expected FirstTokenTime=%v, got %v", firstToken, out.FirstTokenTime) - } - if len(out.Events) != 1 { - t.Fatalf("expected 1 event, got %d", len(out.Events)) - } - if out.Events[0]["name"] != "new_token" { - t.Errorf("expected event name=new_token, got %v", out.Events[0]["name"]) - } -} diff --git a/internal/cmd/install_method.go b/internal/cmd/install_method.go deleted file mode 100644 index 987aa39..0000000 --- a/internal/cmd/install_method.go +++ /dev/null @@ -1,101 +0,0 @@ -package cmd - -import ( - "path/filepath" - "strings" -) - -// installMethod identifies how the langsmith binary was installed, which -// determines whether self-update may replace the binary in place. -type installMethod int - -const ( - // methodManaged means self-update owns the binary: install.sh, install.ps1, - // or a manual download from GitHub Releases. In-place replacement is correct. - methodManaged installMethod = iota - // methodHomebrew means the binary lives in a Homebrew Cellar. - methodHomebrew - // methodScoop means the binary was installed by Scoop. - methodScoop - // methodGo means the binary was installed via `go install`. - methodGo - // methodDev means a local/development build with no release version. - methodDev -) - -// externalManagers describes installs owned by a package manager: how to name -// the manager in output and the command the user should run to update. Only the -// methods self-update must defer to appear here — methodManaged and methodDev are -// handled by self-update itself and intentionally have no entry, so membership in -// this map is the single source of truth for "must defer to a package manager". -var externalManagers = map[installMethod]struct { - label string // machine-readable name, for JSON output - display string // human-readable name, for the pretty message - command string // the command the user should run to update -}{ - methodHomebrew: {"homebrew", "Homebrew", "brew upgrade langchain-ai/tap/langsmith-cli"}, - methodScoop: {"scoop", "Scoop", "scoop update langsmith-cli"}, - methodGo: {"go", "`go install`", "go install github.com/langchain-ai/langsmith-cli/cmd/langsmith@latest"}, -} - -// detectInstallMethod classifies the install method from the resolved executable -// path, the build version, and the relevant environment values. It is pure: all -// I/O (resolving the path, reading env) happens in the caller. -// -// - execPath must already be resolved through filepath.EvalSymlinks so that -// Homebrew's bin symlinks point at the Cellar. -// - goos is runtime.GOOS. -// - gobin/gopath/home come from $GOBIN, $GOPATH, and the user's home dir. -func detectInstallMethod(execPath, version, goos, gobin, gopath, home string) installMethod { - // Normalize both separators to "/" so a single "/segment/" check works for - // Windows paths regardless of the OS this code runs on (matters for tests). - lower := strings.ToLower(strings.ReplaceAll(execPath, `\`, "/")) - - if strings.Contains(lower, "/cellar/") { - return methodHomebrew - } - if strings.Contains(lower, "/scoop/") { - return methodScoop - } - - dir := filepath.Dir(execPath) - for _, bin := range goBinDirs(gobin, gopath, home) { - if bin != "" && filepath.Clean(dir) == filepath.Clean(bin) { - return methodGo - } - } - - if version == "dev" { - return methodDev - } - return methodManaged -} - -// goBinDirs returns the candidate directories where `go install` places binaries, -// in precedence order: $GOBIN, then $GOPATH/bin, then $HOME/go/bin (Go's default -// GOPATH when unset). -func goBinDirs(gobin, gopath, home string) []string { - if gobin != "" { - return []string{gobin} - } - if gopath != "" { - // GOPATH may contain multiple entries; go install uses the first. - first := filepath.SplitList(gopath) - if len(first) > 0 && first[0] != "" { - return []string{filepath.Join(first[0], "bin")} - } - } - if home != "" { - return []string{filepath.Join(home, "go", "bin")} - } - return nil -} - -// shouldDeferToManager reports whether self-update must defer to a package -// manager instead of replacing the binary in place. --force overrides it. -// Membership in externalManagers is the source of truth; methodManaged and -// methodDev are not externally managed and so are never deferred. -func shouldDeferToManager(method installMethod, force bool) bool { - _, external := externalManagers[method] - return external && !force -} diff --git a/internal/cmd/install_method_test.go b/internal/cmd/install_method_test.go deleted file mode 100644 index 78aa832..0000000 --- a/internal/cmd/install_method_test.go +++ /dev/null @@ -1,233 +0,0 @@ -package cmd - -import ( - "encoding/json" - "strings" - "testing" -) - -func TestDetectInstallMethod(t *testing.T) { - tests := []struct { - name string - execPath string - version string - goos string - gobin string - gopath string - home string - want installMethod - }{ - { - name: "homebrew apple silicon cellar", - execPath: "/opt/homebrew/Cellar/langsmith-cli/0.2.38/bin/langsmith", - version: "0.2.38", - goos: "darwin", - home: "/Users/me", - want: methodHomebrew, - }, - { - name: "homebrew intel cellar", - execPath: "/usr/local/Cellar/langsmith-cli/0.2.38/bin/langsmith", - version: "0.2.38", - goos: "darwin", - home: "/Users/me", - want: methodHomebrew, - }, - { - name: "homebrew linux cellar", - execPath: "/home/linuxbrew/.linuxbrew/Cellar/langsmith-cli/0.2.38/bin/langsmith", - version: "0.2.38", - goos: "linux", - home: "/home/me", - want: methodHomebrew, - }, - { - name: "scoop apps", - execPath: `C:\Users\me\scoop\apps\langsmith-cli\current\langsmith.exe`, - version: "0.2.38", - goos: "windows", - home: `C:\Users\me`, - want: methodScoop, - }, - { - name: "scoop shims", - execPath: `C:\Users\me\scoop\shims\langsmith.exe`, - version: "0.2.38", - goos: "windows", - home: `C:\Users\me`, - want: methodScoop, - }, - { - name: "go install via GOBIN", - execPath: "/Users/me/dev/gobin/langsmith", - version: "0.2.38", - goos: "darwin", - gobin: "/Users/me/dev/gobin", - home: "/Users/me", - want: methodGo, - }, - { - name: "go install via GOPATH bin", - execPath: "/Users/me/gopath/bin/langsmith", - version: "0.2.38", - goos: "darwin", - gopath: "/Users/me/gopath", - home: "/Users/me", - want: methodGo, - }, - { - name: "go install via default HOME/go/bin", - execPath: "/Users/me/go/bin/langsmith", - version: "0.2.38", - goos: "darwin", - home: "/Users/me", - want: methodGo, - }, - { - name: "go install with dev version still detected as go", - execPath: "/Users/me/go/bin/langsmith", - version: "dev", - goos: "darwin", - home: "/Users/me", - want: methodGo, - }, - { - name: "managed install.sh usr local bin", - execPath: "/usr/local/bin/langsmith", - version: "0.2.38", - goos: "darwin", - home: "/Users/me", - want: methodManaged, - }, - { - name: "managed install.sh local bin", - execPath: "/Users/me/.local/bin/langsmith", - version: "0.2.38", - goos: "linux", - home: "/Users/me", - want: methodManaged, - }, - { - name: "dev build outside package dirs", - execPath: "/Users/me/repos/langsmith-cli/bin/langsmith", - version: "dev", - goos: "darwin", - home: "/Users/me", - want: methodDev, - }, - { - name: "GOBIN takes precedence over GOPATH", - execPath: "/Users/me/gopath/bin/langsmith", - version: "0.2.38", - goos: "darwin", - gobin: "/Users/me/gobin", - gopath: "/Users/me/gopath", - home: "/Users/me", - // exec is in GOPATH/bin, but GOBIN is set and differs → not the go bin dir - want: methodManaged, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := detectInstallMethod(tt.execPath, tt.version, tt.goos, tt.gobin, tt.gopath, tt.home) - if got != tt.want { - t.Errorf("detectInstallMethod() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestExternalManagers(t *testing.T) { - want := map[installMethod]struct{ label, command string }{ - methodHomebrew: {"homebrew", "brew upgrade langchain-ai/tap/langsmith-cli"}, - methodScoop: {"scoop", "scoop update langsmith-cli"}, - methodGo: {"go", "go install github.com/langchain-ai/langsmith-cli/cmd/langsmith@latest"}, - } - for method, exp := range want { - mgr, ok := externalManagers[method] - if !ok { - t.Errorf("externalManagers missing entry for %v", method) - continue - } - if mgr.label != exp.label { - t.Errorf("externalManagers[%v].label = %q, want %q", method, mgr.label, exp.label) - } - if mgr.command != exp.command { - t.Errorf("externalManagers[%v].command = %q, want %q", method, mgr.command, exp.command) - } - if mgr.display == "" { - t.Errorf("externalManagers[%v].display is empty", method) - } - } - // managed and dev are handled by self-update itself and must not be listed. - for _, method := range []installMethod{methodManaged, methodDev} { - if _, ok := externalManagers[method]; ok { - t.Errorf("externalManagers should not contain %v", method) - } - } -} - -func TestShouldDeferToManager(t *testing.T) { - tests := []struct { - method installMethod - force bool - want bool - }{ - {methodHomebrew, false, true}, - {methodHomebrew, true, false}, // --force overrides the defer - {methodScoop, false, true}, - {methodScoop, true, false}, - {methodGo, false, true}, - {methodGo, true, false}, - {methodManaged, false, false}, - {methodManaged, true, false}, - {methodDev, false, false}, - {methodDev, true, false}, - } - for _, tt := range tests { - if got := shouldDeferToManager(tt.method, tt.force); got != tt.want { - t.Errorf("shouldDeferToManager(%v, force=%v) = %v, want %v", tt.method, tt.force, got, tt.want) - } - } -} - -func TestReportManagedExternally_JSON(t *testing.T) { - oldFmt := flagOutputFormat - flagOutputFormat = "json" - defer func() { flagOutputFormat = oldFmt }() - - output := captureStdout(t, func() { - if err := reportManagedExternally(methodHomebrew, "json"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - }) - - var result map[string]string - if err := json.Unmarshal([]byte(output), &result); err != nil { - t.Fatalf("invalid JSON output: %v, output: %q", err, output) - } - if result["status"] != "managed-externally" { - t.Errorf("expected status managed-externally, got %q", result["status"]) - } - if result["install_method"] != "homebrew" { - t.Errorf("expected install_method homebrew, got %q", result["install_method"]) - } - if result["update_command"] != "brew upgrade langchain-ai/tap/langsmith-cli" { - t.Errorf("unexpected update_command: %q", result["update_command"]) - } -} - -func TestReportManagedExternally_Pretty(t *testing.T) { - output := captureStdout(t, func() { - if err := reportManagedExternally(methodScoop, "pretty"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - }) - if !strings.Contains(output, "scoop update langsmith-cli") { - t.Errorf("expected pretty output to contain the scoop command, got %q", output) - } - if !strings.Contains(output, "--force") { - t.Errorf("expected pretty output to mention --force, got %q", output) - } -} diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 9c8992b..30f514c 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -198,10 +198,6 @@ func resolveClientOptions(refreshOAuth bool) (client.Options, error) { opts.OAuthAccessToken = profile.AccessToken() case hasProfile && profile.APIKey != "": opts.APIKey = profile.APIKey - // Route the resolved profile through the SDK (WithProfile) so an explicit - // selection replaces the config's current_profile instead of inheriting - // its tenant/base URL. APIKey is kept for the raw-HTTP helpers. - opts.ProfileName = profileName } if cfgErr != nil && opts.APIKey == "" { return opts, cfgErr diff --git a/internal/cmd/root_test.go b/internal/cmd/root_test.go index d3734c4..0e8431e 100644 --- a/internal/cmd/root_test.go +++ b/internal/cmd/root_test.go @@ -313,61 +313,6 @@ func TestResolveClientOptionsRefreshesProfileWithoutAccessToken(t *testing.T) { } } -func TestResolveClientOptions_SetsProfileNameForAPIKeyProfile(t *testing.T) { - oldKey := flagAPIKey - oldURL := flagAPIURL - oldProfile := flagProfile - oldWS := flagWorkspaceID - defer func() { - flagAPIKey = oldKey - flagAPIURL = oldURL - flagProfile = oldProfile - flagWorkspaceID = oldWS - }() - flagAPIKey = "" - flagAPIURL = "" - flagWorkspaceID = "" - flagProfile = "aws" - - path := filepath.Join(t.TempDir(), "config.json") - t.Setenv("LANGSMITH_CONFIG_FILE", path) - t.Setenv("LANGSMITH_API_KEY", "") - t.Setenv("LANGSMITH_ENDPOINT", "") - t.Setenv("LANGSMITH_WORKSPACE_ID", "") - t.Setenv("LANGSMITH_TENANT_ID", "") - t.Setenv("LANGSMITH_PROFILE", "") - if err := os.WriteFile(path, []byte(`{ - "current_profile": "prod", - "profiles": { - "prod": { - "api_key": "prod-api-key", - "workspace_id": "prod-workspace-id" - }, - "aws": { - "api_key": "aws-api-key" - } - } -} -`), 0600); err != nil { - t.Fatal(err) - } - - opts, err := resolveClientOptions(false) - if err != nil { - t.Fatalf("resolveClientOptions returned error: %v", err) - } - if opts.APIKey != "aws-api-key" { - t.Fatalf("expected profile api key aws-api-key, got %q", opts.APIKey) - } - // The resolved profile must reach the SDK via WithProfile so an explicit - // --profile replaces current_profile and clears its inherited tenant. Without - // ProfileName set, NewWithOptions passes only WithAPIKey and current_profile's - // tenant leaks (403 cross-workspace). - if opts.ProfileName != "aws" { - t.Fatalf("expected ProfileName=aws, got %q", opts.ProfileName) - } -} - func TestGetOAuthAccessToken_ProfileFallback(t *testing.T) { oldKey := flagAPIKey oldURL := flagAPIURL diff --git a/internal/cmd/run.go b/internal/cmd/run.go index 5b5d358..a3aea47 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -95,8 +95,8 @@ func newRunListCmd() *cobra.Command { addCommonFilterFlags(cmd, &ff, true) addVersionFlag(cmd, &ff) - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") @@ -179,8 +179,8 @@ func newRunGetCmd() *cobra.Command { cmd.Flags().StringVar(&since, "since", "", "Only include runs after this timestamp, e.g. 2024-01-15T00:00:00Z (overrides 7-day default)") cmd.Flags().IntVar(&lastNMinutes, "last-n-minutes", 0, "Only include runs from the last N minutes, e.g. 60 (overrides 7-day default)") cmd.Flags().StringVar(&version, "version", "", `Query API version: "" (v1, default) or "v2" (SmithDB)`) - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") @@ -244,8 +244,8 @@ func newRunExportCmd() *cobra.Command { addCommonFilterFlags(cmd, &ff, true) addVersionFlag(cmd, &ff) - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") diff --git a/internal/cmd/sandbox_box.go b/internal/cmd/sandbox_box.go index 049cec3..3738387 100644 --- a/internal/cmd/sandbox_box.go +++ b/internal/cmd/sandbox_box.go @@ -16,7 +16,7 @@ import ( const defaultBoxPollInterval = 2 * time.Second -func waitForBoxReady(ctx context.Context, c *client.Client, name string) (*langsmith.SandboxStatusResponse, error) { +func waitForBoxReady(ctx context.Context, c *client.Client, name string) (*langsmith.SandboxBoxGetStatusResponse, error) { for { resp, err := c.SDK.Sandboxes.Boxes.GetStatus(ctx, name) if err != nil { diff --git a/internal/cmd/sandbox_test.go b/internal/cmd/sandbox_test.go index 1c7f3d3..30030d3 100644 --- a/internal/cmd/sandbox_test.go +++ b/internal/cmd/sandbox_test.go @@ -288,7 +288,7 @@ func TestSandboxBoxDetailRenderSupportsSDKResponseTypes(t *testing.T) { }{ { name: "new response", - model: langsmith.SandboxResponse{ + model: langsmith.SandboxBoxNewResponse{ ID: "box-new", Name: "new-vm", Status: "running", @@ -305,7 +305,7 @@ func TestSandboxBoxDetailRenderSupportsSDKResponseTypes(t *testing.T) { }, { name: "get response", - model: langsmith.SandboxResponse{ + model: langsmith.SandboxBoxGetResponse{ ID: "box-get", Name: "get-vm", Status: "running", @@ -322,7 +322,7 @@ func TestSandboxBoxDetailRenderSupportsSDKResponseTypes(t *testing.T) { }, { name: "update response", - model: langsmith.SandboxResponse{ + model: langsmith.SandboxBoxUpdateResponse{ ID: "box-update", Name: "update-vm", Status: "stopped", diff --git a/internal/cmd/thread.go b/internal/cmd/thread.go index 92ab76c..57d4eef 100644 --- a/internal/cmd/thread.go +++ b/internal/cmd/thread.go @@ -267,8 +267,8 @@ func newThreadGetCmd() *cobra.Command { } cmd.Flags().StringVar(&project, "project", "", "Project name [env: LANGSMITH_PROJECT]") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().IntVarP(&limit, "limit", "n", 0, "Maximum number of runs (turns) to return") diff --git a/internal/cmd/trace.go b/internal/cmd/trace.go index ef78232..bc001d2 100644 --- a/internal/cmd/trace.go +++ b/internal/cmd/trace.go @@ -144,8 +144,8 @@ func newTraceListCmd() *cobra.Command { addCommonFilterFlags(cmd, &ff, false) cmd.Flags().StringVar(&ff.ProjectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&includeFlagged, "include-flagged", false, "Add flagged_comment field populated from user-flagged trace feedback") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") @@ -222,8 +222,8 @@ func newTraceGetCmd() *cobra.Command { cmd.Flags().StringVar(&projectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") cmd.Flags().StringVar(&since, "since", "", "Only include runs after this timestamp, e.g. 2024-01-15T00:00:00Z (overrides 7-day default)") cmd.Flags().IntVar(&lastNMinutes, "last-n-minutes", 0, "Only include runs from the last N minutes, e.g. 60 (overrides 7-day default)") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Write JSON output to a file") @@ -332,8 +332,8 @@ func newTraceExportCmd() *cobra.Command { addCommonFilterFlags(cmd, &ff, false) cmd.Flags().StringVar(&ff.ProjectID, "project-id", "", "Project (session) UUID; skips the name lookup. Takes precedence over --project / $LANGSMITH_PROJECT") - cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, first_token_time, token_usage, costs, tags, custom_metadata (incl. revision_id)") - cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, error, and events fields") + cmd.Flags().BoolVar(&includeMetadata, "include-metadata", false, "Add status, duration_ms, token_usage, costs, tags, custom_metadata (incl. revision_id)") + cmd.Flags().BoolVar(&includeIO, "include-io", false, "Add inputs, outputs, and error fields") cmd.Flags().BoolVar(&includeFeedback, "include-feedback", false, "Add feedback_stats field") cmd.Flags().BoolVar(&full, "full", false, "Shorthand for --include-metadata --include-io --include-feedback") cmd.Flags().StringVar(&filenamePattern, "filename-pattern", "{trace_id}.jsonl", diff --git a/internal/cmd/update.go b/internal/cmd/update.go index d84deb1..19ba244 100644 --- a/internal/cmd/update.go +++ b/internal/cmd/update.go @@ -28,48 +28,26 @@ var githubDownloadBaseURL = "https://github.com" func newUpdateCmd(rawVersion string) *cobra.Command { var dryRun bool - var force bool cmd := &cobra.Command{ Use: "self-update", Short: "Update langsmith to the latest version", Long: "Check for and install the latest version of the langsmith CLI.", RunE: func(cmd *cobra.Command, args []string) error { - return runUpdate(cmd.Context(), rawVersion, dryRun, force) + return runUpdate(cmd.Context(), rawVersion, dryRun) }, } cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Check for updates without installing") - cmd.Flags().BoolVar(&force, "force", false, "Replace the binary in place even if a package manager (brew, scoop, go) manages this install") return cmd } -func runUpdate(ctx context.Context, currentVersion string, dryRun, force bool) error { - // Resolve the current binary path first: it drives both install-method - // detection and the eventual in-place replacement. - execPath, err := os.Executable() - if err != nil { - return fmt.Errorf("resolving executable path: %w", err) - } - execPath, err = filepath.EvalSymlinks(execPath) - if err != nil { - return fmt.Errorf("resolving symlinks: %w", err) - } - - home, _ := os.UserHomeDir() - method := detectInstallMethod(execPath, currentVersion, runtime.GOOS, os.Getenv("GOBIN"), os.Getenv("GOPATH"), home) - - if method == methodDev { +func runUpdate(ctx context.Context, currentVersion string, dryRun bool) error { + if currentVersion == "dev" { return fmt.Errorf("cannot update a development build; install from a release") } - // For package-manager-owned installs, refuse to touch the binary and point - // the user at the right command instead. --force overrides this. - if shouldDeferToManager(method, force) { - return reportManagedExternally(method, GetFormat()) - } - latest, err := fetchLatestVersion(ctx) if err != nil { return fmt.Errorf("checking for updates: %w", err) @@ -105,6 +83,16 @@ func runUpdate(ctx context.Context, currentVersion string, dryRun, force bool) e return nil } + // Resolve current binary path + execPath, err := os.Executable() + if err != nil { + return fmt.Errorf("resolving executable path: %w", err) + } + execPath, err = filepath.EvalSymlinks(execPath) + if err != nil { + return fmt.Errorf("resolving symlinks: %w", err) + } + archiveName := buildArchiveName() archiveURL := fmt.Sprintf("%s/langchain-ai/langsmith-cli/releases/download/v%s/%s", githubDownloadBaseURL, latest, archiveName) checksumURL := fmt.Sprintf("%s/langchain-ai/langsmith-cli/releases/download/v%s/checksums.txt", githubDownloadBaseURL, latest) @@ -158,31 +146,6 @@ func runUpdate(ctx context.Context, currentVersion string, dryRun, force bool) e return nil } -// reportManagedExternally informs the user that their install is managed by a -// package manager and prints the command to update it, without modifying the -// binary. It returns nil (a successful, informational outcome). -func reportManagedExternally(method installMethod, format string) error { - mgr, ok := externalManagers[method] - if !ok { - // Programming error: only externally-managed methods should reach here - // (methodManaged updates in place, methodDev errors out earlier). - return fmt.Errorf("internal error: install method %d is not externally managed", method) - } - - if format == "pretty" { - fmt.Printf("langsmith was installed with %s, which manages updates for you.\n"+ - "To update, run:\n\n %s\n\n(Use --force to update in place anyway.)\n", mgr.display, mgr.command) - } else { - out, _ := json.Marshal(map[string]string{ - "status": "managed-externally", - "install_method": mgr.label, - "update_command": mgr.command, - }) - fmt.Println(string(out)) - } - return nil -} - // fetchLatestVersion queries the GitHub Releases API for the latest release tag. func fetchLatestVersion(ctx context.Context) (string, error) { url := fmt.Sprintf("%s/repos/langchain-ai/langsmith-cli/releases/latest", githubReleasesBaseURL) diff --git a/internal/cmd/update_test.go b/internal/cmd/update_test.go index 8e52280..7580434 100644 --- a/internal/cmd/update_test.go +++ b/internal/cmd/update_test.go @@ -203,7 +203,7 @@ func TestRunUpdate_AlreadyUpToDate(t *testing.T) { defer func() { flagOutputFormat = oldFmt }() output := captureStdout(t, func() { - err := runUpdate(context.Background(), "0.1.7", false, false) + err := runUpdate(context.Background(), "0.1.7", false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -235,7 +235,7 @@ func TestRunUpdate_DryRun(t *testing.T) { defer func() { flagOutputFormat = oldFmt }() output := captureStdout(t, func() { - err := runUpdate(context.Background(), "0.1.7", true, false) + err := runUpdate(context.Background(), "0.1.7", true) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -254,7 +254,7 @@ func TestRunUpdate_DryRun(t *testing.T) { } func TestRunUpdate_DevBuild(t *testing.T) { - err := runUpdate(context.Background(), "dev", false, false) + err := runUpdate(context.Background(), "dev", false) if err == nil { t.Error("expected error for dev build") } diff --git a/internal/cmdutil/resolve.go b/internal/cmdutil/resolve.go index 7ea4f83..0efa9de 100644 --- a/internal/cmdutil/resolve.go +++ b/internal/cmdutil/resolve.go @@ -187,10 +187,6 @@ func ResolveClientOptions(cmd *cobra.Command, refreshOAuth bool) (client.Options opts.OAuthAccessToken = profile.AccessToken() case hasProfile && profile.APIKey != "": opts.APIKey = profile.APIKey - // Route the resolved profile through the SDK (WithProfile) so an explicit - // selection replaces the config's current_profile instead of inheriting - // its tenant/base URL. APIKey is kept for the raw-HTTP helpers. - opts.ProfileName = profileName } if cfgErr != nil && opts.APIKey == "" { return opts, cfgErr diff --git a/internal/cmdutil/resolve_test.go b/internal/cmdutil/resolve_test.go index 7dacf5e..ba7ec5a 100644 --- a/internal/cmdutil/resolve_test.go +++ b/internal/cmdutil/resolve_test.go @@ -205,44 +205,6 @@ func TestResolveClientOptions_ProfileFlagSetsProfileName(t *testing.T) { } } -func TestResolveClientOptions_APIKeyProfileSetsProfileName(t *testing.T) { - path := filepath.Join(t.TempDir(), "config.json") - t.Setenv("LANGSMITH_CONFIG_FILE", path) - t.Setenv("LANGSMITH_API_KEY", "") - t.Setenv("LANGSMITH_ENDPOINT", "") - if err := os.WriteFile(path, []byte(`{ - "current_profile": "prod", - "profiles": { - "prod": { - "api_key": "prod-api-key", - "workspace_id": "ws-prod" - }, - "aws": { - "api_key": "aws-api-key" - } - } -} -`), 0600); err != nil { - t.Fatal(err) - } - - cmd := newTestCmd() - _ = cmd.PersistentFlags().Set("profile", "aws") - opts, err := ResolveClientOptions(cmd, false) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if opts.APIKey != "aws-api-key" { - t.Fatalf("expected profile api key, got %q", opts.APIKey) - } - // An api-key profile must route through WithProfile too, so it replaces - // current_profile and clears the inherited tenant. This resolver (used by the - // api/sandbox/ssh subcommands) previously omitted ProfileName here. - if opts.ProfileName != "aws" { - t.Fatalf("expected ProfileName=aws, got %q", opts.ProfileName) - } -} - func TestResolveClientOptions_WorkspaceFlagOverridesEnvAndProfile(t *testing.T) { path := filepath.Join(t.TempDir(), "config.json") t.Setenv("LANGSMITH_CONFIG_FILE", path) diff --git a/internal/extract/extract.go b/internal/extract/extract.go index c0dc1e4..5e63ed9 100644 --- a/internal/extract/extract.go +++ b/internal/extract/extract.go @@ -70,7 +70,6 @@ func ExtractRun(run langsmith.RunSchema, includeMetadata, includeIO, includeFeed result["status"] = run.Status result["duration_ms"] = durationMs - result["first_token_time"] = formatTimeNullable(run.FirstTokenTime) result["custom_metadata"] = customMetadata result["token_usage"] = tokenUsage result["costs"] = costs @@ -81,7 +80,6 @@ func ExtractRun(run langsmith.RunSchema, includeMetadata, includeIO, includeFeed result["inputs"] = nilIfEmptyMap(run.Inputs) result["outputs"] = nilIfEmptyMap(run.Outputs) result["error"] = nilIfEmpty(run.Error) - result["events"] = nilIfEmptyEvents(run.Events) } if includeFeedback { @@ -131,13 +129,6 @@ func nilIfEmptyMap(m map[string]any) any { return m } -func nilIfEmptyEvents(events []map[string]any) any { - if len(events) == 0 { - return nil - } - return events -} - // FormatDurationHuman formats milliseconds as human-readable string. func FormatDurationHuman(ms int64) string { if ms < 1000 { diff --git a/internal/extract/extract_test.go b/internal/extract/extract_test.go index 20060c7..79c3974 100644 --- a/internal/extract/extract_test.go +++ b/internal/extract/extract_test.go @@ -51,7 +51,6 @@ func TestExtractRunBase(t *testing.T) { func TestExtractRunWithMetadata(t *testing.T) { start := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC) end := time.Date(2024, 1, 15, 10, 30, 5, 0, time.UTC) - firstToken := time.Date(2024, 1, 15, 10, 30, 1, 0, time.UTC) run := langsmith.RunSchema{ ID: "run-123", @@ -60,7 +59,6 @@ func TestExtractRunWithMetadata(t *testing.T) { RunType: "llm", StartTime: start, EndTime: end, - FirstTokenTime: firstToken, Status: "success", PromptTokens: 100, CompletionTokens: 50, @@ -78,10 +76,6 @@ func TestExtractRunWithMetadata(t *testing.T) { t.Errorf("expected status=success, got %v", result["status"]) } - if result["first_token_time"] != firstToken.Format(time.RFC3339Nano) { - t.Errorf("expected first_token_time=%s, got %v", firstToken.Format(time.RFC3339Nano), result["first_token_time"]) - } - durationMs, ok := result["duration_ms"].(int64) if !ok || durationMs != 5000 { t.Errorf("expected duration_ms=5000, got %v", result["duration_ms"]) @@ -121,7 +115,6 @@ func TestExtractRunWithIO(t *testing.T) { Inputs: map[string]any{"query": "hello"}, Outputs: map[string]any{"response": "world"}, Error: "some error", - Events: []map[string]interface{}{{"name": "new_token", "kwargs": map[string]interface{}{"token": "hi"}}}, } result := ExtractRun(run, false, true, false) @@ -139,30 +132,6 @@ func TestExtractRunWithIO(t *testing.T) { if result["error"] != "some error" { t.Errorf("expected error='some error', got %v", result["error"]) } - - events, ok := result["events"].([]map[string]interface{}) - if !ok { - t.Fatalf("expected events to be []map[string]interface{}, got %T", result["events"]) - } - if len(events) != 1 || events[0]["name"] != "new_token" { - t.Errorf("expected one new_token event, got %v", events) - } -} - -func TestExtractRunWithIOEmptyEvents(t *testing.T) { - run := langsmith.RunSchema{ - ID: "run-123", - TraceID: "trace-456", - Name: "ChatOpenAI", - RunType: "llm", - StartTime: time.Now(), - } - - result := ExtractRun(run, false, true, false) - - if result["events"] != nil { - t.Errorf("expected events=nil for run with no events, got %v", result["events"]) - } } func TestExtractRunNilParent(t *testing.T) { diff --git a/scripts/install.sh b/scripts/install.sh index 284ff6c..57dc130 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -39,29 +39,9 @@ fi # Get version if [ -z "$VERSION" ]; then - API_URL="https://api.github.com/repos/${REPO}/releases/latest" - - fetch_latest() { - if [ -n "$GITHUB_TOKEN" ]; then - curl -fsSL -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_TOKEN" "$API_URL" - else - curl -fsSL -H "Accept: application/vnd.github+json" "$API_URL" - fi - } - - VERSION="" - i=1 - while [ "$i" -le 3 ]; do - VERSION="$(fetch_latest | grep '"tag_name"' | sed -E 's/.*"tag_name": *"([^"]+)".*/\1/')" - [ -n "$VERSION" ] && break - [ "$i" -lt 3 ] && sleep "$i" - i=$((i + 1)) - done - + VERSION="$(curl -sSL "https://api.github.com/repos/${REPO}/releases/latest" | grep '"tag_name"' | sed -E 's/.*"tag_name": *"([^"]+)".*/\1/')" if [ -z "$VERSION" ]; then - echo "Failed to determine latest version (GitHub API unreachable or rate-limited)." >&2 - echo "Set GITHUB_TOKEN to raise the rate limit, or set VERSION to install a specific release." >&2 + echo "Failed to determine latest version" >&2 exit 1 fi fi