diff --git a/README.md b/README.md index add63ed0..5a7bd65c 100644 --- a/README.md +++ b/README.md @@ -311,6 +311,40 @@ langsmith self-update --dry-run langsmith self-update ``` +### `setup` — Trace coding agents to LangSmith + +Configure Claude Code or Codex to send full-content traces (prompts, responses, tool +calls) to a LangSmith project. Enables the LangSmith tracing plugin for the agent and +writes the credentials it needs to the agent's local config, so every future session +traces automatically. Requires an API key (the key is written to the agent config at +`0600`); OAuth profiles are not supported here. The API URL defaults to +`https://api.smith.langchain.com` (override with `--api-url` or `LANGSMITH_ENDPOINT`), +and without `--project` no project is written — each plugin's own default applies +(`claude-code` / `codex`). + +```bash +# Configure Claude Code (writes ~/.claude/settings.json) +langsmith setup claude + +# Configure Codex (writes ~/.codex/config.toml + ~/.codex/langsmith.json) +langsmith setup codex + +# Both at once +langsmith setup all + +# Without an argument, the key resolves from --api-key, $LANGSMITH_API_KEY, or a saved profile +langsmith setup claude + +# Trace to a named project (default: $LANGSMITH_PROJECT, else the plugin's own default) +langsmith setup claude --project my-agent + +# Write project-local config instead of user-global +langsmith setup claude --scope project # ./.claude/settings.local.json +langsmith setup codex --scope project # ./.codex/... +``` + +After running, start the agent. Verify Claude Code with `tail -f ~/.claude/state/hook.log`. + ## Filter Options Most `trace` and `run` commands share these filter options: diff --git a/go.mod b/go.mod index b13184c4..e10d26a7 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/itchyny/gojq v0.12.15 github.com/langchain-ai/langsmith-go v0.10.0 github.com/olekukonko/tablewriter v0.0.5 + github.com/pelletier/go-toml/v2 v2.3.1 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.11.1 github.com/xlab/treeprint v1.2.0 diff --git a/go.sum b/go.sum index 0e8a6179..084d5c30 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= +github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 30f514ce..dd51b796 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -57,7 +57,7 @@ Quick start: } rootCmd.PersistentFlags().StringVar(&flagAPIKey, "api-key", "", "LangSmith API key [env: LANGSMITH_API_KEY]") - rootCmd.PersistentFlags().StringVar(&flagAPIURL, "api-url", "", "LangSmith API URL [env: LANGSMITH_ENDPOINT]") + rootCmd.PersistentFlags().StringVar(&flagAPIURL, "api-url", "", "LangSmith API URL [env: LANGSMITH_ENDPOINT] (default: "+lsconfig.DefaultAPIURL+")") rootCmd.PersistentFlags().StringVar(&flagProfile, "profile", "", "Named profile to use [env: LANGSMITH_PROFILE]") rootCmd.PersistentFlags().StringVar(&flagWorkspaceID, "workspace", "", "LangSmith workspace ID [env: LANGSMITH_WORKSPACE_ID]") rootCmd.PersistentFlags().StringVar(&flagWorkspaceID, "workspace-id", "", "LangSmith workspace ID [env: LANGSMITH_WORKSPACE_ID]") @@ -78,6 +78,7 @@ Quick start: rootCmd.AddCommand(newFleetCmd()) rootCmd.AddCommand(newHubCmd()) rootCmd.AddCommand(newPromptCmd()) + rootCmd.AddCommand(newSetupCmd()) rootCmd.AddCommand(authCommand.Cobra()) rootCmd.AddCommand(newProfileCmd()) rootCmd.AddCommand(newWorkspaceCmd()) diff --git a/internal/cmd/setup.go b/internal/cmd/setup.go new file mode 100644 index 00000000..203c7e12 --- /dev/null +++ b/internal/cmd/setup.go @@ -0,0 +1,201 @@ +package cmd + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/langchain-ai/langsmith-cli/internal/client" + lsconfig "github.com/langchain-ai/langsmith-cli/internal/config" + "github.com/spf13/cobra" +) + +// Env-var contract consumed by the LangSmith tracing plugins for coding agents. +// Claude Code uses the CC_LANGSMITH_* namespace exclusively; a custom endpoint +// is expressed via CC_LANGSMITH_RUNS_ENDPOINTS (a JSON replica array) rather +// than a plain endpoint var. Keep these in sync with langsmith-claude-code-plugins. +const ( + envTraceToLangSmith = "TRACE_TO_LANGSMITH" + envCCLangSmithAPIKey = "CC_LANGSMITH_API_KEY" + envCCLangSmithProject = "CC_LANGSMITH_PROJECT" + envCCLangSmithRunsEndpoints = "CC_LANGSMITH_RUNS_ENDPOINTS" +) + +func newSetupCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "setup", + Short: "Configure coding agents (Claude Code, Codex) to trace to LangSmith", + Long: `Configure coding agents to send full-content traces to LangSmith. + +These commands write the LangSmith tracing config into the agent's local config +files, so every future session traces to a LangSmith project automatically. They +do not launch Claude Code or Codex: Claude Code installs the declared plugin on +its next launch; Codex needs one manual 'codex plugin marketplace add' (printed +after setup) to fetch the plugin code. + +Requires a LangSmith API key, passed as an argument (langsmith setup claude +) or resolved from --api-key, LANGSMITH_API_KEY, or a saved profile. +The key is written to the agent config file at owner-only (0600) permissions. +The API URL defaults to https://api.smith.langchain.com; pass --api-url or set +LANGSMITH_ENDPOINT for self-hosted instances. Without --project, no project is +written and the plugin's own default applies.`, + } + cmd.AddCommand(newSetupClaudeCmd()) + cmd.AddCommand(newSetupCodexCmd()) + cmd.AddCommand(newSetupAllCmd()) + return cmd +} + +func newSetupAllCmd() *cobra.Command { + var ( + project string + scope string + ) + cmd := &cobra.Command{ + Use: "all [api-key]", + Short: "Configure both Claude Code and Codex to trace to LangSmith", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + apiKey := positionalKey(args) + cErr := runSetupClaude(cmd, apiKey, project, scope) + if cErr != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "claude setup failed: %v\n", cErr) + } + xErr := runSetupCodex(cmd, apiKey, project, scope) + if xErr != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "codex setup failed: %v\n", xErr) + } + if cErr != nil && xErr != nil { + return errors.New("both claude and codex setup failed") + } + return nil + }, + } + cmd.Flags().StringVar(&project, "project", "", "LangSmith project name (default: $LANGSMITH_PROJECT, else each plugin's own default)") + cmd.Flags().StringVar(&scope, "scope", "user", "Config scope: user or project") + return cmd +} + +// positionalKey extracts the optional [api-key] argument. +func positionalKey(args []string) string { + if len(args) == 0 { + return "" + } + return strings.TrimSpace(args[0]) +} + +// setupClientOptions resolves credentials and requires an API key, since the +// tracing plugins authenticate with a workspace-scoped key (OAuth won't work). +// A non-empty positionalKey (the optional [api-key] argument) behaves exactly +// like --api-key, taking precedence over the environment and saved profiles. +func setupClientOptions(positionalKey string) (client.Options, error) { + if positionalKey != "" { + if flagAPIKey != "" && flagAPIKey != positionalKey { + return client.Options{}, errors.New("API key passed both as an argument and via --api-key; pass it once") + } + flagAPIKey = positionalKey + } + opts, err := resolveClientOptions(true) + if err != nil { + return opts, err + } + if opts.APIKey == "" { + return opts, errors.New("tracing setup requires a LangSmith API key; pass one as an argument (langsmith setup claude ), set LANGSMITH_API_KEY, or run 'langsmith profile create' with an API-key profile") + } + return opts, nil +} + +// envTraceProject resolves the LangSmith project name from the environment. +// Empty means no project is written, deferring to the plugin's own default. +func envTraceProject() string { + for _, key := range []string{"LANGSMITH_AGENT_PROJECT", "LANGSMITH_PROJECT"} { + if v := strings.TrimSpace(os.Getenv(key)); v != "" { + return v + } + } + return "" +} + +func isDefaultEndpoint(apiURL string) bool { + return apiURL == "" || apiURL == lsconfig.DefaultAPIURL +} + +func scopeOrDefault(scope string) string { + if scope == "" { + return "user" + } + return scope +} + +// claudeConfigDir returns ~/.claude (or $CLAUDE_CONFIG_DIR when set). +func claudeConfigDir() (string, error) { + if d := strings.TrimSpace(os.Getenv("CLAUDE_CONFIG_DIR")); d != "" { + return d, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("resolving home directory: %w", err) + } + return filepath.Join(home, ".claude"), nil +} + +// codexHome returns ~/.codex (or $CODEX_HOME when set). +func codexHome() (string, error) { + if d := strings.TrimSpace(os.Getenv("CODEX_HOME")); d != "" { + return d, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("resolving home directory: %w", err) + } + return filepath.Join(home, ".codex"), nil +} + +// mergeJSONFile reads path as a JSON object (empty when absent), applies mutate, +// and writes it back indented with owner-only (0600) permissions. Unknown keys +// in the existing file are preserved. +func mergeJSONFile(path string, mutate func(map[string]any) error) error { + doc := map[string]any{} + data, err := os.ReadFile(path) + switch { + case err == nil: + if len(strings.TrimSpace(string(data))) > 0 { + if err := json.Unmarshal(data, &doc); err != nil { + return fmt.Errorf("parsing %s: %w", path, err) + } + } + case errors.Is(err, os.ErrNotExist): + // Start from an empty object. + default: + return fmt.Errorf("reading %s: %w", path, err) + } + + if err := mutate(doc); err != nil { + return err + } + + out, err := json.MarshalIndent(doc, "", " ") + if err != nil { + return fmt.Errorf("encoding %s: %w", path, err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("creating %s: %w", filepath.Dir(path), err) + } + if err := os.WriteFile(path, append(out, '\n'), 0o600); err != nil { + return fmt.Errorf("writing %s: %w", path, err) + } + return nil +} + +// jsonObject returns doc[key] as a map, creating it when missing or not an object. +func jsonObject(doc map[string]any, key string) map[string]any { + if existing, ok := doc[key].(map[string]any); ok { + return existing + } + m := map[string]any{} + doc[key] = m + return m +} diff --git a/internal/cmd/setup_claude.go b/internal/cmd/setup_claude.go new file mode 100644 index 00000000..6e880a63 --- /dev/null +++ b/internal/cmd/setup_claude.go @@ -0,0 +1,180 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "path/filepath" + + "github.com/langchain-ai/langsmith-cli/internal/client" + "github.com/spf13/cobra" +) + +// Claude Code LangSmith tracing plugin coordinates. The marketplace name must +// match how the plugin is referenced (langsmith-tracing@), so the +// extraKnownMarketplaces key and enabledPlugins key stay consistent. +const ( + claudeMarketplaceName = "langsmith-claude-code-plugins" + claudeMarketplaceRepo = "langchain-ai/langsmith-claude-code-plugins" + claudeMarketplaceURL = "https://github.com/langchain-ai/langsmith-claude-code-plugins.git" + claudePluginName = "langsmith-tracing" + + // Project the plugin traces to when CC_LANGSMITH_PROJECT is unset (its + // src/config.ts). Display-only — the CLI never writes this value. + claudeDefaultProject = "claude-code" +) + +func newSetupClaudeCmd() *cobra.Command { + var ( + project string + scope string + ) + cmd := &cobra.Command{ + Use: "claude [api-key]", + Short: "Configure Claude Code to trace to LangSmith", + Args: cobra.MaximumNArgs(1), + Long: `Configure Claude Code to send full-content traces to LangSmith. + +Declares the LangSmith tracing marketplace, enables the plugin, and stores your +API key in Claude Code's settings.json. Claude Code installs and enables the +plugin on its next launch — this command does not run Claude Code. + +The API URL defaults to https://api.smith.langchain.com. Without --project, no +project is written and the plugin's own default ("claude-code") applies. + + langsmith setup claude # write ~/.claude/settings.json + langsmith setup claude --project my-agent # trace to a named project + langsmith setup claude --scope project # write ./.claude/settings.local.json`, + RunE: func(cmd *cobra.Command, args []string) error { + return runSetupClaude(cmd, positionalKey(args), project, scope) + }, + } + cmd.Flags().StringVar(&project, "project", "", "LangSmith project name (default: $LANGSMITH_PROJECT, else the plugin's default \"claude-code\")") + cmd.Flags().StringVar(&scope, "scope", "user", "Config scope: user (~/.claude/settings.json) or project (./.claude/settings.local.json)") + return cmd +} + +func runSetupClaude(cmd *cobra.Command, apiKey, project, scope string) error { + opts, err := setupClientOptions(apiKey) + if err != nil { + return err + } + if project == "" { + project = envTraceProject() + } + + settingsPath, err := claudeSettingsPath(scope) + if err != nil { + return err + } + + pluginRef := claudePluginName + "@" + claudeMarketplaceName + if err := writeClaudeSettings(settingsPath, pluginRef, opts, project); err != nil { + return err + } + + return reportClaudeSetup(cmd, settingsPath, project, opts.APIURL, scope, pluginRef) +} + +func claudeSettingsPath(scope string) (string, error) { + switch scope { + case "", "user": + dir, err := claudeConfigDir() + if err != nil { + return "", err + } + return filepath.Join(dir, "settings.json"), nil + case "project": + // settings.local.json is the personal, git-ignored project override — + // the right place for a file that embeds an API key. + return filepath.Join(".claude", "settings.local.json"), nil + default: + return "", fmt.Errorf("invalid --scope %q: must be \"user\" or \"project\"", scope) + } +} + +func writeClaudeSettings(path, pluginRef string, opts client.Options, project string) error { + return mergeJSONFile(path, func(doc map[string]any) error { + markets := jsonObject(doc, "extraKnownMarketplaces") + markets[claudeMarketplaceName] = map[string]any{ + "source": map[string]any{ + "source": "github", + "repo": claudeMarketplaceRepo, + }, + } + + enabled := jsonObject(doc, "enabledPlugins") + enabled[pluginRef] = true + + env := jsonObject(doc, "env") + env[envTraceToLangSmith] = "true" + env[envCCLangSmithAPIKey] = opts.APIKey + // Without an explicit project the var is removed (not written empty) so + // the plugin's own default applies — including on re-runs that drop a + // previously configured project. + if project != "" { + env[envCCLangSmithProject] = project + } else { + delete(env, envCCLangSmithProject) + } + // A non-default (e.g. self-hosted) endpoint is expressed as a single + // replica destination — the documented way to point Claude Code's + // tracing at a custom API URL. + if !isDefaultEndpoint(opts.APIURL) { + replica := map[string]any{ + "apiUrl": opts.APIURL, + "apiKey": opts.APIKey, + } + if project != "" { + replica["projectName"] = project + } + encoded, err := json.Marshal([]map[string]any{replica}) + if err != nil { + return fmt.Errorf("encoding runs endpoints: %w", err) + } + env[envCCLangSmithRunsEndpoints] = string(encoded) + } else { + delete(env, envCCLangSmithRunsEndpoints) + } + return nil + }) +} + +func reportClaudeSetup(cmd *cobra.Command, settingsPath, project, apiURL, scope, pluginRef string) error { + if GetFormat() == "pretty" { + out := cmd.OutOrStdout() + if project == "" { + fmt.Fprintf(out, "Configured Claude Code tracing → LangSmith project %q (plugin default)\n", claudeDefaultProject) + } else { + fmt.Fprintf(out, "Configured Claude Code tracing → LangSmith project %q\n", project) + } + fmt.Fprintf(out, " settings: %s (contains your API key; mode 0600)\n", settingsPath) + fmt.Fprintf(out, " plugin: %s\n", pluginRef) + if !isDefaultEndpoint(apiURL) { + fmt.Fprintf(out, " endpoint: %s\n", apiURL) + } + fmt.Fprintf(out, "\nClaude Code installs and enables the plugin on its next launch.\n") + fmt.Fprintf(out, "If it does not appear, run inside Claude Code:\n") + fmt.Fprintf(out, " /plugin marketplace add %s\n", claudeMarketplaceURL) + fmt.Fprintf(out, " /plugin install %s\n", pluginRef) + fmt.Fprintf(out, "Verify with: tail -f ~/.claude/state/hook.log\n") + return nil + } + + result := map[string]any{ + "status": "configured", + "agent": "claude-code", + "settings_path": settingsPath, + "scope": scopeOrDefault(scope), + "plugin": pluginRef, + "installs_on": "next claude launch", + } + if project != "" { + result["project"] = project + } + if !isDefaultEndpoint(apiURL) { + result["endpoint"] = apiURL + } + enc := json.NewEncoder(cmd.OutOrStdout()) + enc.SetIndent("", " ") + return enc.Encode(result) +} diff --git a/internal/cmd/setup_codex.go b/internal/cmd/setup_codex.go new file mode 100644 index 00000000..8659f7e1 --- /dev/null +++ b/internal/cmd/setup_codex.go @@ -0,0 +1,192 @@ +package cmd + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/langchain-ai/langsmith-cli/internal/client" + "github.com/pelletier/go-toml/v2" + "github.com/spf13/cobra" +) + +// Codex LangSmith tracing plugin coordinates. +const ( + codexMarketplaceURL = "github.com/langchain-ai/langsmith-codex-plugins" + codexPluginRef = "tracing@langsmith-codex-plugins" + + // Project the plugin traces to when langsmith.json has no "project" key + // (its plugins/tracing/src/config.ts). Display-only — the CLI never + // writes this value. + codexDefaultProject = "codex" +) + +func newSetupCodexCmd() *cobra.Command { + var ( + project string + scope string + ) + cmd := &cobra.Command{ + Use: "codex [api-key]", + Short: "Configure Codex to trace to LangSmith", + Args: cobra.MaximumNArgs(1), + Long: `Configure Codex to send full-content traces to LangSmith. + +Enables the LangSmith tracing plugin in Codex's config.toml and stores your API +key in langsmith.json. Codex has no file-only way to fetch the plugin code, so +this command prints one 'codex plugin marketplace add' command to run once — it +does not run Codex itself. + +The API URL defaults to https://api.smith.langchain.com. Without --project, no +project is written and the plugin's own default ("codex") applies. + + langsmith setup codex # writes ~/.codex/{config.toml,langsmith.json} + langsmith setup codex --project my-agent + langsmith setup codex --scope project # writes ./.codex/...`, + RunE: func(cmd *cobra.Command, args []string) error { + return runSetupCodex(cmd, positionalKey(args), project, scope) + }, + } + cmd.Flags().StringVar(&project, "project", "", "LangSmith project name (default: $LANGSMITH_PROJECT, else the plugin's default \"codex\")") + cmd.Flags().StringVar(&scope, "scope", "user", "Config scope: user (~/.codex) or project (./.codex)") + return cmd +} + +func runSetupCodex(cmd *cobra.Command, apiKey, project, scope string) error { + opts, err := setupClientOptions(apiKey) + if err != nil { + return err + } + if project == "" { + project = envTraceProject() + } + + dir, err := codexConfigDir(scope) + if err != nil { + return err + } + + credPath := filepath.Join(dir, "langsmith.json") + if err := writeCodexCredentials(credPath, opts, project); err != nil { + return err + } + + tomlPath := filepath.Join(dir, "config.toml") + if err := enableCodexPlugin(tomlPath); err != nil { + return err + } + + return reportCodexSetup(cmd, credPath, tomlPath, project, opts.APIURL, scope) +} + +func codexConfigDir(scope string) (string, error) { + switch scope { + case "", "user": + return codexHome() + case "project": + return ".codex", nil + default: + return "", fmt.Errorf("invalid --scope %q: must be \"user\" or \"project\"", scope) + } +} + +func writeCodexCredentials(path string, opts client.Options, project string) error { + return mergeJSONFile(path, func(doc map[string]any) error { + doc["enabled"] = true + doc["api_key"] = opts.APIKey + // Without an explicit project the key is removed (not written empty — + // the plugin treats "" as an explicit value) so its own default + // applies, including on re-runs that drop a configured project. + if project != "" { + doc["project"] = project + } else { + delete(doc, "project") + } + if !isDefaultEndpoint(opts.APIURL) { + doc["api_url"] = opts.APIURL + } else { + delete(doc, "api_url") + } + return nil + }) +} + +// enableCodexPlugin sets features.plugin_hooks=true and enables the tracing +// plugin in config.toml, preserving any existing TOML keys. +func enableCodexPlugin(path string) error { + doc := map[string]any{} + data, err := os.ReadFile(path) + switch { + case err == nil: + if len(strings.TrimSpace(string(data))) > 0 { + if err := toml.Unmarshal(data, &doc); err != nil { + return fmt.Errorf("parsing %s: %w", path, err) + } + } + case errors.Is(err, os.ErrNotExist): + // Start from an empty document. + default: + return fmt.Errorf("reading %s: %w", path, err) + } + + features := jsonObject(doc, "features") + features["plugin_hooks"] = true + + plugins := jsonObject(doc, "plugins") + entry := jsonObject(plugins, codexPluginRef) + entry["enabled"] = true + + out, err := toml.Marshal(doc) + if err != nil { + return fmt.Errorf("encoding %s: %w", path, err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("creating %s: %w", filepath.Dir(path), err) + } + if err := os.WriteFile(path, out, 0o600); err != nil { + return fmt.Errorf("writing %s: %w", path, err) + } + return nil +} + +func reportCodexSetup(cmd *cobra.Command, credPath, tomlPath, project, apiURL, scope string) error { + if GetFormat() == "pretty" { + out := cmd.OutOrStdout() + if project == "" { + fmt.Fprintf(out, "Configured Codex tracing → LangSmith project %q (plugin default)\n", codexDefaultProject) + } else { + fmt.Fprintf(out, "Configured Codex tracing → LangSmith project %q\n", project) + } + fmt.Fprintf(out, " credentials: %s (contains your API key; mode 0600)\n", credPath) + fmt.Fprintf(out, " config: %s (plugin enabled)\n", tomlPath) + if !isDefaultEndpoint(apiURL) { + fmt.Fprintf(out, " endpoint: %s\n", apiURL) + } + fmt.Fprintf(out, "\nOne manual step — fetch the plugin code (run once):\n") + fmt.Fprintf(out, " codex plugin marketplace add %s\n", codexMarketplaceURL) + fmt.Fprintf(out, "Then start Codex and traces will flow to LangSmith.\n") + return nil + } + + result := map[string]any{ + "status": "configured", + "agent": "codex", + "credentials_path": credPath, + "config_path": tomlPath, + "scope": scopeOrDefault(scope), + "plugin": codexPluginRef, + "manual_step": "codex plugin marketplace add " + codexMarketplaceURL, + } + if project != "" { + result["project"] = project + } + if !isDefaultEndpoint(apiURL) { + result["endpoint"] = apiURL + } + enc := json.NewEncoder(cmd.OutOrStdout()) + enc.SetIndent("", " ") + return enc.Encode(result) +} diff --git a/internal/cmd/setup_test.go b/internal/cmd/setup_test.go new file mode 100644 index 00000000..edc66452 --- /dev/null +++ b/internal/cmd/setup_test.go @@ -0,0 +1,373 @@ +package cmd + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/pelletier/go-toml/v2" +) + +// hermeticSetupEnv points the agent config dirs and LangSmith credential +// resolution at a throwaway HOME so setup tests never touch the real machine. +func hermeticSetupEnv(t *testing.T) (claudeDir, codexDir string) { + t.Helper() + home := t.TempDir() + claudeDir = filepath.Join(home, ".claude") + codexDir = filepath.Join(home, ".codex") + t.Setenv("HOME", home) + t.Setenv("CLAUDE_CONFIG_DIR", claudeDir) + t.Setenv("CODEX_HOME", codexDir) + t.Setenv("LANGSMITH_CONFIG_FILE", filepath.Join(home, "ls-config.json")) + t.Setenv("LANGSMITH_API_KEY", "") + t.Setenv("LANGSMITH_ENDPOINT", "") + t.Setenv("LANGSMITH_PROFILE", "") + t.Setenv("LANGSMITH_PROJECT", "") + t.Setenv("LANGSMITH_AGENT_PROJECT", "") + t.Setenv("LANGSMITH_WORKSPACE_ID", "") + t.Setenv("LANGSMITH_TENANT_ID", "") + return claudeDir, codexDir +} + +func readJSONFile(t *testing.T, path string) map[string]any { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading %s: %v", path, err) + } + var doc map[string]any + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatalf("parsing %s: %v\n%s", path, err, data) + } + return doc +} + +func assertPerm0600(t *testing.T, path string) { + t.Helper() + fi, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + if perm := fi.Mode().Perm(); perm != 0o600 { + t.Fatalf("expected %s to be 0600, got %o", path, perm) + } +} + +func claudeEnv(t *testing.T, claudeDir string) map[string]any { + t.Helper() + doc := readJSONFile(t, filepath.Join(claudeDir, "settings.json")) + env, _ := doc["env"].(map[string]any) + return env +} + +func TestSetupClaudeWritesSettings(t *testing.T) { + claudeDir, _ := hermeticSetupEnv(t) + + // Pre-existing settings with an unrelated top-level key and env key that + // must survive the merge. + if err := os.MkdirAll(claudeDir, 0o700); err != nil { + t.Fatal(err) + } + settingsPath := filepath.Join(claudeDir, "settings.json") + if err := os.WriteFile(settingsPath, []byte(`{"model":"opus","env":{"FOO":"bar"}}`), 0o600); err != nil { + t.Fatal(err) + } + + stdout, err := executeCommand(t, + "--format=json", + "setup", "claude", + "--api-key", "test-key-abc", + "--project", "demo", + ) + if err != nil { + t.Fatalf("setup claude error: %v\n%s", err, stdout) + } + if strings.Contains(stdout, "test-key-abc") { + t.Fatalf("setup output leaked the api key: %s", stdout) + } + + var result map[string]any + if err := json.Unmarshal([]byte(stdout), &result); err != nil { + t.Fatalf("stdout not JSON: %v\n%s", err, stdout) + } + if result["status"] != "configured" || result["agent"] != "claude-code" || result["project"] != "demo" { + t.Fatalf("unexpected result: %+v", result) + } + if result["plugin"] != "langsmith-tracing@langsmith-claude-code-plugins" { + t.Fatalf("unexpected plugin ref: %+v", result["plugin"]) + } + if _, ok := result["endpoint"]; ok { + t.Fatalf("endpoint should be omitted for the default SaaS URL: %+v", result) + } + + doc := readJSONFile(t, settingsPath) + if doc["model"] != "opus" { + t.Fatalf("unrelated key 'model' was not preserved: %+v", doc) + } + markets, _ := doc["extraKnownMarketplaces"].(map[string]any) + mk, _ := markets["langsmith-claude-code-plugins"].(map[string]any) + src, _ := mk["source"].(map[string]any) + if src["source"] != "github" || src["repo"] != "langchain-ai/langsmith-claude-code-plugins" { + t.Fatalf("marketplace not written correctly: %+v", markets) + } + enabled, _ := doc["enabledPlugins"].(map[string]any) + if enabled["langsmith-tracing@langsmith-claude-code-plugins"] != true { + t.Fatalf("plugin not enabled: %+v", enabled) + } + env, _ := doc["env"].(map[string]any) + if env["FOO"] != "bar" { + t.Fatalf("unrelated env key was not preserved: %+v", env) + } + if env["TRACE_TO_LANGSMITH"] != "true" || env["CC_LANGSMITH_API_KEY"] != "test-key-abc" || + env["CC_LANGSMITH_PROJECT"] != "demo" { + t.Fatalf("tracing env not written correctly: %+v", env) + } + for _, key := range []string{"LANGSMITH_API_KEY", "LANGSMITH_PROJECT", "LANGSMITH_ENDPOINT", "CC_LANGSMITH_RUNS_ENDPOINTS"} { + if _, ok := env[key]; ok { + t.Fatalf("env key %s should not be written: %+v", key, env) + } + } + assertPerm0600(t, settingsPath) +} + +func TestSetupClaudePositionalKeyDefaults(t *testing.T) { + claudeDir, _ := hermeticSetupEnv(t) + + stdout, err := executeCommand(t, "--format=json", "setup", "claude", "test-key-abc") + if err != nil { + t.Fatalf("setup claude error: %v\n%s", err, stdout) + } + if strings.Contains(stdout, "test-key-abc") { + t.Fatalf("setup output leaked the api key: %s", stdout) + } + + var result map[string]any + if err := json.Unmarshal([]byte(stdout), &result); err != nil { + t.Fatalf("stdout not JSON: %v\n%s", err, stdout) + } + if _, ok := result["project"]; ok { + t.Fatalf("project should be omitted when not configured (plugin default applies): %+v", result) + } + + env := claudeEnv(t, claudeDir) + if env["TRACE_TO_LANGSMITH"] != "true" || env["CC_LANGSMITH_API_KEY"] != "test-key-abc" { + t.Fatalf("tracing env not written correctly: %+v", env) + } + if _, ok := env["CC_LANGSMITH_PROJECT"]; ok { + t.Fatalf("CC_LANGSMITH_PROJECT should be omitted so the plugin default applies: %+v", env) + } + if _, ok := env["CC_LANGSMITH_RUNS_ENDPOINTS"]; ok { + t.Fatalf("runs endpoints should be omitted for the default SaaS URL: %+v", env) + } +} + +func TestSetupClaudeKeyConflict(t *testing.T) { + hermeticSetupEnv(t) + + _, err := executeCommand(t, "setup", "claude", "key-one", "--api-key", "key-two") + if err == nil { + t.Fatal("expected an error when the key is passed both positionally and via --api-key") + } + if !strings.Contains(err.Error(), "--api-key") { + t.Fatalf("expected conflicting-key error, got: %v", err) + } +} + +func TestSetupClaudeSelfHostedEndpoint(t *testing.T) { + claudeDir, _ := hermeticSetupEnv(t) + + if _, err := executeCommand(t, + "setup", "claude", + "--api-key", "k", "--project", "p", + "--api-url", "https://ls.internal.example.com/api/v1", + ); err != nil { + t.Fatalf("setup claude error: %v", err) + } + + env := claudeEnv(t, claudeDir) + raw, _ := env["CC_LANGSMITH_RUNS_ENDPOINTS"].(string) + if raw == "" { + t.Fatalf("expected CC_LANGSMITH_RUNS_ENDPOINTS for a self-hosted URL: %+v", env) + } + var replicas []map[string]any + if err := json.Unmarshal([]byte(raw), &replicas); err != nil { + t.Fatalf("runs endpoints not valid JSON: %v\n%s", err, raw) + } + if len(replicas) != 1 { + t.Fatalf("expected one replica, got %+v", replicas) + } + if replicas[0]["apiUrl"] != "https://ls.internal.example.com" || replicas[0]["apiKey"] != "k" || + replicas[0]["projectName"] != "p" { + t.Fatalf("unexpected replica: %+v", replicas[0]) + } +} + +func TestSetupClaudeSelfHostedEndpointNoProject(t *testing.T) { + claudeDir, _ := hermeticSetupEnv(t) + + if _, err := executeCommand(t, + "setup", "claude", "k", + "--api-url", "https://ls.internal.example.com", + ); err != nil { + t.Fatalf("setup claude error: %v", err) + } + + env := claudeEnv(t, claudeDir) + raw, _ := env["CC_LANGSMITH_RUNS_ENDPOINTS"].(string) + var replicas []map[string]any + if err := json.Unmarshal([]byte(raw), &replicas); err != nil { + t.Fatalf("runs endpoints not valid JSON: %v\n%s", err, raw) + } + if len(replicas) != 1 { + t.Fatalf("expected one replica, got %+v", replicas) + } + if _, ok := replicas[0]["projectName"]; ok { + t.Fatalf("projectName should be omitted when no project is configured: %+v", replicas[0]) + } +} + +func TestSetupClaudeRequiresAPIKey(t *testing.T) { + hermeticSetupEnv(t) + + _, err := executeCommand(t, "setup", "claude") + if err == nil { + t.Fatal("expected an error when no API key is available") + } + if !strings.Contains(err.Error(), "API key") { + t.Fatalf("expected API-key error, got: %v", err) + } +} + +func TestSetupClaudeIdempotentAndDropsStaleProject(t *testing.T) { + claudeDir, _ := hermeticSetupEnv(t) + + if _, err := executeCommand(t, "setup", "claude", "k", "--project", "p"); err != nil { + t.Fatalf("first run error: %v", err) + } + if env := claudeEnv(t, claudeDir); env["CC_LANGSMITH_PROJECT"] != "p" { + t.Fatalf("first run did not write the project: %+v", env) + } + + // Re-running without --project removes the stale value so the plugin + // default applies again. + if _, err := executeCommand(t, "setup", "claude", "k"); err != nil { + t.Fatalf("second run error: %v", err) + } + env := claudeEnv(t, claudeDir) + if _, ok := env["CC_LANGSMITH_PROJECT"]; ok { + t.Fatalf("re-run without --project should remove CC_LANGSMITH_PROJECT: %+v", env) + } + + doc := readJSONFile(t, filepath.Join(claudeDir, "settings.json")) + enabled, _ := doc["enabledPlugins"].(map[string]any) + if len(enabled) != 1 || enabled["langsmith-tracing@langsmith-claude-code-plugins"] != true { + t.Fatalf("re-run should not duplicate enabled plugins: %+v", enabled) + } +} + +func TestSetupCodexWritesConfig(t *testing.T) { + _, codexDir := hermeticSetupEnv(t) + + // Pre-existing config.toml with an unrelated table that must survive. + if err := os.MkdirAll(codexDir, 0o700); err != nil { + t.Fatal(err) + } + tomlPath := filepath.Join(codexDir, "config.toml") + if err := os.WriteFile(tomlPath, []byte("[model]\nname = \"gpt-5\"\n"), 0o600); err != nil { + t.Fatal(err) + } + + stdout, err := executeCommand(t, + "--format=json", + "setup", "codex", + "--api-key", "test-key-xyz", + "--project", "demo", + "--api-url", "https://ls.internal.example.com", + ) + if err != nil { + t.Fatalf("setup codex error: %v\n%s", err, stdout) + } + if strings.Contains(stdout, "test-key-xyz") { + t.Fatalf("setup output leaked the api key: %s", stdout) + } + + // Credentials file. + cred := readJSONFile(t, filepath.Join(codexDir, "langsmith.json")) + if cred["enabled"] != true || cred["api_key"] != "test-key-xyz" || + cred["project"] != "demo" || cred["api_url"] != "https://ls.internal.example.com" { + t.Fatalf("langsmith.json not written correctly: %+v", cred) + } + assertPerm0600(t, filepath.Join(codexDir, "langsmith.json")) + + // config.toml round-trips and preserves the unrelated table. + data, err := os.ReadFile(tomlPath) + if err != nil { + t.Fatal(err) + } + var conf map[string]any + if err := toml.Unmarshal(data, &conf); err != nil { + t.Fatalf("config.toml is not valid TOML: %v\n%s", err, data) + } + model, _ := conf["model"].(map[string]any) + if model["name"] != "gpt-5" { + t.Fatalf("unrelated [model] table not preserved: %+v", conf) + } + features, _ := conf["features"].(map[string]any) + if features["plugin_hooks"] != true { + t.Fatalf("plugin_hooks not enabled: %+v", features) + } + plugins, _ := conf["plugins"].(map[string]any) + entry, _ := plugins["tracing@langsmith-codex-plugins"].(map[string]any) + if entry["enabled"] != true { + t.Fatalf("codex plugin not enabled: %+v", plugins) + } + assertPerm0600(t, tomlPath) +} + +func TestSetupCodexPositionalKeyDefaults(t *testing.T) { + _, codexDir := hermeticSetupEnv(t) + + stdout, err := executeCommand(t, "--format=json", "setup", "codex", "test-key-xyz") + if err != nil { + t.Fatalf("setup codex error: %v\n%s", err, stdout) + } + + var result map[string]any + if err := json.Unmarshal([]byte(stdout), &result); err != nil { + t.Fatalf("stdout not JSON: %v\n%s", err, stdout) + } + if _, ok := result["project"]; ok { + t.Fatalf("project should be omitted when not configured (plugin default applies): %+v", result) + } + + cred := readJSONFile(t, filepath.Join(codexDir, "langsmith.json")) + if cred["enabled"] != true || cred["api_key"] != "test-key-xyz" { + t.Fatalf("langsmith.json not written correctly: %+v", cred) + } + for _, key := range []string{"project", "api_url"} { + if _, ok := cred[key]; ok { + t.Fatalf("%s should be omitted so the plugin default applies: %+v", key, cred) + } + } +} + +func TestSetupAllPositionalKey(t *testing.T) { + claudeDir, codexDir := hermeticSetupEnv(t) + + if _, err := executeCommand(t, "setup", "all", "shared-key"); err != nil { + t.Fatalf("setup all error: %v", err) + } + + if env := claudeEnv(t, claudeDir); env["CC_LANGSMITH_API_KEY"] != "shared-key" { + t.Fatalf("claude settings not written by setup all: %+v", env) + } + cred := readJSONFile(t, filepath.Join(codexDir, "langsmith.json")) + if cred["api_key"] != "shared-key" { + t.Fatalf("codex credentials not written by setup all: %+v", cred) + } + if _, err := os.Stat(filepath.Join(codexDir, "config.toml")); err != nil { + t.Fatalf("codex config.toml not written by setup all: %v", err) + } +}