diff --git a/internal/client/client.go b/internal/client/client.go index 4f13138..ca7232a 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -139,6 +140,31 @@ type httpResponse struct { body []byte } +// HTTPError is returned by RawGet/RawPost/RawPatch/RawDelete when the server +// responds with a 4xx/5xx status. Its Error() text matches the plain +// "HTTP : " format these methods have always returned; the +// StatusCode field lets callers detect specific codes (e.g. 404) via +// errors.As instead of parsing the message. +type HTTPError struct { + StatusCode int + Body []byte +} + +func (e *HTTPError) Error() string { + return fmt.Sprintf("HTTP %d: %s", e.StatusCode, formatHTTPErrorBody(e.Body)) +} + +// IsNotFound reports whether err is an HTTPError with a 404 status. +func IsNotFound(err error) bool { + var httpErr *HTTPError + return errors.As(err, &httpErr) && httpErr.StatusCode == http.StatusNotFound +} + +func IsConflict(err error) bool { + var httpErr *HTTPError + return errors.As(err, &httpErr) && httpErr.StatusCode == http.StatusConflict +} + type httpErrorBody struct { Error string `json:"error"` Message string `json:"message"` @@ -146,6 +172,32 @@ type httpErrorBody struct { Detail any `json:"detail"` } +// rejectMethodChangingRedirect is the Client's http.Client.CheckRedirect. +// Go's default redirect policy silently downgrades POST/PATCH/PUT/DELETE to a +// bodyless GET on a 301/302/303 (per RFC 7231 — the POST/Redirect/GET +// pattern some servers use intentionally). LangSmith's API is a plain JSON +// REST surface with no such intentional redirects, so the only time this +// client would ever see one is a misconfigured endpoint scheme (http instead +// of https, redirected to https by the server). Following it silently turns +// a write into a no-op GET — sometimes still 2xx, if a same-path GET happens +// to exist — which is far worse than a clear error. Same-method redirects +// (a GET following an http->https upgrade, say) are harmless and still +// followed. +func rejectMethodChangingRedirect(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + prev := via[len(via)-1] + if req.Method != prev.Method { + return fmt.Errorf( + "refusing to follow redirect that would change %s %s into %s %s"+ + " (check LANGSMITH_ENDPOINT is using https, not http)", + prev.Method, prev.URL, req.Method, req.URL, + ) + } + return nil +} + // doHTTP is the shared low-level helper used by RawDo and rawRequest. func (c *Client) doHTTP(ctx context.Context, method, path string, body io.Reader, extraHeaders http.Header) (*httpResponse, error) { url := c.apiURL + path @@ -169,7 +221,10 @@ func (c *Client) doHTTP(ctx context.Context, method, path string, body io.Reader req.Header[k] = vals } - httpClient := &http.Client{Timeout: 30 * time.Second} + httpClient := &http.Client{ + Timeout: 30 * time.Second, + CheckRedirect: rejectMethodChangingRedirect, + } resp, err := httpClient.Do(req) if err != nil { return nil, fmt.Errorf("HTTP %s %s: %w", method, path, err) @@ -226,7 +281,7 @@ func (c *Client) rawRequest(ctx context.Context, method, path string, body any, } if resp.statusCode >= 400 { - return fmt.Errorf("HTTP %d: %s", resp.statusCode, formatHTTPErrorBody(resp.body)) + return &HTTPError{StatusCode: resp.statusCode, Body: resp.body} } if result != nil { diff --git a/internal/client/client_test.go b/internal/client/client_test.go index 41aeb26..274891e 100644 --- a/internal/client/client_test.go +++ b/internal/client/client_test.go @@ -632,3 +632,52 @@ func TestAPIURL(t *testing.T) { t.Errorf("expected http://localhost:1234, got %q", c.APIURL()) } } + +// ---------- Redirect handling ---------- + +func TestRawDo_RejectsMethodChangingRedirect(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/feedback" { + http.Redirect(w, r, "/feedback-redirected", http.StatusFound) // 302 + return + } + t.Errorf("redirect target should never be reached for a rejected redirect, got %s %s", r.Method, r.URL.Path) + })) + defer ts.Close() + + c := New("key", ts.URL) + _, _, _, _, err := c.RawDo(context.Background(), "POST", "/feedback", strings.NewReader(`{"key":"note"}`), nil) + if err == nil { + t.Fatal("expected an error, got nil") + } + if !strings.Contains(err.Error(), "would change") { + t.Errorf("expected error to mention the method change, got: %v", err) + } +} + +func TestRawDo_FollowsSameMethodRedirect(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/runs" { + http.Redirect(w, r, "/runs-redirected", http.StatusFound) // 302 + return + } + if r.Method != "GET" { + t.Errorf("expected GET on redirect target, got %s", r.Method) + } + w.WriteHeader(200) + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + defer ts.Close() + + c := New("key", ts.URL) + status, _, _, body, err := c.RawDo(context.Background(), "GET", "/runs", nil, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status != 200 { + t.Errorf("expected 200, got %d", status) + } + if string(body) != `{"ok":true}` { + t.Errorf("unexpected body: %s", body) + } +} diff --git a/internal/cmd/apps.go b/internal/cmd/apps.go new file mode 100644 index 0000000..3725c81 --- /dev/null +++ b/internal/cmd/apps.go @@ -0,0 +1,210 @@ +package cmd + +import ( + "bytes" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" +) + +// appsLinkDir and appsLinkFile name the local file that records which +// remote custom app a directory is linked to, mirroring how `vercel link`/ +// `netlify link` avoid needing a server-side unique name to upsert against. +const ( + appsLinkDir = ".langsmith" + appsLinkFile = "app.json" +) + +const ( + appsMaxFileEntries = 500 + appsMaxFileSizeBytes = 1 << 20 // 1MB per file, matches hub push's limit. +) + +// appLink is the contents of /.langsmith/app.json. It is the durable +// record of "what this app is" that `apps push` reads on every run to decide +// whether to create a new custom app or update the one it created last time. +type appLink struct { + AppID string `json:"app_id"` + Name string `json:"name"` +} + +// customApp mirrors the JSON shape returned by smith-go's +// /v1/platform/custom-apps endpoints (smith-go/customapps/types.go). +type customApp struct { + ID string `json:"id"` + TenantID string `json:"tenant_id,omitempty"` + Name string `json:"name"` + Description *string `json:"description,omitempty"` + Files map[string]string `json:"files,omitempty"` + Entrypoint string `json:"entrypoint"` + IsEnabled bool `json:"is_enabled"` + CreatedAt string `json:"created_at,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` + CreatedBy *string `json:"created_by,omitempty"` +} + +var appsExcludedDirs = map[string]bool{ + ".git": true, + "node_modules": true, + ".langsmith": true, + ".cache": true, + ".next": true, +} + +var appsExcludedFiles = map[string]bool{ + ".DS_Store": true, + "Thumbs.db": true, +} + +func newAppsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "apps", + Short: "Build, upload, and manage Custom Apps", + Long: `Build, upload, and manage Custom Apps — UIs you build locally against the +LangSmith API and run inside LangSmith. + +A custom app is a small set of files rendered in a locked-down sandbox: no +direct network access, no npm modules at runtime — only a postMessage +bridge (window.langsmith.call/setData) proxied through the viewer's own +LangSmith session. Use React/JSX or npm dependencies freely; the scaffolded +starter bundles them locally (Vite) into the single dependency-free file the +sandbox expects. + +Pick a starter with --template (blank, annotation-queue, +annotation-queue-grid); they differ only in the UI they scaffold — every app +is uploaded and rendered the same way. + +Examples: + langsmith apps init --name my-app + langsmith apps init --name my-queue-app --template annotation-queue + langsmith apps dev + langsmith apps push + langsmith apps list + langsmith apps delete --yes + +All of the above (except list/delete, which take an app ID) act on the +current directory — cd into your app's directory first.`, + } + cmd.AddCommand(newAppsInitCmd()) + cmd.AddCommand(newAppsDevCmd()) + cmd.AddCommand(newAppsPushCmd()) + cmd.AddCommand(newAppsListCmd()) + cmd.AddCommand(newAppsDeleteCmd()) + return cmd +} + +// readAppLink reads /.langsmith/app.json. It returns (nil, nil) if the +// file does not exist — callers treat that as "not linked yet". +func readAppLink(dir string) (*appLink, error) { + path := filepath.Join(dir, appsLinkDir, appsLinkFile) + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("reading %s: %w", path, err) + } + var link appLink + if err := json.Unmarshal(data, &link); err != nil { + return nil, fmt.Errorf("parsing %s: %w", path, err) + } + return &link, nil +} + +// writeAppLink writes /.langsmith/app.json, creating the directory if +// needed. +func writeAppLink(dir string, link appLink) error { + linkDir := filepath.Join(dir, appsLinkDir) + if err := os.MkdirAll(linkDir, 0o755); err != nil { + return fmt.Errorf("creating %s: %w", linkDir, err) + } + data, err := json.MarshalIndent(link, "", " ") + if err != nil { + return fmt.Errorf("marshaling app link: %w", err) + } + data = append(data, '\n') + path := filepath.Join(linkDir, appsLinkFile) + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("writing %s: %w", path, err) + } + return nil +} + +// readDirectoryAsAppFiles walks root and returns a flat relative-path → +// content map suitable for custom_apps.files. Unlike hub's equivalent +// (readDirectoryAsFiles in hub_push.go), there is no {type, content} +// wrapper — the custom-apps API takes a plain map[string]string — and +// .langsmith/ itself is always excluded so the link file never gets +// uploaded as an app file. +func readDirectoryAsAppFiles(root string) (map[string]string, error) { + info, err := os.Stat(root) + if err != nil { + return nil, fmt.Errorf("opening %s: %w", root, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("%s is not a directory", root) + } + + files := make(map[string]string) + err = filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + base := d.Name() + if d.IsDir() { + if appsExcludedDirs[base] { + return filepath.SkipDir + } + return nil + } + if d.Type()&fs.ModeSymlink != 0 { + return nil + } + if appsExcludedFiles[base] || strings.HasPrefix(base, ".env") { + return nil + } + + rel = filepath.ToSlash(rel) + + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %s: %w", rel, err) + } + if len(data) > appsMaxFileSizeBytes { + return fmt.Errorf("file %s is %d bytes; exceeds %d-byte limit", rel, len(data), appsMaxFileSizeBytes) + } + if isAppFileBinary(data) { + return fmt.Errorf("file %s appears to be binary; custom apps do not support binary assets yet", rel) + } + + files[rel] = string(data) + if len(files) > appsMaxFileEntries { + return fmt.Errorf("directory contains more than %d files; reduce before pushing", appsMaxFileEntries) + } + return nil + }) + if err != nil { + return nil, err + } + return files, nil +} + +func isAppFileBinary(data []byte) bool { + n := len(data) + if n > 512 { + n = 512 + } + return bytes.IndexByte(data[:n], 0) != -1 +} diff --git a/internal/cmd/apps_crud_test.go b/internal/cmd/apps_crud_test.go new file mode 100644 index 0000000..2447eb2 --- /dev/null +++ b/internal/cmd/apps_crud_test.go @@ -0,0 +1,72 @@ +package cmd + +import ( + "encoding/json" + "net/http" + "strings" + "testing" +) + +// Apps are uniform now — list takes no context_type filter and hits the +// endpoint with no query string. +func TestAppsList_ListsAllApps(t *testing.T) { + var sawQuery string + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + sawQuery = r.URL.RawQuery + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]customApp{ + {ID: "a1", Name: "one"}, + }) + }) + defer setupTestEnv(t, srv.URL)() + + out := captureStdout(t, func() { + cmd := newAppsCmd() + cmd.SetArgs([]string{"list"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v", err) + } + }) + + if sawQuery != "" { + t.Errorf("expected no query string (no context_type filter), got %q", sawQuery) + } + if !strings.Contains(out, `"id": "a1"`) { + t.Errorf("expected app in output:\n%s", out) + } + + // The context_type filter flag must be gone. + listCmd, _, err := newAppsCmd().Find([]string{"list"}) + if err != nil { + t.Fatalf("find list: %v", err) + } + if f := listCmd.Flags().Lookup("context-type"); f != nil { + t.Error("expected --context-type flag to be gone from apps list") + } +} + +func TestAppsDelete_SkipsConfirmationWithYes(t *testing.T) { + var sawDelete bool + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method != "DELETE" || r.URL.Path != "/v1/platform/custom-apps/a1" { + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + } + sawDelete = true + w.WriteHeader(http.StatusNoContent) + }) + defer setupTestEnv(t, srv.URL)() + + out := captureStdout(t, func() { + cmd := newAppsCmd() + cmd.SetArgs([]string{"delete", "a1", "--yes"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v", err) + } + }) + if !sawDelete { + t.Error("expected DELETE request") + } + if !strings.Contains(out, `"status": "deleted"`) { + t.Errorf("expected deleted status:\n%s", out) + } +} diff --git a/internal/cmd/apps_delete.go b/internal/cmd/apps_delete.go new file mode 100644 index 0000000..adf763d --- /dev/null +++ b/internal/cmd/apps_delete.go @@ -0,0 +1,48 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "github.com/langchain-ai/langsmith-cli/internal/output" + "github.com/spf13/cobra" +) + +func newAppsDeleteCmd() *cobra.Command { + var yes bool + + cmd := &cobra.Command{ + Use: "delete APP_ID", + Short: "Delete a custom app", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + id := args[0] + if !yes { + fmt.Fprintf(os.Stderr, "Delete custom app '%s'? [y/N] ", id) + var confirm string + _, _ = fmt.Scanln(&confirm) + if strings.ToLower(confirm) != "y" { + return errors.New("aborted") + } + } + + c := MustGetClient() + ctx := context.Background() + + if err := c.RawDelete(ctx, "/v1/platform/custom-apps/"+id, nil); err != nil { + return fmt.Errorf("deleting custom app %s: %w", id, err) + } + output.OutputJSON(map[string]any{ + "status": "deleted", + "app_id": id, + }, "") + return nil + }, + } + + cmd.Flags().BoolVar(&yes, "yes", false, "Skip confirmation prompt") + return cmd +} diff --git a/internal/cmd/apps_dev.go b/internal/cmd/apps_dev.go new file mode 100644 index 0000000..295b953 --- /dev/null +++ b/internal/cmd/apps_dev.go @@ -0,0 +1,796 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "html" + "io" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "os/signal" + "path/filepath" + "regexp" + "strings" + "syscall" + "time" + + "github.com/langchain-ai/langsmith-cli/internal/output" + "github.com/spf13/cobra" +) + +func newAppsDevCmd() *cobra.Command { + var ( + entrypoint string + noOpen bool + noWatch bool + ) + + cmd := &cobra.Command{ + Use: "dev", + Short: "Run the current directory's custom app locally in a real sandbox", + Long: `Serve the current directory's custom app inside a real sandboxed iframe +(sandbox="allow-scripts", no allow-same-origin — identical restrictions to +production) and open it in your browser. This is entirely self-contained: +no LangSmith web app involved. + +The app's window.langsmith.call(operation, args) is proxied to the real +LangSmith API using your local LANGSMITH_API_KEY — the same generic +passthrough production uses (proxied through the injected platform token +there instead). It is not a curated allowlist: operation is a +" " string (e.g. "GET /api/v1/annotation-queues/{id}/runs"), +forwarded as-is. + +If the current directory has a package.json with a "watch" script (both +starter templates do), this runs it for you automatically as a managed +child process, so editing source files and saving is enough to see the +change — no second terminal needed. The page itself polls for the +entrypoint changing and reloads whenever the watcher rebuilds it. Pass +--no-watch to skip this (e.g. you're already running your own build +process, or want to use a different one).`, + RunE: func(cmd *cobra.Command, args []string) error { + dir, err := os.Getwd() + if err != nil { + return fmt.Errorf("getting current directory: %w", err) + } + + if _, err := getClient(); err != nil { + return err + } + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + return runAppsDev(ctx, dir, entrypoint, noOpen, noWatch) + }, + } + + cmd.Flags().StringVar(&entrypoint, "entrypoint", "dist/bundle.js", "Path (relative to the current directory) of the file to render") + cmd.Flags().BoolVar(&noOpen, "no-open", false, "Print the local URL instead of opening a browser") + cmd.Flags().BoolVar(&noWatch, "no-watch", false, "Don't automatically run the build's \"watch\" script (e.g. \"npm run watch\") — use this if you're already running your own build process") + return cmd +} + +// runAppsDev starts the local sandboxed-preview server for dir, opens it in +// a browser, and blocks until ctx is cancelled (e.g. by SIGINT) or the +// server fails. +func runAppsDev(ctx context.Context, dir, entrypoint string, noOpen, noWatch bool) error { + srv, ln, previewURL, err := prepareAppsDevServer(dir, entrypoint) + if err != nil { + return err + } + + if !noWatch { + entrypointPath := filepath.Join(dir, filepath.FromSlash(entrypoint)) + var prevBuildTime time.Time + if info, statErr := os.Stat(entrypointPath); statErr == nil { + prevBuildTime = info.ModTime() + } + if startWatchProcess(ctx, dir) { + // Most build tools (including both starter templates' "vite + // build --watch") empty their output directory before their + // first rebuild. Without this, that would make an entrypoint we + // already had (e.g. from "apps init"'s one-time build) briefly + // disappear right as the browser opens, flashing the "waiting + // for build" page for no reason. Wait for a fresh build before + // continuing — the served page's own poll+reload is still + // there as a fallback if this build is slow. + waitForFreshEntrypoint(ctx, entrypointPath, prevBuildTime, 10*time.Second) + } + } + + serveErrCh := make(chan error, 1) + go func() { serveErrCh <- srv.Serve(ln) }() + + if !noOpen { + _ = openBrowser(previewURL) + } + output.OutputJSON(map[string]any{ + "status": "serving", + "url": previewURL, + }, "") + fmt.Fprintf(os.Stderr, "Serving %s at %s (sandboxed) — press Ctrl+C to stop\n", dir, previewURL) + + select { + case <-ctx.Done(): + case serveErr := <-serveErrCh: + if serveErr != nil && serveErr != http.ErrServerClosed { + return fmt.Errorf("local server: %w", serveErr) + } + } + + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 2*time.Second) + defer shutdownCancel() + if err := srv.Shutdown(shutdownCtx); err != nil && err != http.ErrServerClosed { + return fmt.Errorf("shutting down local server: %w", err) + } + return nil +} + +// startWatchProcess runs dir/package.json's "watch" script (e.g. +// "vite build --watch", aliased to "npm run watch" by both starter +// templates) as a child process tied to ctx, so editing source files +// rebuilds the entrypoint without a second terminal. It never fails +// runAppsDev — if there's no package.json, no "watch" script, or no npm on +// PATH, it just prints a note and returns false, leaving the existing +// run-your-own-build workflow intact. Returns true only if the process was +// actually started. +func startWatchProcess(ctx context.Context, dir string) bool { + script, pkgJSONExists, err := packageJSONScript(dir, "watch") + if err != nil { + fmt.Fprintf(os.Stderr, "warning: couldn't read package.json to find a \"watch\" script: %v\n", err) + return false + } + if script == "" { + if pkgJSONExists { + fmt.Fprintln(os.Stderr, `note: no "watch" script in package.json — start your own build/watch process to see live updates`) + } + return false + } + if _, lookErr := exec.LookPath("npm"); lookErr != nil { + fmt.Fprintln(os.Stderr, `note: npm not found on PATH — run "npm run watch" yourself to see live updates`) + return false + } + + fmt.Fprintln(os.Stderr, "Starting build watcher: npm run watch") + watchCmd := exec.CommandContext(ctx, "npm", "run", "watch") + watchCmd.Dir = dir + watchCmd.Stdout = os.Stderr + watchCmd.Stderr = os.Stderr + if startErr := watchCmd.Start(); startErr != nil { + fmt.Fprintf(os.Stderr, "warning: failed to start \"npm run watch\": %v\n", startErr) + return false + } + // Reap the process (and honor ctx cancellation, which exec.CommandContext + // enforces via Wait) without blocking runAppsDev on it. + go func() { _ = watchCmd.Wait() }() + return true +} + +// waitForFreshEntrypoint blocks until path's mtime is newer than after +// (meaning a build completed since we started watching), ctx is cancelled, +// or timeout elapses — whichever comes first. A zero-value after matches +// the first time path ever appears (no prior build to compare against). +func waitForFreshEntrypoint(ctx context.Context, path string, after time.Time, timeout time.Duration) { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if info, err := os.Stat(path); err == nil && info.ModTime().After(after) { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(100 * time.Millisecond): + } + } +} + +// packageJSONScript reads dir/package.json and returns the named script. +// pkgJSONExists distinguishes "no package.json at all" (nothing to warn +// about — this may not be an npm project) from "package.json exists but has +// no such script" (worth telling the user about). +func packageJSONScript(dir, name string) (script string, pkgJSONExists bool, err error) { + raw, readErr := os.ReadFile(filepath.Join(dir, "package.json")) + if readErr != nil { + if os.IsNotExist(readErr) { + return "", false, nil + } + return "", false, readErr + } + var pkg struct { + Scripts map[string]string `json:"scripts"` + } + if jsonErr := json.Unmarshal(raw, &pkg); jsonErr != nil { + return "", true, jsonErr + } + return pkg.Scripts[name], true, nil +} + +// prepareAppsDevServer builds (but does not start) an HTTP server, bound to +// an OS-assigned free port on 127.0.0.1, that serves: +// +// - "/" — a host page embedding the app in a real +// sandbox="allow-scripts" iframe (no allow-same-origin), re-read from +// disk on every request so a rebuild is reflected on the next reload. +// - "/__ls_dev/mtime" — polled by the host page to detect a rebuild (or +// the entrypoint appearing for the first time) and trigger a reload. +// - "/__ls_dev/call" — the generic window.langsmith.call(...) proxy, +// forwarding to the real LangSmith API via the authenticated CLI client. +// +// The directory is read the same way "apps push" reads it +// (readDirectoryAsAppFiles), so local dev sees exactly what would be +// uploaded — including the same .env/.git/node_modules exclusions. +func prepareAppsDevServer(dir, entrypoint string) (srv *http.Server, ln net.Listener, previewURL string, err error) { + mux := http.NewServeMux() + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Content-Type", "text/html; charset=utf-8") + + files, readErr := readDirectoryAsAppFiles(dir) + if readErr != nil { + _, _ = w.Write([]byte(devWaitingHTML(fmt.Sprintf("reading %s: %s", dir, readErr)))) + return + } + if _, ok := files[entrypoint]; !ok { + _, _ = w.Write([]byte(devWaitingHTML(fmt.Sprintf("entrypoint %q does not exist yet in %s — waiting for the initial build to finish", entrypoint, dir)))) + return + } + _, _ = w.Write([]byte(renderDevHostHTML(files, entrypoint))) + }) + + mux.HandleFunc("/__ls_dev/mtime", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Content-Type", "application/json") + info, statErr := os.Stat(filepath.Join(dir, filepath.FromSlash(entrypoint))) + if statErr != nil { + _ = json.NewEncoder(w).Encode(map[string]any{"exists": false}) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"exists": true, "mtime": info.ModTime().UnixNano()}) + }) + + mux.HandleFunc("/__ls_dev/call", handleLsDevCall) + + ln, err = net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, nil, "", fmt.Errorf("starting local server: %w", err) + } + return &http.Server{Handler: mux}, ln, "http://" + ln.Addr().String() + "/", nil +} + +var allowedProxyMethods = map[string]bool{ + "GET": true, "POST": true, "PATCH": true, "PUT": true, "DELETE": true, +} + +type lsDevCallRequest struct { + Operation string `json:"operation"` + Args lsDevCallArgs `json:"args"` +} + +type lsDevCallArgs struct { + Params map[string]any `json:"params,omitempty"` + Body any `json:"body,omitempty"` +} + +// handleLsDevCall is the local-dev twin of smith-frontend's apiProxy.ts +// createLangSmithApiProxy: a generic passthrough, not a curated allowlist. +// It forwards operation (" ") to the real LangSmith API using +// the CLI's authenticated client (the same one "langsmith api" uses), so an +// app can exercise real endpoints while it's being developed. +func handleLsDevCall(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + var req lsDevCallRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "invalid request body: "+err.Error(), http.StatusBadRequest) + return + } + + spaceIdx := strings.IndexByte(req.Operation, ' ') + if spaceIdx == -1 { + http.Error(w, fmt.Sprintf("invalid operation %q — expected \" \"", req.Operation), http.StatusBadRequest) + return + } + method := strings.ToUpper(req.Operation[:spaceIdx]) + path := req.Operation[spaceIdx+1:] + + if !allowedProxyMethods[method] { + http.Error(w, fmt.Sprintf("method %q is not permitted", method), http.StatusBadRequest) + return + } + if !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") || strings.Contains(path, "://") { + http.Error(w, fmt.Sprintf("path %q must be a relative path starting with \"/\"", path), http.StatusBadRequest) + return + } + if len(req.Args.Params) > 0 { + path += "?" + encodeProxyParams(req.Args.Params) + } + + var bodyReader io.Reader + if req.Args.Body != nil { + b, marshalErr := json.Marshal(req.Args.Body) + if marshalErr != nil { + http.Error(w, "encoding body: "+marshalErr.Error(), http.StatusInternalServerError) + return + } + bodyReader = bytes.NewReader(b) + } + + c, err := getClient() + if err != nil { + // Not MustGetClient: that exits the process, so one unauthenticated + // app call would kill the whole dev server. Return 502 instead. + http.Error(w, "not authenticated: "+err.Error(), http.StatusBadGateway) + return + } + status, _, respHeaders, respBody, err := c.RawDo(r.Context(), method, path, bodyReader, nil) + if err != nil { + http.Error(w, "request failed: "+err.Error(), http.StatusBadGateway) + return + } + if ct := respHeaders.Get("Content-Type"); ct != "" { + w.Header().Set("Content-Type", ct) + } + w.WriteHeader(status) + _, _ = w.Write(respBody) +} + +func encodeProxyParams(params map[string]any) string { + values := url.Values{} + for k, v := range params { + switch val := v.(type) { + case []any: + for _, item := range val { + values.Add(k, fmt.Sprintf("%v", item)) + } + default: + values.Set(k, fmt.Sprintf("%v", val)) + } + } + return values.Encode() +} + +var scriptCloseTagPattern = regexp.MustCompile(`(?i)`) + +func escapeForScript(jsonBytes []byte) string { + return scriptCloseTagPattern.ReplaceAllString(string(jsonBytes), "<\\/script>") +} + +// devWaitingHTML is served at "/" when the entrypoint doesn't exist yet (or +// the directory can't be read) — it polls the same mtime endpoint the real +// host page does, so it reloads itself into the real preview the moment a +// build produces the entrypoint. +func devWaitingHTML(message string) string { + return ` +Waiting for build… + +

` + html.EscapeString(message) + `

+ +` +} + +// renderDevHostHTML builds the top-level host page: it embeds the app in a +// real sandboxed iframe (mirroring smith-frontend's sandbox.ts +// buildSandboxSrcdoc as closely as possible, including the multi-file +// require() loader, so local dev exercises the exact same module-loading +// behavior as production) and implements the postMessage bridge from the +// host side — LANGSMITH_READY/LANGSMITH_HEIGHT/LS_API/LS_MUTATION — with +// LS_API forwarded to /__ls_dev/call, which is real network access the +// iframe itself can never have. +// +// Every app gets a config toolbar, labeled so it reads clearly as the dev +// tool's own chrome rather than part of the app below it; its first (and +// only) control is a Light/Dark mode toggle (posting LANGSMITH_METADATA). +func renderDevHostHTML(files map[string]string, entrypoint string) string { + filesJSON, _ := json.Marshal(files) + entrypointJSON, _ := json.Marshal(entrypoint) + + inner := strings.NewReplacer( + "__FILES_JSON__", escapeForScript(filesJSON), + "__ENTRYPOINT_JSON__", escapeForScript(entrypointJSON), + ).Replace(sandboxInnerHTMLTemplate) + + return strings.NewReplacer( + "__SANDBOX_SRCDOC__", html.EscapeString(inner), + ).Replace(devHostHTMLTemplate) +} + +const devHostHTMLTemplate = ` + + + +Local sandboxed preview + + + +
+ LangSmith Custom Apps — local preview below +
+
+ + +
+
+
+ + + +` + +// sandboxInnerHTMLTemplate is a close Go port of smith-frontend's +// sandbox.ts buildSandboxSrcdoc — same virtual filesystem + require() +// loader, same render(data, root) contract, same window.langsmith bridge +// (call/setData/feedback.create). Kept in sync by hand; there is no shared +// source between the two repos. +const sandboxInnerHTMLTemplate = ` + + + + + + + + +
+ + +` diff --git a/internal/cmd/apps_dev_test.go b/internal/cmd/apps_dev_test.go new file mode 100644 index 0000000..54f6f83 --- /dev/null +++ b/internal/cmd/apps_dev_test.go @@ -0,0 +1,570 @@ +package cmd + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestAppsDevCmd_Flags(t *testing.T) { + cmd := newAppsCmd() + dev, _, err := cmd.Find([]string{"dev"}) + if err != nil { + t.Fatalf("find dev command: %v", err) + } + // --queue-id is gone: apps are uniform now and the sandbox always gets {}. + for _, gone := range []string{"url", "web-url", "data", "queue-id"} { + if f := dev.Flags().Lookup(gone); f != nil { + t.Errorf("expected --%s flag to be gone from apps dev", gone) + } + } + if f := dev.Flags().Lookup("entrypoint"); f == nil || f.DefValue != "dist/bundle.js" { + t.Errorf("expected --entrypoint flag defaulting to dist/bundle.js, got %+v", f) + } + if f := dev.Flags().Lookup("no-open"); f == nil { + t.Error("expected --no-open flag to exist") + } + if f := dev.Flags().Lookup("no-watch"); f == nil { + t.Error("expected --no-watch flag to exist") + } +} + +func seedDevApp(t *testing.T, dir, entrypointContent string) string { + t.Helper() + if err := os.MkdirAll(filepath.Join(dir, "dist"), 0o755); err != nil { + t.Fatalf("mkdir dist: %v", err) + } + entrypointAbs := filepath.Join(dir, "dist", "bundle.js") + if err := os.WriteFile(entrypointAbs, []byte(entrypointContent), 0o644); err != nil { + t.Fatalf("seed bundle.js: %v", err) + } + return entrypointAbs +} + +func TestPrepareAppsDevServer_ServesRealSandboxedIframe(t *testing.T) { + dir := t.TempDir() + entrypointAbs := seedDevApp(t, dir, "module.exports = { render: function(d, r) { r.textContent = 'v1'; } }") + if err := os.WriteFile(filepath.Join(dir, ".env"), []byte("SUPER_SECRET_VALUE"), 0o644); err != nil { + t.Fatalf("seed .env: %v", err) + } + + srv, ln, previewURL, err := prepareAppsDevServer(dir, "dist/bundle.js") + if err != nil { + t.Fatalf("prepareAppsDevServer: %v", err) + } + go func() { _ = srv.Serve(ln) }() + defer func() { _ = srv.Close() }() + + resp, err := http.Get(previewURL) + if err != nil { + t.Fatalf("GET %s: %v", previewURL, err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + page := string(body) + + if !strings.Contains(page, `sandbox="allow-scripts"`) { + t.Errorf("expected a real sandbox=\"allow-scripts\" iframe:\n%s", page) + } + if strings.Contains(page, `sandbox="allow-scripts allow-same-origin"`) || + strings.Contains(page, `sandbox="allow-same-origin allow-scripts"`) { + t.Errorf("must never grant allow-same-origin — that defeats the sandbox:\n%s", page) + } + if !strings.Contains(page, "v1") { + t.Errorf("expected entrypoint content to be embedded in the sandboxed srcdoc:\n%s", page) + } + if strings.Contains(page, "SUPER_SECRET_VALUE") { + t.Errorf(".env content leaked into the served page:\n%s", page) + } + + // Rewriting the entrypoint must be reflected on the next request. + if err := os.WriteFile(entrypointAbs, []byte("module.exports = { render: function(d, r) { r.textContent = 'v2'; } }"), 0o644); err != nil { + t.Fatalf("rewrite bundle.js: %v", err) + } + resp2, err := http.Get(previewURL) + if err != nil { + t.Fatalf("GET %s (2nd): %v", previewURL, err) + } + defer func() { _ = resp2.Body.Close() }() + body2, _ := io.ReadAll(resp2.Body) + if !strings.Contains(string(body2), "v2") { + t.Error("expected rewritten entrypoint content to be reflected on next request") + } +} + +// A regression test for a real bug: renderApp() used to unconditionally do +// `root.innerHTML = ”` before calling window.__render(currentData, root). +// The entrypoint caches its React root across calls (`if (!root) root = +// createRoot(rootEl)`) specifically so repeat renders (e.g. a theme toggle +// re-posting LANGSMITH_METADATA) reconcile against the same tree. Wiping the +// DOM out from under React between calls corrupts that reconciliation — +// harmless on the very first render (the container starts empty anyway), but +// the app renders blank on every render after that. +func TestPrepareAppsDevServer_DoesNotClearRootBeforeSuccessfulRender(t *testing.T) { + dir := t.TempDir() + seedDevApp(t, dir, "module.exports = { render: function(d, r) { r.textContent = 'ok'; } }") + + srv, ln, previewURL, err := prepareAppsDevServer(dir, "dist/bundle.js") + if err != nil { + t.Fatalf("prepareAppsDevServer: %v", err) + } + go func() { _ = srv.Serve(ln) }() + defer func() { _ = srv.Close() }() + + resp, err := http.Get(previewURL) + if err != nil { + t.Fatalf("GET %s: %v", previewURL, err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + page := string(body) + + if strings.Contains(page, "root.innerHTML = '';") { + t.Errorf("renderApp() must not unconditionally clear #root before calling window.__render — it corrupts the entrypoint's cached React root on every render after the first:\n%s", page) + } +} + +// The config toolbar (and its Light/Dark mode toggle) renders for every app. +func TestPrepareAppsDevServer_ServesModeToggle(t *testing.T) { + dir := t.TempDir() + seedDevApp(t, dir, "module.exports = { render: function(){} }") + + srv, ln, previewURL, err := prepareAppsDevServer(dir, "dist/bundle.js") + if err != nil { + t.Fatalf("prepareAppsDevServer: %v", err) + } + go func() { _ = srv.Serve(ln) }() + defer func() { _ = srv.Close() }() + + resp, err := http.Get(previewURL) + if err != nil { + t.Fatalf("GET %s: %v", previewURL, err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + page := string(body) + + if !strings.Contains(page, `id="ls-dev-mode-light"`) || !strings.Contains(page, `id="ls-dev-mode-dark"`) { + t.Errorf("every app should get always-visible Light/Dark buttons:\n%s", page) + } + // The queue selector is gone entirely now. + if strings.Contains(page, `id="ls-dev-queue-select"`) { + t.Errorf("the queue selector should no longer be served:\n%s", page) + } + // On READY the host posts LANGSMITH_METADATA, and the toggle re-posts it. + if !strings.Contains(page, "LANGSMITH_METADATA") { + t.Errorf("expected the host page to post LANGSMITH_METADATA:\n%s", page) + } + // The served page embeds the sandbox srcdoc; the 3-arg render call has no + // characters html.EscapeString would touch, so it appears verbatim. + if !strings.Contains(page, "window.__render(currentData, root, currentMetadata)") { + t.Errorf("expected the sandbox srcdoc to call render with (data, root, metadata):\n%s", page) + } +} + +// The sandbox side must implement the pinned theme/metadata contract: handle +// LANGSMITH_METADATA, set html.dark from mode, and pass metadata as render's +// 3rd arg. Asserting on the template constant avoids fighting the HTML-escaping +// the srcdoc goes through when embedded in the host page. +func TestSandboxImplementsThemeMetadataContract(t *testing.T) { + for _, want := range []string{ + "LANGSMITH_METADATA", + "currentMetadata = msg.metadata", + "classList.toggle('dark', currentMetadata && currentMetadata.mode === 'dark')", + "window.__render(currentData, root, currentMetadata)", + // First render gates on all three of theme, metadata, and data. + "themeReady && currentMetadata !== null && currentData !== null", + } { + if !strings.Contains(sandboxInnerHTMLTemplate, want) { + t.Errorf("sandboxInnerHTMLTemplate missing %q", want) + } + } +} + +func TestPrepareAppsDevServer_ServesWaitingPageWhenEntrypointMissing(t *testing.T) { + dir := t.TempDir() + srv, ln, previewURL, err := prepareAppsDevServer(dir, "dist/bundle.js") + if err != nil { + t.Fatalf("prepareAppsDevServer: %v", err) + } + go func() { _ = srv.Serve(ln) }() + defer func() { _ = srv.Close() }() + + resp, err := http.Get(previewURL) + if err != nil { + t.Fatalf("GET %s: %v", previewURL, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Errorf("expected 200 (a friendly waiting page, not an error status), got %d", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + if !strings.Contains(string(body), "/__ls_dev/mtime") { + t.Error("expected the waiting page to poll for the entrypoint appearing") + } +} + +func TestPrepareAppsDevServer_MtimeReflectsFileState(t *testing.T) { + dir := t.TempDir() + srv, ln, previewURL, err := prepareAppsDevServer(dir, "dist/bundle.js") + if err != nil { + t.Fatalf("prepareAppsDevServer: %v", err) + } + go func() { _ = srv.Serve(ln) }() + defer func() { _ = srv.Close() }() + + mtimeURL := strings.TrimSuffix(previewURL, "/") + "/__ls_dev/mtime" + get := func() (bool, int64) { + resp, err := http.Get(mtimeURL) + if err != nil { + t.Fatalf("GET %s: %v", mtimeURL, err) + } + defer func() { _ = resp.Body.Close() }() + var body struct { + Exists bool `json:"exists"` + MTime int64 `json:"mtime"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode: %v", err) + } + return body.Exists, body.MTime + } + + if exists, _ := get(); exists { + t.Error("expected exists=false before the entrypoint is built") + } + + seedDevApp(t, dir, "module.exports = { render: function(){} }") + exists, mtime := get() + if !exists || mtime == 0 { + t.Errorf("expected exists=true and nonzero mtime once built, got exists=%v mtime=%d", exists, mtime) + } +} + +func TestHandleLsDevCall_ForwardsOperationToRealClient(t *testing.T) { + var sawMethod, sawPath, sawBody string + upstream := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + sawMethod = r.Method + sawPath = r.URL.Path + "?" + r.URL.RawQuery + b, _ := io.ReadAll(r.Body) + sawBody = string(b) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id": "fb_1"}`)) + }) + defer setupTestEnv(t, upstream.URL)() + + reqBody := `{"operation":"POST /api/v1/feedback","args":{"body":{"key":"correctness","score":1}}}` + req := httptest.NewRequest(http.MethodPost, "/__ls_dev/call", strings.NewReader(reqBody)) + rec := httptest.NewRecorder() + handleLsDevCall(rec, req) + + if rec.Code != http.StatusCreated { + t.Fatalf("expected 201 passed through, got %d: %s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), `"fb_1"`) { + t.Errorf("expected upstream response body passed through, got %s", rec.Body.String()) + } + if sawMethod != "POST" { + t.Errorf("expected upstream POST, got %s", sawMethod) + } + if !strings.HasPrefix(sawPath, "/api/v1/feedback") { + t.Errorf("expected upstream path /api/v1/feedback, got %s", sawPath) + } + if !strings.Contains(sawBody, `"correctness"`) { + t.Errorf("expected request body forwarded, got %s", sawBody) + } +} + +func TestHandleLsDevCall_ForwardsParamsAsQueryString(t *testing.T) { + var sawQuery string + upstream := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + sawQuery = r.URL.RawQuery + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[]`)) + }) + defer setupTestEnv(t, upstream.URL)() + + reqBody := `{"operation":"GET /api/v1/annotation-queues/q1/runs","args":{"params":{"status":"needs_my_review"}}}` + req := httptest.NewRequest(http.MethodPost, "/__ls_dev/call", strings.NewReader(reqBody)) + rec := httptest.NewRecorder() + handleLsDevCall(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", rec.Code, rec.Body.String()) + } + if sawQuery != "status=needs_my_review" { + t.Errorf("expected query string forwarded, got %q", sawQuery) + } +} + +func TestHandleLsDevCall_RejectsMalformedOperation(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/__ls_dev/call", strings.NewReader(`{"operation":"nospacehere"}`)) + rec := httptest.NewRecorder() + handleLsDevCall(rec, req) + if rec.Code != http.StatusBadRequest { + t.Errorf("expected 400 for malformed operation, got %d", rec.Code) + } +} + +func TestHandleLsDevCall_RejectsDisallowedMethod(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/__ls_dev/call", strings.NewReader(`{"operation":"TRACE /api/v1/runs"}`)) + rec := httptest.NewRecorder() + handleLsDevCall(rec, req) + if rec.Code != http.StatusBadRequest { + t.Errorf("expected 400 for disallowed method, got %d", rec.Code) + } +} + +func TestHandleLsDevCall_RejectsPathEscapingOrigin(t *testing.T) { + for _, op := range []string{ + "GET https://evil.example.com/steal", + "GET //evil.example.com/steal", + "GET not-a-path", + } { + req := httptest.NewRequest(http.MethodPost, "/__ls_dev/call", strings.NewReader(`{"operation":"`+op+`"}`)) + rec := httptest.NewRecorder() + handleLsDevCall(rec, req) + if rec.Code != http.StatusBadRequest { + t.Errorf("operation %q: expected 400, got %d", op, rec.Code) + } + } +} + +func writeFile(t *testing.T, dir, name, content string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } +} + +func TestPackageJSONScript_NoPackageJSON(t *testing.T) { + dir := t.TempDir() + script, exists, err := packageJSONScript(dir, "watch") + if err != nil || exists || script != "" { + t.Errorf("got script=%q exists=%v err=%v, want \"\", false, nil", script, exists, err) + } +} + +func TestPackageJSONScript_ExistsWithoutTheScript(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "package.json", `{"scripts":{"build":"vite build"}}`) + script, exists, err := packageJSONScript(dir, "watch") + if err != nil || !exists || script != "" { + t.Errorf("got script=%q exists=%v err=%v, want \"\", true, nil", script, exists, err) + } +} + +func TestPackageJSONScript_ReturnsTheScript(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "package.json", `{"scripts":{"watch":"vite build --watch"}}`) + script, exists, err := packageJSONScript(dir, "watch") + if err != nil || !exists || script != "vite build --watch" { + t.Errorf("got script=%q exists=%v err=%v, want \"vite build --watch\", true, nil", script, exists, err) + } +} + +func TestPackageJSONScript_MalformedJSON(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "package.json", `{not json`) + if _, _, err := packageJSONScript(dir, "watch"); err == nil { + t.Error("expected an error for malformed package.json") + } +} + +// fakeNpmOnPath puts a fake "npm" script on PATH (for the duration of the +// test) that touches markerPath and then blocks, so tests can assert +// startWatchProcess actually launched "npm run watch" without depending on +// npm or a real bundler being installed in the test environment. +func fakeNpmOnPath(t *testing.T, markerPath string) { + t.Helper() + fakeNpmDir := t.TempDir() + script := "#!/bin/sh\ntouch \"" + markerPath + "\"\nwhile true; do sleep 0.1; done\n" + if err := os.WriteFile(filepath.Join(fakeNpmDir, "npm"), []byte(script), 0o755); err != nil { + t.Fatalf("write fake npm: %v", err) + } + t.Setenv("PATH", fakeNpmDir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func TestStartWatchProcess_SpawnsWatchScriptAndIsKilledOnContextCancel(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "package.json", `{"scripts":{"watch":"anything"}}`) + marker := filepath.Join(dir, "watch-ran.marker") + fakeNpmOnPath(t, marker) + + ctx, cancel := context.WithCancel(context.Background()) + if started := startWatchProcess(ctx, dir); !started { + t.Fatal("expected startWatchProcess to report started=true") + } + + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(marker); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("watch script never ran (marker file not created)") + } + time.Sleep(20 * time.Millisecond) + } + + // Cancelling ctx should kill the long-running fake npm process rather + // than leaving it (and the test process) hanging. + cancel() + time.Sleep(200 * time.Millisecond) +} + +func TestStartWatchProcess_NoWatchScriptDoesNotSpawn(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "package.json", `{"scripts":{"build":"vite build"}}`) + marker := filepath.Join(dir, "watch-ran.marker") + fakeNpmOnPath(t, marker) + + if started := startWatchProcess(context.Background(), dir); started { + t.Error("expected started=false when package.json has no \"watch\" script") + } + time.Sleep(100 * time.Millisecond) + + if _, err := os.Stat(marker); err == nil { + t.Error("expected no process to be spawned when package.json has no \"watch\" script") + } +} + +func TestStartWatchProcess_NoPackageJSONDoesNotSpawn(t *testing.T) { + dir := t.TempDir() + marker := filepath.Join(dir, "watch-ran.marker") + fakeNpmOnPath(t, marker) + + if started := startWatchProcess(context.Background(), dir); started { + t.Error("expected started=false when there's no package.json") + } + time.Sleep(100 * time.Millisecond) + + if _, err := os.Stat(marker); err == nil { + t.Error("expected no process to be spawned when there's no package.json") + } +} + +func TestWaitForFreshEntrypoint_ReturnsOnceFileFirstAppears(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "dist", "bundle.js") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + + go func() { + time.Sleep(150 * time.Millisecond) + _ = os.WriteFile(path, []byte("v1"), 0o644) + }() + + start := time.Now() + waitForFreshEntrypoint(context.Background(), path, time.Time{}, 5*time.Second) + if elapsed := time.Since(start); elapsed < 100*time.Millisecond { + t.Errorf("returned too early (%v) — should have waited for the file to appear", elapsed) + } + if _, err := os.Stat(path); err != nil { + t.Error("expected the file to exist once waitForFreshEntrypoint returns") + } +} + +func TestWaitForFreshEntrypoint_WaitsForNewerMTimeThanPrevBuild(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "dist", "bundle.js") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(path, []byte("stale"), 0o644); err != nil { + t.Fatalf("seed stale build: %v", err) + } + staleInfo, err := os.Stat(path) + if err != nil { + t.Fatalf("stat: %v", err) + } + prevBuildTime := staleInfo.ModTime() + + // Simulate the watch tool's "empty outDir, then rebuild" sequence: the + // file briefly disappears, then reappears with a newer mtime. + go func() { + time.Sleep(50 * time.Millisecond) + _ = os.Remove(path) + time.Sleep(100 * time.Millisecond) + _ = os.WriteFile(path, []byte("fresh"), 0o644) + }() + + done := make(chan struct{}) + go func() { + waitForFreshEntrypoint(context.Background(), path, prevBuildTime, 5*time.Second) + close(done) + }() + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("waitForFreshEntrypoint did not return once a fresh build appeared") + } + + info, err := os.Stat(path) + if err != nil || !info.ModTime().After(prevBuildTime) { + t.Error("expected the entrypoint's mtime to be newer than prevBuildTime once returned") + } +} + +func TestWaitForFreshEntrypoint_GivesUpAfterTimeout(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "dist", "bundle.js") // never created + + start := time.Now() + waitForFreshEntrypoint(context.Background(), path, time.Time{}, 200*time.Millisecond) + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Errorf("expected to give up close to the timeout, took %v", elapsed) + } +} + +func TestWaitForFreshEntrypoint_ReturnsOnContextCancel(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "dist", "bundle.js") // never created + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + start := time.Now() + waitForFreshEntrypoint(ctx, path, time.Time{}, 5*time.Second) + if elapsed := time.Since(start); elapsed > 1*time.Second { + t.Errorf("expected an already-cancelled context to return immediately, took %v", elapsed) + } +} + +func TestRunAppsDev_ExitsOnContextCancel(t *testing.T) { + dir := t.TempDir() + seedDevApp(t, dir, "hi") + + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + errCh <- runAppsDev(ctx, dir, "dist/bundle.js", true, true) + }() + + time.Sleep(100 * time.Millisecond) + cancel() + + select { + case err := <-errCh: + if err != nil { + t.Errorf("runAppsDev returned error: %v", err) + } + case <-time.After(5 * time.Second): + t.Error("runAppsDev did not exit after context cancel") + } +} diff --git a/internal/cmd/apps_init.go b/internal/cmd/apps_init.go new file mode 100644 index 0000000..a5e8f50 --- /dev/null +++ b/internal/cmd/apps_init.go @@ -0,0 +1,482 @@ +package cmd + +import ( + "embed" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "text/template" + + "github.com/langchain-ai/langsmith-cli/internal/output" + "github.com/spf13/cobra" +) + +// The all: prefix makes embed.FS include dotfiles (.gitignore) and _-prefixed +// files; plain embed drops any path segment starting with "." or "_". +// +//go:embed all:templates/blank +var blankStarterFS embed.FS + +//go:embed all:templates/annotation-queue +var annotationQueueStarterFS embed.FS + +//go:embed all:templates/annotation-queue-grid +var annotationQueueGridStarterFS embed.FS + +//go:embed all:templates/coding-agent-dashboard +var codingAgentDashboardStarterFS embed.FS + +//go:embed all:templates/experiment-comparison +var experimentComparisonStarterFS embed.FS + +//go:embed all:templates/agents-md +var agentsMDFS embed.FS + +// Canonical copies of small files reused across templates (SearchableSelect, +// Spinner, the cn() helper, ...). Templates don't carry their own copies — +// scaffoldCustomAppStarter infers which ones a given template needs by +// scanning its own source for the import statements that would reference +// them (see sharedFilesUsedBy), so there's no per-template list to maintain. +// This keeps the *source* single-copy while every scaffolded app still ends +// up with its own real, standalone file — nothing in the generated output +// depends on this repo or on other templates at runtime. +// +//go:embed all:templates/_shared +var sharedFS embed.FS + +const sharedRoot = "templates/_shared" + +// appType is one --template choice: which starter to scaffold and which +// AGENTS.md ships with it. Map key = the --template value. +type appType struct { + templateFS embed.FS + templateRoot string + agentsMD string // template-specific AGENTS.md fragment; "" = none (generic base only) +} + +var appTypes = map[string]appType{ + "blank": { + templateFS: blankStarterFS, + templateRoot: "templates/blank", + agentsMD: "", + }, + "annotation-queue": { + templateFS: annotationQueueStarterFS, + templateRoot: "templates/annotation-queue", + agentsMD: "annotation_queue", + }, + "annotation-queue-grid": { + templateFS: annotationQueueGridStarterFS, + templateRoot: "templates/annotation-queue-grid", + agentsMD: "annotation_queue_grid", + }, + "coding-agent-dashboard": { + templateFS: codingAgentDashboardStarterFS, + templateRoot: "templates/coding-agent-dashboard", + agentsMD: "coding_agent_dashboard", + }, + "experiment-comparison": { + templateFS: experimentComparisonStarterFS, + templateRoot: "templates/experiment-comparison", + agentsMD: "experiment_comparison", + }, +} + +// appTypeNames returns the valid --template values, sorted, keeping error +// text and help in sync with the map. +func appTypeNames() []string { + names := make([]string, 0, len(appTypes)) + for name := range appTypes { + if name == "blank" { + continue + } + names = append(names, name) + } + sort.Strings(names) + return names +} + +type customAppStarterVars struct { + Name string + Description string +} + +func newAppsInitCmd() *cobra.Command { + var ( + name string + description string + templateFlag string + force bool + skipInstall bool + ) + + cmd := &cobra.Command{ + Use: "init --name NAME [--template annotation-queue|annotation-queue-grid|coding-agent-dashboard|experiment-comparison]", + Short: "Scaffold a starter custom app in the current directory", + Long: `Scaffold a starter custom app in the current directory. + +--template picks which starter gets scaffolded; omit it for a blank +single-file starter. Every app is uploaded and rendered the same way +regardless of choice. + + annotation-queue A real, working queue-review UI (run list, + inputs/outputs viewer, feedback rubric, reviewer + notes). It picks a queue itself and fetches everything + from the LangSmith API. + annotation-queue-grid + Same queue-review workflow as annotation-queue, but + rendered as a spreadsheet: rows are queue runs, columns + are rubric keys, cells edited inline, Done per row. + coding-agent-dashboard + A charts dashboard over coding-agent runs: pick a + project (or scan all projects), see integration/model + share, token/cost/cache economics, tool and subagent + usage, errors, activity over time, and per-thread + breakdowns. + experiment-comparison + Compare evaluation experiments: pick a dataset, a + baseline, and comparisons; see aggregates and a + per-example table colored vs the baseline. + +This also writes an AGENTS.md describing the LangSmith API +surface available to this app, and a README explaining the bridge contract. +This only writes local files — it does not create anything remotely. Run +"langsmith apps push" once you're ready to upload it. + +By default this also runs "npm install" and "npm run build" in the new +directory, so "langsmith apps dev" has a dist/bundle.js to serve right away +instead of 404ing until you build it yourself. Pass --skip-install to just +write the files. A failed install/build doesn't fail the command — it's a +convenience, not a requirement — but you'll need to build manually before +"apps dev" or "apps push" will work.`, + RunE: func(cmd *cobra.Command, args []string) error { + templateName := templateFlag + if templateName == "" { + templateName = "blank" + } else if templateName == "blank" { + return fmt.Errorf("--template must be one of: %s", strings.Join(appTypeNames(), ", ")) + } + at, ok := appTypes[templateName] + if !ok { + return fmt.Errorf("--template must be one of: %s", strings.Join(appTypeNames(), ", ")) + } + dir, err := os.Getwd() + if err != nil { + return fmt.Errorf("getting current directory: %w", err) + } + written, err := scaffoldCustomAppStarter(dir, name, description, at, force) + if err != nil { + return err + } + sort.Strings(written) + + built := false + if !skipInstall { + if buildErr := installAndBuildCustomAppStarter(dir); buildErr != nil { + fmt.Fprintf(os.Stderr, "warning: automatic \"npm install && npm run build\" failed: %v\n(run those yourself — or \"npm run watch\" — before \"langsmith apps dev\"/\"apps push\")\n", buildErr) + } else { + built = true + } + } + + output.OutputJSON(map[string]any{ + "status": "scaffolded", + "dir": dir, + "name": name, + "template": templateName, + "files": written, + "built": built, + }, "") + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name for the app, written into package.json/README (required)") + cmd.Flags().StringVar(&description, "description", "", "One-line description written into README.md") + cmd.Flags().StringVar(&templateFlag, "template", "", "Starter template: "+strings.Join(appTypeNames(), ", ")+" (omit for a blank starter)") + cmd.Flags().BoolVar(&force, "force", false, "Write even if the current directory is non-empty") + cmd.Flags().BoolVar(&skipInstall, "skip-install", false, "Skip running \"npm install && npm run build\" after scaffolding") + _ = cmd.MarkFlagRequired("name") + return cmd +} + +// installAndBuildCustomAppStarter runs "npm install" then "npm run build" in +// dir, so a freshly scaffolded app has a dist/bundle.js immediately instead +// of only after the user remembers to build it themselves. +func installAndBuildCustomAppStarter(dir string) error { + if _, err := exec.LookPath("npm"); err != nil { + return fmt.Errorf("npm not found on PATH") + } + if err := runInDir(dir, "npm", "install"); err != nil { + return fmt.Errorf("npm install: %w", err) + } + if err := runInDir(dir, "npm", "run", "build"); err != nil { + return fmt.Errorf("npm run build: %w", err) + } + return nil +} + +func runInDir(dir, name string, args ...string) error { + c := exec.Command(name, args...) + c.Dir = dir + c.Stdout = os.Stderr + c.Stderr = os.Stderr + return c.Run() +} + +func scaffoldCustomAppStarter(dir, name, description string, at appType, force bool) ([]string, error) { + if name == "" { + return nil, fmt.Errorf("--name is required") + } + if at.templateRoot == "" { + return nil, fmt.Errorf("--template must be one of: %s", strings.Join(appTypeNames(), ", ")) + } + + if info, err := os.Stat(dir); err == nil { + if !info.IsDir() { + return nil, fmt.Errorf("%s exists and is not a directory", dir) + } + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", dir, err) + } + if len(entries) > 0 && !force { + return nil, fmt.Errorf("%s is not empty; pass --force to write anyway", dir) + } + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("stat %s: %w", dir, err) + } + + vars := customAppStarterVars{Name: name, Description: description} + if vars.Description == "" { + vars.Description = "TODO: one-sentence description of what this app does." + } + + var written []string + err := fs.WalkDir(at.templateFS, at.templateRoot, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + rel := strings.TrimPrefix(path, at.templateRoot+"/") + if rel == at.templateRoot || d.IsDir() { + return nil + } + + raw, err := at.templateFS.ReadFile(path) + if err != nil { + return fmt.Errorf("reading embedded template %s: %w", rel, err) + } + + dest := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return fmt.Errorf("creating directory for %s: %w", rel, err) + } + + // Only README.md/package.json actually reference {{.Name}}/{{.Description}}. + // Running every file through text/template would also try to parse any + // literal "{{"/"}}" elsewhere as template actions — e.g. React's + // style={{...}} inline-style syntax — and fail. Copy everything else + // byte-for-byte. + if !templatedStarterFiles[rel] { + if err := os.WriteFile(dest, raw, 0o644); err != nil { + return fmt.Errorf("writing %s: %w", rel, err) + } + written = append(written, rel) + return nil + } + + tmpl, err := template.New(rel).Parse(string(raw)) + if err != nil { + return fmt.Errorf("parsing template %s: %w", rel, err) + } + f, err := os.Create(dest) + if err != nil { + return fmt.Errorf("creating %s: %w", rel, err) + } + defer f.Close() + if err := tmpl.Execute(f, vars); err != nil { + return fmt.Errorf("rendering %s: %w", rel, err) + } + + written = append(written, rel) + return nil + }) + if err != nil { + return nil, err + } + + sharedWritten, err := writeUsedSharedFiles(dir, at) + if err != nil { + return nil, err + } + written = append(written, sharedWritten...) + + agentsMD, err := assembleAgentsMD(at.agentsMD) + if err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(dir, "AGENTS.md"), agentsMD, 0o644); err != nil { + return nil, fmt.Errorf("writing AGENTS.md: %w", err) + } + written = append(written, "AGENTS.md") + + // Record the name now so a later push reuses it, not the directory basename. + if err := writeAppLink(dir, appLink{Name: name}); err != nil { + return nil, fmt.Errorf("writing .langsmith/app.json: %w", err) + } + written = append(written, filepath.Join(appsLinkDir, appsLinkFile)) + + return written, nil +} + +// writeUsedSharedFiles copies whichever templates/_shared/ files this +// template's own source actually imports into dir/src/, so the scaffolded +// app ends up with a real, standalone file — the shared/ source stays the +// only physical copy in this repo. "Actually imports" is determined by +// scanning the template's own .ts/.tsx source for the relative-import +// specifier a file at that shared path would be referenced by (see +// sharedFileImportSpecifiers), not by a maintained per-template list. +func writeUsedSharedFiles(dir string, at appType) ([]string, error) { + sharedPaths, err := allSharedFilePaths() + if err != nil { + return nil, err + } + if len(sharedPaths) == 0 { + return nil, nil + } + + src, err := concatenatedTemplateSource(at) + if err != nil { + return nil, err + } + + var written []string + for _, relPath := range sharedPaths { + used := false + for _, spec := range sharedFileImportSpecifiers(relPath) { + if strings.Contains(src, "'"+spec+"'") || strings.Contains(src, "\""+spec+"\"") { + used = true + break + } + } + if !used { + continue + } + + raw, err := sharedFS.ReadFile(sharedRoot + "/" + relPath) + if err != nil { + return nil, fmt.Errorf("reading embedded shared file %s: %w", relPath, err) + } + destRel := filepath.Join("src", relPath) + dest := filepath.Join(dir, destRel) + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return nil, fmt.Errorf("creating directory for %s: %w", destRel, err) + } + if err := os.WriteFile(dest, raw, 0o644); err != nil { + return nil, fmt.Errorf("writing %s: %w", destRel, err) + } + written = append(written, destRel) + } + return written, nil +} + +// allSharedFilePaths lists every file under templates/_shared/, relative to +// that root (e.g. "components/SearchableSelect.tsx", "lib/utils.ts"). +func allSharedFilePaths() ([]string, error) { + var paths []string + err := fs.WalkDir(sharedFS, sharedRoot, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + paths = append(paths, strings.TrimPrefix(path, sharedRoot+"/")) + return nil + }) + if err != nil { + return nil, fmt.Errorf("walking embedded shared files: %w", err) + } + return paths, nil +} + +// concatenatedTemplateSource returns every .ts/.tsx file in this template +// concatenated together, for a simple substring scan — cheap and sufficient +// at this repo's scale (a handful of small, flat template directories). +func concatenatedTemplateSource(at appType) (string, error) { + var out strings.Builder + err := fs.WalkDir(at.templateFS, at.templateRoot, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() || !(strings.HasSuffix(path, ".ts") || strings.HasSuffix(path, ".tsx")) { + return nil + } + b, err := at.templateFS.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + out.Write(b) + out.WriteByte('\n') + return nil + }) + if err != nil { + return "", err + } + return out.String(), nil +} + +// sharedFileImportSpecifiers returns the plausible relative-import +// specifiers a template file could use to reference the shared file at +// sharedRelPath (relative to templates/_shared/), depending on which +// directory within the template's src/ the importing file lives in. Matches +// this repo's template convention: everything lives directly under src/, +// src/components/, or src/lib/, one level deep — e.g. a file at +// "components/SearchableSelect.tsx" is imported as "./SearchableSelect" by +// sibling files under src/components/, or "./components/SearchableSelect" +// by a top-level file like src/App.tsx. +func sharedFileImportSpecifiers(sharedRelPath string) []string { + base := strings.TrimSuffix(filepath.Base(sharedRelPath), filepath.Ext(sharedRelPath)) + dir := filepath.Dir(sharedRelPath) // "components", "lib", or "." for a top-level shared file + specifiers := []string{"./" + base} + if dir != "." { + specifiers = append(specifiers, + "./"+dir+"/"+base, // top-level src/*.tsx importing src// + "../"+dir+"/"+base, // a different src// file importing src// + ) + } + return specifiers +} + +var templatedStarterFiles = map[string]bool{ + "README.md": true, + "package.json": true, +} + +// assembleAgentsMD builds an app's AGENTS.md from the generic base +// (_common.md) with the template-specific fragment injected at the marker. +// A blank template (agentsMD == "") yields the base alone. +func assembleAgentsMD(agentsMD string) ([]byte, error) { + const marker = "" + common, err := agentsMDFS.ReadFile("templates/agents-md/_common.md") + if err != nil { + return nil, fmt.Errorf("reading embedded _common.md: %w", err) + } + fragment := "" + if agentsMD != "" { + b, err := agentsMDFS.ReadFile("templates/agents-md/" + agentsMD + ".md") + if err != nil { + return nil, fmt.Errorf("reading embedded AGENTS.md fragment %q: %w", agentsMD, err) + } + fragment = strings.TrimSpace(string(b)) + } + out := strings.Replace(string(common), marker, fragment, 1) + for strings.Contains(out, "\n\n\n") { + out = strings.ReplaceAll(out, "\n\n\n", "\n\n") + } + return []byte(strings.TrimSpace(out) + "\n"), nil +} diff --git a/internal/cmd/apps_init_test.go b/internal/cmd/apps_init_test.go new file mode 100644 index 0000000..d71a9af --- /dev/null +++ b/internal/cmd/apps_init_test.go @@ -0,0 +1,734 @@ +package cmd + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +// "apps init" records the name in .langsmith/app.json so a later push reuses +// it instead of the directory basename. +func TestAppsInit_WritesPartialAppLinkForImmediateAppsDevUse(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "my-app") + + if _, err := scaffoldCustomAppStarter(target, "my-app", "", appTypes["annotation-queue"], false); err != nil { + t.Fatalf("scaffold: %v", err) + } + + link, err := readAppLink(target) + if err != nil { + t.Fatalf("readAppLink: %v", err) + } + if link == nil { + t.Fatal("expected apps init to write .langsmith/app.json immediately") + } + if link.AppID != "" { + t.Errorf("expected no app_id yet (app doesn't exist remotely until the first push), got %q", link.AppID) + } + if link.Name != "my-app" { + t.Errorf("expected --name to be recorded in the link file so a later \"apps push\" doesn't fall back to the directory basename, got %q", link.Name) + } +} + +func TestAppsInit_BlankAlsoWritesPartialAppLink(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "my-app") + + if _, err := scaffoldCustomAppStarter(target, "my-app", "", appTypes["blank"], false); err != nil { + t.Fatalf("scaffold: %v", err) + } + + link, err := readAppLink(target) + if err != nil { + t.Fatalf("readAppLink: %v", err) + } + if link == nil || link.Name != "my-app" || link.AppID != "" { + t.Errorf("expected a partial link recording the name with no app_id, got %+v", link) + } +} + +// The grid variant scaffolds its own spreadsheet UI, not the 3-pane components. +func TestAppsInit_ScaffoldsAnnotationQueueGridFiles(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "my-app") + + written, err := scaffoldCustomAppStarter(target, "my-app", "", appTypes["annotation-queue-grid"], false) + if err != nil { + t.Fatalf("scaffold: %v", err) + } + + writtenSet := map[string]bool{} + for _, w := range written { + writtenSet[w] = true + } + for _, want := range []string{ + "package.json", + "README.md", + "AGENTS.md", + ".gitignore", + "vite.config.ts", + "tsconfig.json", + "src/entry.tsx", + "src/App.tsx", + "src/api.ts", + "src/types.ts", + "src/components/DataGrid.tsx", + "src/components/GridCell.tsx", + } { + if !writtenSet[want] { + t.Errorf("expected %q to be written, got %v", want, written) + } + } + // The grid variant must not carry the 3-pane app's components. + for _, unwanted := range []string{ + "src/components/RunList.tsx", + "src/components/RunViewer.tsx", + "src/components/FeedbackPanel.tsx", + } { + if writtenSet[unwanted] { + t.Errorf("grid variant should not scaffold the 3-pane component %q", unwanted) + } + } + + // A partial link is written recording the name (no app_id until push). + link, err := readAppLink(target) + if err != nil { + t.Fatalf("readAppLink: %v", err) + } + if link == nil || link.Name != "my-app" { + t.Errorf("expected a partial link recording the name, got %+v", link) + } +} + +// The coding-agent dashboard is a standalone charts app; it scaffolds its own +// components and gets its own AGENTS.md. +func TestAppsInit_ScaffoldsCodingAgentDashboardFiles(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "my-app") + + written, err := scaffoldCustomAppStarter(target, "my-app", "", appTypes["coding-agent-dashboard"], false) + if err != nil { + t.Fatalf("scaffold: %v", err) + } + + writtenSet := map[string]bool{} + for _, w := range written { + writtenSet[w] = true + } + for _, want := range []string{ + "package.json", + "README.md", + "AGENTS.md", + ".gitignore", + "vite.config.ts", + "tsconfig.json", + "src/entry.tsx", + "src/App.tsx", + "src/api.ts", + "src/types.ts", + "src/components/ProjectBar.tsx", + "src/components/OverviewPanel.tsx", + "src/components/RunsTable.tsx", + } { + if !writtenSet[want] { + t.Errorf("expected %q to be written, got %v", want, written) + } + } + // It must not carry the annotation-queue templates' components. + if writtenSet["src/components/QueueBar.tsx"] || writtenSet["src/components/DataGrid.tsx"] { + t.Error("coding-agent dashboard should not scaffold annotation-queue components") + } + + agents, err := os.ReadFile(filepath.Join(target, "AGENTS.md")) + if err != nil { + t.Fatalf("read AGENTS.md: %v", err) + } + if !strings.Contains(string(agents), "ls_agent_purpose") || !strings.Contains(string(agents), "runs/query") { + t.Errorf("expected the coding-agent dashboard's AGENTS.md, got:\n%s", agents) + } +} + +// The experiment-comparison dashboard is a standalone app with its own +// components and AGENTS.md. +func TestAppsInit_ScaffoldsExperimentComparisonFiles(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "my-app") + + written, err := scaffoldCustomAppStarter(target, "my-app", "", appTypes["experiment-comparison"], false) + if err != nil { + t.Fatalf("scaffold: %v", err) + } + + writtenSet := map[string]bool{} + for _, w := range written { + writtenSet[w] = true + } + for _, want := range []string{ + "package.json", + "README.md", + "AGENTS.md", + ".gitignore", + "vite.config.ts", + "tsconfig.json", + "src/entry.tsx", + "src/App.tsx", + "src/api.ts", + "src/types.ts", + "src/lib/delta.ts", + "src/lib/metrics.ts", + "src/components/Pickers.tsx", + "src/components/SummaryPanel.tsx", + "src/components/ExampleTable.tsx", + "src/components/Scorecard.tsx", + "src/components/ScatterPlot.tsx", + } { + if !writtenSet[want] { + t.Errorf("expected %q to be written, got %v", want, written) + } + } + // It must not carry other templates' components. + if writtenSet["src/components/QueueBar.tsx"] || writtenSet["src/components/PieChart.tsx"] { + t.Error("experiment-comparison should not scaffold other templates' components") + } + + agents, err := os.ReadFile(filepath.Join(target, "AGENTS.md")) + if err != nil { + t.Fatalf("read AGENTS.md: %v", err) + } + if !strings.Contains(string(agents), "datasets/{dataset_id}/runs") || !strings.Contains(string(agents), "delta.ts") { + t.Errorf("expected the experiment-comparison AGENTS.md, got:\n%s", agents) + } +} + +// The two AQ templates must get distinct AGENTS.md files — guards the per-template agentsMD selection against regression. +func TestAppsInit_GridGetsDistinctAgentsMD(t *testing.T) { + dir := t.TempDir() + + gridTarget := filepath.Join(dir, "grid-app") + if _, err := scaffoldCustomAppStarter(gridTarget, "grid-app", "", appTypes["annotation-queue-grid"], false); err != nil { + t.Fatalf("scaffold grid: %v", err) + } + gridAgents, err := os.ReadFile(filepath.Join(gridTarget, "AGENTS.md")) + if err != nil { + t.Fatalf("read grid AGENTS.md: %v", err) + } + if !strings.Contains(string(gridAgents), "spreadsheet") || !strings.Contains(string(gridAgents), "DataGrid.tsx") { + t.Errorf("expected the grid-specific AGENTS.md, got:\n%s", gridAgents) + } + + paneTarget := filepath.Join(dir, "pane-app") + if _, err := scaffoldCustomAppStarter(paneTarget, "pane-app", "", appTypes["annotation-queue"], false); err != nil { + t.Fatalf("scaffold 3-pane: %v", err) + } + paneAgents, err := os.ReadFile(filepath.Join(paneTarget, "AGENTS.md")) + if err != nil { + t.Fatalf("read 3-pane AGENTS.md: %v", err) + } + if string(gridAgents) == string(paneAgents) { + t.Error("expected the grid variant to get a different AGENTS.md than the 3-pane template") + } + if strings.Contains(string(paneAgents), "DataGrid.tsx") { + t.Error("the 3-pane AGENTS.md should not mention the grid's DataGrid component") + } +} + +func TestAppsInit_ScaffoldsAnnotationQueueFiles(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "my-app") + + written, err := scaffoldCustomAppStarter(target, "my-app", "Does the thing", appTypes["annotation-queue"], false) + if err != nil { + t.Fatalf("scaffold: %v", err) + } + + writtenSet := map[string]bool{} + for _, w := range written { + writtenSet[w] = true + } + for _, want := range []string{ + "package.json", + "README.md", + "AGENTS.md", + ".gitignore", + "vite.config.ts", + "tailwind.config.js", + "tsconfig.json", + "src/entry.tsx", + "src/App.tsx", + "src/api.ts", + "src/global.d.ts", + } { + if !writtenSet[want] { + t.Errorf("expected %q to be written, got %v", want, written) + } + } + + pkg, err := os.ReadFile(filepath.Join(target, "package.json")) + if err != nil { + t.Fatalf("read package.json: %v", err) + } + if !strings.Contains(string(pkg), `"name": "my-app"`) { + t.Errorf("package.json missing templated name:\n%s", pkg) + } + + readme, err := os.ReadFile(filepath.Join(target, "README.md")) + if err != nil { + t.Fatalf("read README.md: %v", err) + } + if !strings.Contains(string(readme), "Does the thing") { + t.Errorf("README.md missing templated description:\n%s", readme) + } +} + +func TestAppsInit_ScaffoldsBlankFiles(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "my-app") + + written, err := scaffoldCustomAppStarter(target, "my-app", "", appTypes["blank"], false) + if err != nil { + t.Fatalf("scaffold: %v", err) + } + + writtenSet := map[string]bool{} + for _, w := range written { + writtenSet[w] = true + } + for _, want := range []string{"package.json", "README.md", "AGENTS.md", ".gitignore", "src/App.tsx", "src/entry.tsx"} { + if !writtenSet[want] { + t.Errorf("expected %q to be written, got %v", want, written) + } + } + // The blank template must never scaffold the annotation-queue app's own + // components — it has its own (much simpler) App.tsx. + if writtenSet["src/components/RunList.tsx"] { + t.Error("blank template should not scaffold the annotation-queue app's components") + } + + appTsx, err := os.ReadFile(filepath.Join(target, "src", "App.tsx")) + if err != nil { + t.Fatalf("read App.tsx: %v", err) + } + if strings.Contains(string(appTsx), "queueId") { + t.Errorf("expected the blank App.tsx to have no queue-specific code, got:\n%s", appTsx) + } +} + +// Non-templated files (anything but README.md/package.json) must be copied +// byte-for-byte — running them through text/template would choke on, or +// silently mangle, any literal "{{"/"}}" they contain, e.g. React's +// style={{...}} inline-style syntax. +func TestAppsInit_CopiesNonTemplatedFilesVerbatim(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "my-app") + + if _, err := scaffoldCustomAppStarter(target, "my-app", "", appTypes["annotation-queue"], false); err != nil { + t.Fatalf("scaffold: %v", err) + } + + got, err := os.ReadFile(filepath.Join(target, "src", "components", "FeedbackChip.tsx")) + if err != nil { + t.Fatalf("read FeedbackChip.tsx: %v", err) + } + if !strings.Contains(string(got), "style={{ backgroundColor: color") { + t.Errorf("expected literal style={{...}} to survive scaffolding unmodified, got:\n%s", got) + } +} + +func TestAppsInit_DefaultsDescription(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "my-app") + + if _, err := scaffoldCustomAppStarter(target, "my-app", "", appTypes["annotation-queue"], false); err != nil { + t.Fatalf("scaffold: %v", err) + } + readme, err := os.ReadFile(filepath.Join(target, "README.md")) + if err != nil { + t.Fatalf("read README.md: %v", err) + } + if !strings.Contains(string(readme), "TODO: one-sentence description") { + t.Errorf("expected default description placeholder:\n%s", readme) + } +} + +func TestAppsInit_RejectsNonEmptyDirWithoutForce(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "existing.txt"), []byte("hi"), 0o644); err != nil { + t.Fatalf("seed: %v", err) + } + _, err := scaffoldCustomAppStarter(dir, "my-app", "", appTypes["annotation-queue"], false) + if err == nil || !strings.Contains(err.Error(), "not empty") { + t.Errorf("expected not-empty error, got %v", err) + } +} + +func TestAppsInit_ForceWritesOverNonEmpty(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "existing.txt"), []byte("hi"), 0o644); err != nil { + t.Fatalf("seed: %v", err) + } + if _, err := scaffoldCustomAppStarter(dir, "my-app", "", appTypes["annotation-queue"], true); err != nil { + t.Fatalf("scaffold with force: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "package.json")); err != nil { + t.Errorf("package.json not written: %v", err) + } +} + +func TestAppsInit_RequiresName(t *testing.T) { + dir := t.TempDir() + if _, err := scaffoldCustomAppStarter(filepath.Join(dir, "app"), "", "", appTypes["annotation-queue"], false); err == nil { + t.Fatal("expected error when --name is empty") + } +} + +func TestAppsInit_RequiresValidType(t *testing.T) { + dir := t.TempDir() + if _, err := scaffoldCustomAppStarter(dir, "my-app", "", appType{}, false); err == nil { + t.Fatal("expected error for a zero-value (invalid) app type") + } +} + +func TestAppsInit_WritesTemplateSpecificAgentsMD(t *testing.T) { + dir := t.TempDir() + + blankTarget := filepath.Join(dir, "blank-app") + if _, err := scaffoldCustomAppStarter(blankTarget, "blank-app", "", appTypes["blank"], false); err != nil { + t.Fatalf("scaffold blank: %v", err) + } + blankAgents, err := os.ReadFile(filepath.Join(blankTarget, "AGENTS.md")) + if err != nil { + t.Fatalf("read AGENTS.md: %v", err) + } + if !strings.Contains(string(blankAgents), "window.langsmith.call") { + t.Errorf("expected the blank template's AGENTS.md, got:\n%s", blankAgents) + } + + aqTarget := filepath.Join(dir, "aq-app") + if _, err := scaffoldCustomAppStarter(aqTarget, "aq-app", "", appTypes["annotation-queue"], false); err != nil { + t.Fatalf("scaffold annotation-queue: %v", err) + } + aqAgents, err := os.ReadFile(filepath.Join(aqTarget, "AGENTS.md")) + if err != nil { + t.Fatalf("read AGENTS.md: %v", err) + } + if !strings.Contains(string(aqAgents), "3-pane") { + t.Errorf("expected the annotation-queue template's AGENTS.md, got:\n%s", aqAgents) + } +} + +func TestAppsInitCmd_TemplateFlagDefaultsEmpty(t *testing.T) { + cmd := newAppsCmd() + initCmd, _, err := cmd.Find([]string{"init"}) + if err != nil { + t.Fatalf("find init command: %v", err) + } + f := initCmd.Flags().Lookup("template") + if f == nil { + t.Fatal("expected --template flag to exist") + } + if f.DefValue != "" { + t.Errorf("expected --template to default to \"\", got %q", f.DefValue) + } + // The old --type flag must be gone, not just aliased. + if got := initCmd.Flags().Lookup("type"); got != nil { + t.Error("expected --type to be gone from apps init (renamed to --template)") + } +} + +func TestAppsInitCmd_RejectsExplicitTemplateBlank(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + cmd := newAppsCmd() + cmd.SetArgs([]string{"init", "--name", "my-app", "--template", "blank", "--skip-install"}) + if err := cmd.Execute(); err == nil { + t.Error("expected --template blank to be rejected; omit --template instead") + } +} + +func TestAppsInitCmd_DefaultsToBlankWhenTemplateOmitted(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + cmd := newAppsCmd() + cmd.SetArgs([]string{"init", "--name", "my-app", "--skip-install"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("expected init to succeed with --template omitted, got: %v", err) + } + + // Omitting --template scaffolds the blank starter: a partial link and a + // queue-free App.tsx. + link, err := readAppLink(dir) + if err != nil { + t.Fatalf("readAppLink: %v", err) + } + if link == nil || link.Name != "my-app" { + t.Errorf("expected a partial link recording the name, got %+v", link) + } + appTsx, err := os.ReadFile(filepath.Join(dir, "src", "App.tsx")) + if err != nil { + t.Fatalf("read App.tsx: %v", err) + } + if strings.Contains(string(appTsx), "queueId") { + t.Errorf("expected the blank App.tsx (no queue code) when --template is omitted, got:\n%s", appTsx) + } +} + +func TestAppsInitCmd_AcceptsAnnotationQueueGridTemplate(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + cmd := newAppsCmd() + cmd.SetArgs([]string{"init", "--name", "grid-app", "--template", "annotation-queue-grid", "--skip-install"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("expected init to succeed for --template annotation-queue-grid, got: %v", err) + } + + // The grid variant's own component is scaffolded. + if _, err := os.Stat(filepath.Join(dir, "src", "components", "DataGrid.tsx")); err != nil { + t.Errorf("expected the grid template's DataGrid.tsx to be scaffolded: %v", err) + } + link, err := readAppLink(dir) + if err != nil { + t.Fatalf("readAppLink: %v", err) + } + if link == nil || link.Name != "grid-app" { + t.Errorf("expected a partial link recording the name, got %+v", link) + } +} + +func TestAppsInitCmd_AcceptsCodingAgentDashboardTemplate(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + cmd := newAppsCmd() + cmd.SetArgs([]string{"init", "--name", "dash", "--template", "coding-agent-dashboard", "--skip-install"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("expected init to succeed for --template coding-agent-dashboard, got: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "src", "components", "OverviewPanel.tsx")); err != nil { + t.Errorf("expected the dashboard's OverviewPanel.tsx to be scaffolded: %v", err) + } +} + +func TestAppsInitCmd_AcceptsExperimentComparisonTemplate(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + cmd := newAppsCmd() + cmd.SetArgs([]string{"init", "--name", "cmp", "--template", "experiment-comparison", "--skip-install"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("expected init to succeed for --template experiment-comparison, got: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "src", "components", "SummaryPanel.tsx")); err != nil { + t.Errorf("expected the SummaryPanel.tsx to be scaffolded: %v", err) + } +} + +func TestAppsInitCmd_RejectsInvalidTemplate(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + cmd := newAppsCmd() + cmd.SetArgs([]string{"init", "--name", "my-app", "--template", "bogus", "--skip-install"}) + if err := cmd.Execute(); err == nil { + t.Error("expected error for an invalid --template") + } +} + +func TestAppsInitCmd_HasSkipInstallFlag(t *testing.T) { + cmd := newAppsCmd() + initCmd, _, err := cmd.Find([]string{"init"}) + if err != nil { + t.Fatalf("find init command: %v", err) + } + if f := initCmd.Flags().Lookup("skip-install"); f == nil { + t.Error("expected --skip-install flag to exist") + } +} + +func TestInstallAndBuildCustomAppStarter_ErrorsWhenNpmMissing(t *testing.T) { + t.Setenv("PATH", "") + dir := t.TempDir() + if err := installAndBuildCustomAppStarter(dir); err == nil { + t.Error("expected error when npm is not on PATH") + } +} + +// fakeNpm writes an executable named "npm" into a fresh directory prepended +// to PATH, standing in for a real npm install: no network access, no real +// esbuild download. It only needs to satisfy the two invocations +// installAndBuildCustomAppStarter makes: "install" and "run build". +func fakeNpm(t *testing.T, onRunBuild string) { + t.Helper() + binDir := t.TempDir() + script := "#!/bin/sh\n" + + "if [ \"$1\" = \"install\" ]; then exit 0; fi\n" + + "if [ \"$1\" = \"run\" ] && [ \"$2\" = \"build\" ]; then\n" + onRunBuild + "\nexit 0\nfi\n" + + "exit 1\n" + npmPath := filepath.Join(binDir, "npm") + if err := os.WriteFile(npmPath, []byte(script), 0o755); err != nil { + t.Fatalf("write fake npm: %v", err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func TestInstallAndBuildCustomAppStarter_RunsInstallThenBuild(t *testing.T) { + dir := t.TempDir() + fakeNpm(t, `mkdir -p dist && printf 'module.exports={render:function(){}}' > dist/bundle.js`) + + if err := installAndBuildCustomAppStarter(dir); err != nil { + t.Fatalf("installAndBuildCustomAppStarter: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "dist", "bundle.js")); err != nil { + t.Errorf("expected dist/bundle.js to be produced by the build step: %v", err) + } +} + +func TestInstallAndBuildCustomAppStarter_ErrorsWhenBuildFails(t *testing.T) { + dir := t.TempDir() + fakeNpm(t, `exit 1`) + + if err := installAndBuildCustomAppStarter(dir); err == nil { + t.Error("expected error when the build step fails") + } +} + +// Shared files (templates/_shared/) aren't tracked per template — a +// template gets one by actually importing it, discovered by scanning its +// own source. These tests pin down that inference so it can't silently +// regress into either a maintained list or a "copy everything" fallback. +func TestSharedFileImportSpecifiers_MatchesTemplateConventions(t *testing.T) { + tests := []struct { + sharedRelPath string + want []string + }{ + { + sharedRelPath: "components/SearchableSelect.tsx", + want: []string{ + "./SearchableSelect", + "./components/SearchableSelect", + "../components/SearchableSelect", + }, + }, + { + sharedRelPath: "lib/utils.ts", + want: []string{ + "./utils", + "./lib/utils", + "../lib/utils", + }, + }, + } + for _, tt := range tests { + got := sharedFileImportSpecifiers(tt.sharedRelPath) + gotSet := map[string]bool{} + for _, g := range got { + gotSet[g] = true + } + for _, want := range tt.want { + if !gotSet[want] { + t.Errorf("sharedFileImportSpecifiers(%q) = %v, missing %q", tt.sharedRelPath, got, want) + } + } + } +} + +// annotation-queue-grid imports SearchableSelect, Spinner, and the cn() +// helper from lib/utils — all three should be pulled in, byte-identical to +// the canonical _shared/ source. +func TestAppsInit_PullsInEverySharedFileATemplateImports(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "my-app") + + written, err := scaffoldCustomAppStarter(target, "my-app", "", appTypes["annotation-queue-grid"], false) + if err != nil { + t.Fatalf("scaffold: %v", err) + } + + writtenSet := map[string]bool{} + for _, w := range written { + writtenSet[w] = true + } + for _, sharedRelPath := range []string{ + "src/components/SearchableSelect.tsx", + "src/components/Spinner.tsx", + "src/lib/utils.ts", + } { + if !writtenSet[sharedRelPath] { + t.Errorf("expected %q to be pulled in from _shared/, got %v", sharedRelPath, written) + continue + } + got, err := os.ReadFile(filepath.Join(target, sharedRelPath)) + if err != nil { + t.Fatalf("reading scaffolded %s: %v", sharedRelPath, err) + } + want, err := sharedFS.ReadFile(sharedRoot + "/" + strings.TrimPrefix(sharedRelPath, "src/")) + if err != nil { + t.Fatalf("reading embedded shared %s: %v", sharedRelPath, err) + } + if string(got) != string(want) { + t.Errorf("scaffolded %s does not match the canonical _shared/ source", sharedRelPath) + } + } +} + +// experiment-comparison only imports SearchableSelect — it has no Spinner +// and never had a lib/utils.ts, so those must not appear even though +// they're valid _shared/ files another template uses. +func TestAppsInit_OnlyPullsInSharedFilesActuallyImported(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "my-app") + + written, err := scaffoldCustomAppStarter(target, "my-app", "", appTypes["experiment-comparison"], false) + if err != nil { + t.Fatalf("scaffold: %v", err) + } + + writtenSet := map[string]bool{} + for _, w := range written { + writtenSet[w] = true + } + for _, wanted := range []string{"src/components/SearchableSelect.tsx", "src/components/Spinner.tsx", "src/lib/utils.ts"} { + if !writtenSet[wanted] { + t.Errorf("expected %q to be pulled in, got %v", wanted, written) + } + } +} + +// go:embed has no concept of .gitignore — it embeds every file physically +// present under a template's source directory at build time, node_modules +// included. If a local `npm install`/`vite build` run in a template's source +// tree isn't cleaned up before `go build`/`go install`, the *entire* +// node_modules (and dist/) gets silently baked into the CLI binary and +// scaffolded into every user's project — with a working package.json but no +// working node_modules/.bin symlinks (embed can't preserve symlinks), so +// "npm run watch" fails with "vite: command not found" despite the files +// visibly being there. This walks the actual embedded FS content (not just +// the source tree, which can look clean between commits) so a stray +// node_modules/dist left behind before a build gets caught here instead of +// shipping. +func TestEmbeddedTemplates_CarryNoStrayBuildArtifacts(t *testing.T) { + check := func(name string, embedded fs.FS) { + t.Helper() + err := fs.WalkDir(embedded, ".", func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + base := filepath.Base(path) + if d.IsDir() && (base == "node_modules" || base == "dist") { + t.Errorf("%s: embedded FS contains %q — a local build wasn't cleaned up before go build/go install", name, path) + return fs.SkipDir + } + if !d.IsDir() && (base == "package-lock.json" || base == "pnpm-lock.yaml") { + t.Errorf("%s: embedded FS contains %q — shouldn't be committed or embedded", name, path) + } + return nil + }) + if err != nil { + t.Fatalf("%s: walking embedded FS: %v", name, err) + } + } + + for name, at := range appTypes { + check(name, at.templateFS) + } + check("_shared", sharedFS) +} diff --git a/internal/cmd/apps_list.go b/internal/cmd/apps_list.go new file mode 100644 index 0000000..00bc260 --- /dev/null +++ b/internal/cmd/apps_list.go @@ -0,0 +1,41 @@ +package cmd + +import ( + "context" + "fmt" + + "github.com/langchain-ai/langsmith-cli/internal/output" + "github.com/spf13/cobra" +) + +func newAppsListCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "List custom apps", + RunE: func(cmd *cobra.Command, args []string) error { + c := MustGetClient() + ctx := context.Background() + + var apps []customApp + if err := c.RawGet(ctx, "/v1/platform/custom-apps", &apps); err != nil { + return fmt.Errorf("listing custom apps: %w", err) + } + + data := make([]map[string]any, 0, len(apps)) + for _, a := range apps { + data = append(data, map[string]any{ + "id": a.ID, + "name": a.Name, + "description": a.Description, + "entrypoint": a.Entrypoint, + "is_enabled": a.IsEnabled, + "updated_at": a.UpdatedAt, + }) + } + output.OutputJSON(data, "") + return nil + }, + } + + return cmd +} diff --git a/internal/cmd/apps_push.go b/internal/cmd/apps_push.go new file mode 100644 index 0000000..cd3081b --- /dev/null +++ b/internal/cmd/apps_push.go @@ -0,0 +1,200 @@ +package cmd + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + + "github.com/langchain-ai/langsmith-cli/internal/client" + "github.com/langchain-ai/langsmith-cli/internal/output" + "github.com/spf13/cobra" +) + +func newAppsPushCmd() *cobra.Command { + var ( + name string + description string + entrypoint string + buildCmd string + noBuild bool + ) + + cmd := &cobra.Command{ + Use: "push", + Short: "Upload the current directory as a custom app", + Long: `Upload the current directory as a custom app. + +Builds before uploading: if package.json has a "build" script (both starter +templates do), this runs it first (like "npm run build"), so the upload +always reflects your current source — not whatever happened to be left in +dist/ by a previous "apps dev" session (e.g. one interrupted mid-rebuild). +Pass --build to run a different command instead, or --no-build to upload +the directory exactly as-is with no build step. The first push creates the +app and writes .langsmith/app.json recording its ID; every push after that +reads that file and updates the same app in place. Commit .langsmith/app.json +so teammates' pushes update the same app instead of creating new ones. + +--name only takes effect on the first push (creation).`, + RunE: func(cmd *cobra.Command, args []string) error { + dir, err := os.Getwd() + if err != nil { + return fmt.Errorf("getting current directory: %w", err) + } + + if noBuild && buildCmd != "" { + return fmt.Errorf("--build and --no-build are mutually exclusive") + } + + if !noBuild { + cmdToRun := buildCmd + if cmdToRun == "" { + script, pkgJSONExists, scriptErr := packageJSONScript(dir, "build") + if scriptErr != nil { + return fmt.Errorf("reading package.json to find a \"build\" script: %w", scriptErr) + } + if script != "" { + cmdToRun = "npm run build" + } else if pkgJSONExists { + fmt.Fprintln(os.Stderr, `note: no "build" script in package.json — skipping automatic build; pass --build to run one`) + } + } + if cmdToRun != "" { + if err := runAppsBuildCmd(dir, cmdToRun); err != nil { + return err + } + } + } + + files, err := readDirectoryAsAppFiles(dir) + if err != nil { + return err + } + if len(files) == 0 { + return fmt.Errorf("no files found under %s (after applying exclusions)", dir) + } + if _, ok := files[entrypoint]; !ok { + return fmt.Errorf("entrypoint %q not found among uploaded files; pass --entrypoint or check --build produced it", entrypoint) + } + + link, err := readAppLink(dir) + if err != nil { + return err + } + // "apps init" writes .langsmith/app.json immediately (recording the + // name) with no app_id yet — that's only known once an app is + // actually created. So a link file existing isn't enough to mean + // "already created"; app_id being set is. + notYetCreated := link == nil || link.AppID == "" + + c := MustGetClient() + ctx := context.Background() + + var app customApp + updated := false + if !notYetCreated { + payload := map[string]any{ + "files": files, + "entrypoint": entrypoint, + } + if name != "" { + payload["name"] = name + } + if description != "" { + payload["description"] = description + } + err := c.RawPatch(ctx, "/v1/platform/custom-apps/"+link.AppID, payload, &app) + switch { + case err == nil: + updated = true + case client.IsConflict(err): + return fmt.Errorf("a custom app named %q already exists in this workspace", name) + case client.IsNotFound(err): + // The linked app_id no longer exists server-side (e.g. it + // was deleted through the UI) — .langsmith/app.json is + // stale. Recreate under the same name rather than failing, + // and relink to the new app below. + fmt.Fprintf(os.Stderr, "note: custom app %s no longer exists (it may have been deleted) — creating a new one\n", link.AppID) + default: + return fmt.Errorf("updating custom app %s: %w", link.AppID, err) + } + } + + if updated { + if err := writeAppLink(dir, appLink{ + AppID: app.ID, + Name: app.Name, + }); err != nil { + return err + } + } else { + appName := name + if appName == "" && link != nil && link.Name != "" { + appName = link.Name + } + if appName == "" { + appName = filepath.Base(filepath.Clean(dir)) + } + payload := map[string]any{ + "name": appName, + "files": files, + "entrypoint": entrypoint, + } + if description != "" { + payload["description"] = description + } + if err := c.RawPost(ctx, "/v1/platform/custom-apps", payload, &app); err != nil { + if client.IsConflict(err) { + return fmt.Errorf("a custom app named %q already exists in this workspace. try `langsmith apps push --name \"New Name\"` instead", appName) + } + return fmt.Errorf("creating custom app: %w", err) + } + if err := writeAppLink(dir, appLink{ + AppID: app.ID, + Name: app.Name, + }); err != nil { + return err + } + } + + paths := make([]string, 0, len(files)) + for k := range files { + paths = append(paths, k) + } + sort.Strings(paths) + + status := "created" + if updated { + status = "updated" + } + output.OutputJSON(map[string]any{ + "status": status, + "app_id": app.ID, + "name": app.Name, + "entrypoint": app.Entrypoint, + "files": paths, + }, "") + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "App name (required on first push; renames on later pushes if passed)") + cmd.Flags().StringVar(&description, "description", "", "App description") + cmd.Flags().StringVar(&entrypoint, "entrypoint", "dist/bundle.js", "Path (relative to the current directory) of the file to render") + cmd.Flags().StringVar(&buildCmd, "build", "", "Shell command to run in the current directory before uploading, overriding the auto-detected package.json \"build\" script (e.g. \"npm run build\")") + cmd.Flags().BoolVar(&noBuild, "no-build", false, "Skip building before uploading, even if package.json has a \"build\" script") + return cmd +} + +func runAppsBuildCmd(dir, buildCmd string) error { + c := exec.Command("sh", "-c", buildCmd) + c.Dir = dir + c.Stdout = os.Stderr + c.Stderr = os.Stderr + if err := c.Run(); err != nil { + return fmt.Errorf("--build command %q failed: %w", buildCmd, err) + } + return nil +} diff --git a/internal/cmd/apps_push_test.go b/internal/cmd/apps_push_test.go new file mode 100644 index 0000000..f221913 --- /dev/null +++ b/internal/cmd/apps_push_test.go @@ -0,0 +1,375 @@ +package cmd + +import ( + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" +) + +func seedAppDir(t *testing.T, dir string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(dir, "dist"), 0o755); err != nil { + t.Fatalf("mkdir dist: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "dist", "bundle.js"), []byte("module.exports = { render: function() {} }"), 0o644); err != nil { + t.Fatalf("seed bundle.js: %v", err) + } +} + +func TestAppsPush_CreatesAndWritesLink(t *testing.T) { + var sawPost bool + var postBody map[string]any + + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == "POST" && r.URL.Path == "/v1/platform/custom-apps": + sawPost = true + body, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(body, &postBody) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(customApp{ + ID: "app_new", + Name: "my-app", + Entrypoint: "dist/bundle.js", + }) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + } + }) + defer setupTestEnv(t, srv.URL)() + + dir := t.TempDir() + seedAppDir(t, dir) + t.Chdir(dir) + + out := captureStdout(t, func() { + cmd := newAppsCmd() + cmd.SetArgs([]string{"push", "--name", "my-app"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v", err) + } + }) + + if !sawPost { + t.Fatal("expected POST to create the app") + } + if postBody["name"] != "my-app" { + t.Errorf("expected name in create payload, got %v", postBody) + } + if !strings.Contains(out, `"status": "created"`) { + t.Errorf("expected created status in output:\n%s", out) + } + + link, err := readAppLink(dir) + if err != nil { + t.Fatalf("readAppLink: %v", err) + } + if link == nil || link.AppID != "app_new" { + t.Errorf("expected link file with app_new, got %+v", link) + } +} + +func TestAppsPush_UpdatesWhenAlreadyLinked(t *testing.T) { + var sawPatch, sawPost bool + var patchPath string + + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == "POST" && r.URL.Path == "/v1/platform/custom-apps": + sawPost = true + case r.Method == "PATCH": + sawPatch = true + patchPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(customApp{ + ID: "app_existing", + Name: "my-app", + Entrypoint: "dist/bundle.js", + }) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + } + }) + defer setupTestEnv(t, srv.URL)() + + dir := t.TempDir() + seedAppDir(t, dir) + if err := writeAppLink(dir, appLink{AppID: "app_existing", Name: "my-app"}); err != nil { + t.Fatalf("seed link: %v", err) + } + t.Chdir(dir) + + captureStdout(t, func() { + cmd := newAppsCmd() + cmd.SetArgs([]string{"push"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v", err) + } + }) + + if sawPost { + t.Error("expected update path to PATCH, not POST (create)") + } + if !sawPatch { + t.Fatal("expected PATCH to update the existing app") + } + if patchPath != "/v1/platform/custom-apps/app_existing" { + t.Errorf("unexpected PATCH path: %s", patchPath) + } +} + +// A link with a name but no app_id (what "apps init" writes) must POST, not +// PATCH an empty ID. +func TestAppsPush_CreatesWhenLinkedButNotYetCreated(t *testing.T) { + var sawPost bool + var postBody map[string]any + + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == "POST" && r.URL.Path == "/v1/platform/custom-apps": + sawPost = true + body, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(body, &postBody) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(customApp{ + ID: "app_new", + Name: "my-aq-app", + Entrypoint: "dist/bundle.js", + }) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + } + }) + defer setupTestEnv(t, srv.URL)() + + dir := t.TempDir() + seedAppDir(t, dir) + // Mirrors what "apps init --name my-aq-app --template annotation-queue" writes. + if err := writeAppLink(dir, appLink{Name: "my-aq-app"}); err != nil { + t.Fatalf("seed partial link: %v", err) + } + t.Chdir(dir) + + out := captureStdout(t, func() { + cmd := newAppsCmd() + cmd.SetArgs([]string{"push"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v", err) + } + }) + + if !sawPost { + t.Fatal("expected POST to create the app (app_id was empty, so this isn't an update)") + } + if postBody["name"] != "my-aq-app" { + t.Errorf("expected name from the link file used as fallback, got %v", postBody) + } + if !strings.Contains(out, `"status": "created"`) { + t.Errorf("expected created status, got:\n%s", out) + } + + link, err := readAppLink(dir) + if err != nil { + t.Fatalf("readAppLink: %v", err) + } + if link == nil || link.AppID != "app_new" { + t.Errorf("expected link file updated with the real app_id, got %+v", link) + } +} + +// If the linked app was deleted out-of-band (e.g. through the UI), +// .langsmith/app.json still has its old app_id and push's PATCH 404s. Push +// must recreate the app under the same name rather than failing, and relink +// to the new app_id. +func TestAppsPush_RecreatesWhenLinkedAppWasDeleted(t *testing.T) { + var sawPatch, sawPost bool + var postBody map[string]any + + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == "PATCH" && r.URL.Path == "/v1/platform/custom-apps/app_deleted": + sawPatch = true + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(map[string]string{"error": "custom app not found"}) + case r.Method == "POST" && r.URL.Path == "/v1/platform/custom-apps": + sawPost = true + body, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(body, &postBody) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(customApp{ + ID: "app_recreated", + Name: "my-app", + Entrypoint: "dist/bundle.js", + }) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + } + }) + defer setupTestEnv(t, srv.URL)() + + dir := t.TempDir() + seedAppDir(t, dir) + if err := writeAppLink(dir, appLink{AppID: "app_deleted", Name: "my-app"}); err != nil { + t.Fatalf("seed link: %v", err) + } + t.Chdir(dir) + + out := captureStdout(t, func() { + cmd := newAppsCmd() + cmd.SetArgs([]string{"push"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v", err) + } + }) + + if !sawPatch { + t.Fatal("expected push to try PATCH against the linked (now-deleted) app_id first") + } + if !sawPost { + t.Fatal("expected push to fall back to POST (create) after the PATCH 404s") + } + if postBody["name"] != "my-app" { + t.Errorf("expected the recreated app to reuse the old app's name, got %v", postBody) + } + if !strings.Contains(out, `"status": "created"`) { + t.Errorf("expected created status, got:\n%s", out) + } + + link, err := readAppLink(dir) + if err != nil { + t.Fatalf("readAppLink: %v", err) + } + if link == nil || link.AppID != "app_recreated" { + t.Errorf("expected link file relinked to the new app_id, got %+v", link) + } +} + +func TestAppsPush_ErrorsWhenEntrypointMissing(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + }) + defer setupTestEnv(t, srv.URL)() + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "other.js"), []byte("x"), 0o644); err != nil { + t.Fatalf("seed: %v", err) + } + t.Chdir(dir) + + cmd := newAppsCmd() + cmd.SetArgs([]string{"push", "--name", "my-app"}) + if err := cmd.Execute(); err == nil { + t.Fatal("expected error when default entrypoint dist/bundle.js is missing") + } +} + +// Regression test for the actual bug that motivated auto-building: "apps +// dev"'s watch process empties dist/ before every rebuild, so interrupting +// it mid-rebuild leaves an empty dist/ behind. Push must not trust that +// leftover state — it should rebuild from source before uploading. +func TestAppsPush_BuildsFromPackageJSONByDefault(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == "POST" && r.URL.Path == "/v1/platform/custom-apps" { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(customApp{ID: "app_new", Name: "my-app", Entrypoint: "dist/bundle.js"}) + return + } + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + }) + defer setupTestEnv(t, srv.URL)() + + dir := t.TempDir() + fakeNpm(t, `mkdir -p dist && printf 'module.exports={render:function(){}}' > dist/bundle.js`) + if err := os.WriteFile(filepath.Join(dir, "package.json"), []byte(`{"scripts":{"build":"vite build"}}`), 0o644); err != nil { + t.Fatalf("seed package.json: %v", err) + } + // No dist/ seeded at all — simulates an "apps dev" watch process that was + // interrupted mid-rebuild (emptyOutDir cleared it, nothing was written back). + t.Chdir(dir) + + captureStdout(t, func() { + cmd := newAppsCmd() + cmd.SetArgs([]string{"push", "--name", "my-app"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v", err) + } + }) + + if _, err := os.Stat(filepath.Join(dir, "dist", "bundle.js")); err != nil { + t.Errorf("expected the auto-detected \"build\" script to produce dist/bundle.js: %v", err) + } +} + +func TestAppsPush_NoBuildSkipsAutomaticBuild(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + }) + defer setupTestEnv(t, srv.URL)() + + dir := t.TempDir() + fakeNpm(t, `mkdir -p dist && printf 'module.exports={render:function(){}}' > dist/bundle.js`) + if err := os.WriteFile(filepath.Join(dir, "package.json"), []byte(`{"scripts":{"build":"vite build"}}`), 0o644); err != nil { + t.Fatalf("seed package.json: %v", err) + } + t.Chdir(dir) + + cmd := newAppsCmd() + cmd.SetArgs([]string{"push", "--name", "my-app", "--no-build"}) + if err := cmd.Execute(); err == nil { + t.Fatal("expected error: --no-build should skip the build, leaving dist/bundle.js missing") + } + if _, err := os.Stat(filepath.Join(dir, "dist", "bundle.js")); err == nil { + t.Error("--no-build must not run the package.json \"build\" script") + } +} + +func TestAppsPush_BuildAndNoBuildAreMutuallyExclusive(t *testing.T) { + dir := t.TempDir() + seedAppDir(t, dir) + t.Chdir(dir) + + cmd := newAppsCmd() + cmd.SetArgs([]string{"push", "--name", "my-app", "--build", "true", "--no-build"}) + if err := cmd.Execute(); err == nil { + t.Fatal("expected error when both --build and --no-build are passed") + } +} + +func TestAppsPush_CustomBuildCommandOverridesAutoDetection(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == "POST" && r.URL.Path == "/v1/platform/custom-apps" { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(customApp{ID: "app_new", Name: "my-app", Entrypoint: "dist/bundle.js"}) + return + } + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + }) + defer setupTestEnv(t, srv.URL)() + + dir := t.TempDir() + // A package.json "build" script exists but must NOT run — --build wins. + if err := os.WriteFile(filepath.Join(dir, "package.json"), []byte(`{"scripts":{"build":"exit 1"}}`), 0o644); err != nil { + t.Fatalf("seed package.json: %v", err) + } + t.Chdir(dir) + + captureStdout(t, func() { + cmd := newAppsCmd() + cmd.SetArgs([]string{ + "push", "--name", "my-app", + "--build", "mkdir -p dist && printf 'module.exports={render:function(){}}' > dist/bundle.js", + }) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v", err) + } + }) + + if _, err := os.Stat(filepath.Join(dir, "dist", "bundle.js")); err != nil { + t.Errorf("expected the explicit --build command to produce dist/bundle.js: %v", err) + } +} diff --git a/internal/cmd/apps_test.go b/internal/cmd/apps_test.go new file mode 100644 index 0000000..27f5375 --- /dev/null +++ b/internal/cmd/apps_test.go @@ -0,0 +1,114 @@ +package cmd + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestAppLink_RoundTrip(t *testing.T) { + dir := t.TempDir() + + if link, err := readAppLink(dir); err != nil || link != nil { + t.Fatalf("expected (nil, nil) for missing link file, got (%v, %v)", link, err) + } + + want := appLink{AppID: "app_1", Name: "my-app"} + if err := writeAppLink(dir, want); err != nil { + t.Fatalf("writeAppLink: %v", err) + } + + got, err := readAppLink(dir) + if err != nil { + t.Fatalf("readAppLink: %v", err) + } + if got == nil || *got != want { + t.Errorf("readAppLink = %+v, want %+v", got, want) + } + + // Written under .langsmith/app.json specifically, not loose in dir. + if _, err := os.Stat(filepath.Join(dir, appsLinkDir, appsLinkFile)); err != nil { + t.Errorf(".langsmith/app.json not found: %v", err) + } +} + +func TestAppLink_OverwritesExisting(t *testing.T) { + dir := t.TempDir() + if err := writeAppLink(dir, appLink{AppID: "app_1"}); err != nil { + t.Fatalf("first write: %v", err) + } + if err := writeAppLink(dir, appLink{AppID: "app_2"}); err != nil { + t.Fatalf("second write: %v", err) + } + got, err := readAppLink(dir) + if err != nil || got == nil || got.AppID != "app_2" { + t.Errorf("expected app_2 after overwrite, got %+v (err %v)", got, err) + } +} + +func TestReadDirectoryAsAppFiles_ExcludesConfiguredDirsAndFiles(t *testing.T) { + dir := t.TempDir() + seed := map[string]string{ + "index.js": "module.exports = {}", + "node_modules/pkg/index.js": "should be excluded", + ".git/HEAD": "should be excluded", + ".langsmith/app.json": "should be excluded", + ".DS_Store": "should be excluded", + ".env": "should be excluded", + ".env.production": "should be excluded", + } + for rel, content := range seed { + full := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatalf("mkdir for %s: %v", rel, err) + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", rel, err) + } + } + + files, err := readDirectoryAsAppFiles(dir) + if err != nil { + t.Fatalf("readDirectoryAsAppFiles: %v", err) + } + if len(files) != 1 { + t.Fatalf("expected exactly 1 file, got %v", files) + } + if files["index.js"] != "module.exports = {}" { + t.Errorf("unexpected content for index.js: %q", files["index.js"]) + } +} + +func TestReadDirectoryAsAppFiles_RejectsBinary(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "bin.dat"), []byte{0x00, 0x01, 0x02}, 0o644); err != nil { + t.Fatalf("seed: %v", err) + } + _, err := readDirectoryAsAppFiles(dir) + if err == nil || !strings.Contains(err.Error(), "binary") { + t.Errorf("expected binary-rejection error, got %v", err) + } +} + +func TestReadDirectoryAsAppFiles_RejectsOversizedFile(t *testing.T) { + dir := t.TempDir() + big := make([]byte, appsMaxFileSizeBytes+1) + for i := range big { + big[i] = 'a' + } + if err := os.WriteFile(filepath.Join(dir, "big.txt"), big, 0o644); err != nil { + t.Fatalf("seed: %v", err) + } + _, err := readDirectoryAsAppFiles(dir) + if err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Errorf("expected size-limit error, got %v", err) + } +} + +func TestReadDirectoryAsAppFiles_ErrorsOnMissingDir(t *testing.T) { + _, err := readDirectoryAsAppFiles(filepath.Join(t.TempDir(), "does-not-exist")) + if err == nil { + t.Fatal("expected error for missing directory") + } +} diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 9c8992b..64cc585 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -77,6 +77,7 @@ Quick start: rootCmd.AddCommand(newInsightsCmd()) rootCmd.AddCommand(newFleetCmd()) rootCmd.AddCommand(newHubCmd()) + rootCmd.AddCommand(newAppsCmd()) rootCmd.AddCommand(newPromptCmd()) rootCmd.AddCommand(authCommand.Cobra()) rootCmd.AddCommand(newProfileCmd()) @@ -118,14 +119,24 @@ func GetFormat() string { // MustGetClient creates a LangSmith client or exits with an error. func MustGetClient() *client.Client { - opts, err := resolveClientOptions(true) + c, err := getClient() if err != nil { ExitError(err.Error()) } + return c +} + +// getClient is the non-exiting sibling of MustGetClient: it returns an error +// instead of calling os.Exit, so request handlers can't crash. +func getClient() (*client.Client, error) { + opts, err := resolveClientOptions(true) + if err != nil { + return nil, err + } if opts.APIKey == "" && opts.OAuthAccessToken == "" { - ExitError("not authenticated; run 'langsmith auth login', set LANGSMITH_API_KEY, or pass --api-key") + return nil, fmt.Errorf("not authenticated; run 'langsmith auth login', set LANGSMITH_API_KEY, or pass --api-key") } - return client.NewWithOptions(opts) + return client.NewWithOptions(opts), nil } func resolveClientOptions(refreshOAuth bool) (client.Options, error) { diff --git a/internal/cmd/templates/_shared/components/SearchableSelect.tsx b/internal/cmd/templates/_shared/components/SearchableSelect.tsx new file mode 100644 index 0000000..d022ab4 --- /dev/null +++ b/internal/cmd/templates/_shared/components/SearchableSelect.tsx @@ -0,0 +1,226 @@ +// Canonical single source. apps_init.go copies this into a scaffolded app's +// src/ at generation time, and only if the template imports it — there are no +// standing per-template copies and no sync script. Deliberately +// self-contained (own cn()/chevron, no external icon package) so it drops +// into any template unmodified regardless of what other dependencies that +// template happens to have. +import { useEffect, useRef, useState } from 'react'; + +function cn(...classes: (string | false | null | undefined)[]): string { + return classes.filter(Boolean).join(' '); +} + +const PAGE_SIZE = 25; + +function ChevronDownIcon({ className }: { className?: string }) { + return ( + + + + ); +} + +export interface SearchableSelectItem { + id: string; + name: string; +} + +interface Props { + id?: string; + value: string; + onSelect: (item: T) => void; + /** Fetches one page of results for the given search term. Called with + * offset 0 whenever the dropdown opens or the search term changes, and + * with an increasing offset as the user scrolls the list. A page shorter + * than PAGE_SIZE (25) is treated as the last page. */ + fetchPage: (search: string, offset: number, limit: number) => Promise; + placeholder: string; + searchPlaceholder?: string; + emptyLabel?: string; + disabled?: boolean; + className?: string; +} + +// A native setSearch(e.target.value)} + placeholder={searchPlaceholder} + className="w-full rounded border border-secondary bg-primary px-2 py-1 text-sm text-primary focus:border-brand focus:outline-none" + /> + +
+ {loading ? ( +
Loading…
+ ) : items.length === 0 ? ( +
{emptyLabel}
+ ) : ( + <> + {items.map((item) => ( + + ))} + {hasMore && ( +
+ {loadingMore ? 'Loading more…' : ''} +
+ )} + + )} +
+ + )} + + ); +} diff --git a/internal/cmd/templates/_shared/components/Spinner.tsx b/internal/cmd/templates/_shared/components/Spinner.tsx new file mode 100644 index 0000000..2a7d445 --- /dev/null +++ b/internal/cmd/templates/_shared/components/Spinner.tsx @@ -0,0 +1,26 @@ +// Canonical single source. apps_init.go copies this into a scaffolded app's +// src/ at generation time, and only if the template imports it — there are no +// standing per-template copies and no sync script. +import { cn } from '../lib/utils'; + +interface SpinnerProps { + size?: 'sm' | 'md'; + className?: string; +} + +const SIZE_CLASSES: Record, string> = { + sm: 'h-3 w-3 border', + md: 'h-4 w-4 border-2', +}; + +export function Spinner({ size = 'md', className }: SpinnerProps) { + return ( + + ); +} diff --git a/internal/cmd/templates/_shared/lib/utils.ts b/internal/cmd/templates/_shared/lib/utils.ts new file mode 100644 index 0000000..a5f1f4d --- /dev/null +++ b/internal/cmd/templates/_shared/lib/utils.ts @@ -0,0 +1,6 @@ +// Canonical single source. apps_init.go copies this into a scaffolded app's +// src/ at generation time, and only if the template imports it — there are no +// standing per-template copies and no sync script. +export function cn(...classes: (string | false | null | undefined)[]): string { + return classes.filter(Boolean).join(' '); +} diff --git a/internal/cmd/templates/agents-md/_common.md b/internal/cmd/templates/agents-md/_common.md new file mode 100644 index 0000000..0c1ab08 --- /dev/null +++ b/internal/cmd/templates/agents-md/_common.md @@ -0,0 +1,78 @@ +# AGENTS.md — building a LangSmith custom app + +This app runs in a sandboxed iframe with no network access of its own — +every LangSmith API call goes through `window.langsmith.call`. `src/entry.tsx` +exports `render(data, root, metadata)`; keep that shape, the sandbox depends +on it. `data` is normally `{}`; call `window.langsmith.setData(patch)` if you +need to push a mutation out for the host to persist. + +## Calling the LangSmith API + +```ts +const projects = await window.langsmith.call('GET /api/v1/sessions', { + params: { limit: '20' }, +}); +``` + +`operation` is `" "` — use the full path including its prefix +(`/api/v1/...` for Python-hosted endpoints, `/v1/platform/...` and `/v2/...` +for Go-hosted ones). `args` carries `params` (query string) and/or `body` +(JSON). This is a generic passthrough, not a curated allowlist — anything +your API key can already do works; a permission error is a real limit of the +key. Full reference: https://docs.langchain.com/langsmith/home. Base URL: +`https://api.smith.langchain.com` (or your self-hosted instance's URL). + +## Theme + +`metadata.mode` is `"dark"` | `"light"`. The sandbox sets `html.dark` from it +before every render, so Tailwind/token-based UIs theme for free with no +branching. Only branch on it yourself if you're using inline styles — and +re-check it every render, since it can change without a remount. + +## Filter DSL for metadata equality + +Query endpoints take a `filter` string. Metadata equality is **two paired +clauses**, not `eq(metadata.key, ...)`: + +``` +and(eq(metadata_key, "ls_agent_purpose"), eq(metadata_value, "coding")) +``` + +Combine with `and(...)` / `or(...)`; other examples: `has(tags, "prod")`, +`eq(name, "ChatOpenAI")`, `eq(is_root, true)`. + + + +## More of the LangSmith API (starting points, not exhaustive) + +**Runs** +- `POST /api/v1/runs/query` — query runs (body: `session`, `filter`, `is_root`, `run_type`, `start_time`, `limit`, `select`); returns `{ runs, cursor }` +- `POST /api/v1/runs/stats` — server-side aggregates over a filtered set of runs, no row limit (counts, error rate, latency percentiles, token/cost sums) — prefer this over paging through `runs/query` for any headline number +- `POST /api/v1/runs/group/stats` — same, grouped (e.g. `group_by: "conversation"` for a distinct thread count) +- `GET /api/v1/runs/{run_id}` — fetch one full run (all fields + inputs/outputs) +- `POST /api/v1/runs` / `PATCH /api/v1/runs/{run_id}` — create / update a run + +**Projects (tracing sessions)** +- `GET /api/v1/sessions` — list projects +- `GET /api/v1/sessions/{session_id}` — get a project + +**Datasets & experiments** +- `GET /api/v1/datasets` — list datasets +- `POST /api/v1/datasets/{dataset_id}/runs` — per-example rows across experiments +- `POST /v1/platform/datasets/{dataset_id}/examples` — create examples + +**Feedback** +- `POST /api/v1/feedback` — create feedback +- `GET /api/v1/feedback?run={run_id}` — list feedback for a run +- `GET /api/v1/feedback-configs?key={key}` — a feedback key's type / direction + +**Annotation queues** +- `GET /api/v1/annotation-queues` — list queues +- `GET /api/v1/annotation-queues/{queue_id}/runs` — list runs in a queue + +**Threads** +- `POST /v2/threads/query` — query threads +- `GET /v2/threads/{thread_id}/traces` — get a thread's traces + +If you need something not listed, check the docs — almost everything in the +LangSmith API is reachable this way. diff --git a/internal/cmd/templates/agents-md/annotation_queue.md b/internal/cmd/templates/agents-md/annotation_queue.md new file mode 100644 index 0000000..701b0c5 --- /dev/null +++ b/internal/cmd/templates/agents-md/annotation_queue.md @@ -0,0 +1,9 @@ +## This template: annotation-queue review + +A 3-pane queue-review UI: run list, inputs/outputs viewer, feedback rubric. +It picks a queue via `GET /api/v1/annotation-queues` and drives everything +else through `window.langsmith.call` — see `src/api.ts`. + +This is just a starting point, not a spec. Change the layout, the review +flow, or the whole concept — rip out anything here and build whatever app +you actually want. diff --git a/internal/cmd/templates/agents-md/annotation_queue_grid.md b/internal/cmd/templates/agents-md/annotation_queue_grid.md new file mode 100644 index 0000000..420271e --- /dev/null +++ b/internal/cmd/templates/agents-md/annotation_queue_grid.md @@ -0,0 +1,9 @@ +## This template: annotation-queue review, as a spreadsheet grid + +Reviews an annotation queue as a spreadsheet grid — rows are queue runs, +columns are rubric keys, editable inline. See `src/api.ts` and +`src/components/DataGrid.tsx`. + +This is just a starting point, not a spec. Change the grid, the review flow, +or the whole concept — rip out anything here and build whatever app you +actually want. diff --git a/internal/cmd/templates/agents-md/coding_agent_dashboard.md b/internal/cmd/templates/agents-md/coding_agent_dashboard.md new file mode 100644 index 0000000..1e8548d --- /dev/null +++ b/internal/cmd/templates/agents-md/coding_agent_dashboard.md @@ -0,0 +1,10 @@ +## This template: coding-agent dashboard + +Headline stats (turns, threads, cost, tokens, errors, latency) plus a +recent-runs table, for one project's coding-agent traces +(`ls_agent_purpose == "coding"`). Stats come from LangSmith's `runs/stats` / +`runs/group/stats` endpoints rather than sampling raw runs — see `src/api.ts`. + +This is just a starting point, not a spec. Change the metrics, the layout, or +the whole concept — rip out anything here and build whatever app you actually +want. diff --git a/internal/cmd/templates/agents-md/experiment_comparison.md b/internal/cmd/templates/agents-md/experiment_comparison.md new file mode 100644 index 0000000..2dba202 --- /dev/null +++ b/internal/cmd/templates/agents-md/experiment_comparison.md @@ -0,0 +1,9 @@ +## This template: experiment-comparison dashboard + +Compares evaluation experiments for one dataset — regressions in feedback +scores, latency, cost, and tokens vs a chosen baseline. See `src/api.ts` for +the calls it makes and `src/lib/delta.ts` for how deltas are computed. + +This is just a starting point, not a spec. Change the metrics, the charts, +or the whole concept — rip out anything here and build whatever app you +actually want. diff --git a/internal/cmd/templates/annotation-queue-grid/.gitignore b/internal/cmd/templates/annotation-queue-grid/.gitignore new file mode 100644 index 0000000..b947077 --- /dev/null +++ b/internal/cmd/templates/annotation-queue-grid/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +dist/ diff --git a/internal/cmd/templates/annotation-queue-grid/README.md b/internal/cmd/templates/annotation-queue-grid/README.md new file mode 100644 index 0000000..a1cefb9 --- /dev/null +++ b/internal/cmd/templates/annotation-queue-grid/README.md @@ -0,0 +1,90 @@ +# {{.Name}} + +{{.Description}} + +A LangSmith custom app, scaffolded by `langsmith apps init` — a real, +working annotation-queue review UI rendered as a **spreadsheet**: one row per +queue run, one column per rubric key, cells edited inline and saved as you go, +`Mark Completed` to complete a row. Meant to vibe-code from here, not a toy example. +**Read `AGENTS.md` first** — it documents the LangSmith API surface this app +can call. + +## What this is + +A custom app is a small React/TS UI rendered inside LangSmith in a +locked-down sandbox (`sandbox="allow-scripts"`, no `allow-same-origin`). It +never gets direct network access or a real credential — everything goes +through a `postMessage` bridge to the host page (`window.langsmith.call`), +which proxies to the real LangSmith API under the viewer's own session (or +your local `LANGSMITH_API_KEY` when running `langsmith apps dev`). + +Since the sandbox has no bundler or npm access at runtime, this app is built +with Vite in **library mode** into a single dependency-free CJS file +(`dist/bundle.js`) — see `vite.config.ts` and `src/entry.tsx`. Use real +React/TS and npm dependencies freely; they all get inlined at build time. +Tailwind's compiled CSS is inlined too (`src/index.css` imported with +`?inline` in `entry.tsx`) and injected via a `