[experimental][draft] in-browser agent for playing with ComfyUI

browser_tests/fixtures/ComfyPage.ts

@@ -518,6 +518,16 @@ export const comfyPageFixture = base.extend<{
 
     await comfyPage.setup()
 
+    // Hide agent UI in all tests except those explicitly testing the agent.
+    // The FAB is positioned over the canvas viewport, which would cause
+    // unrelated screenshot tests to fail.
+    if (!testInfo.tags.includes('@agent')) {
+      await page.addStyleTag({
+        content:
+          '[data-testid="agent-fab"],[data-testid="agent-panel"]{display:none!important}'
+      })
+    }
+
     if (isVueNodes) {
       await comfyPage.vueNodes.waitForNodes()
     }

browser_tests/tests/agentTerminal.spec.ts

@@ -0,0 +1,162 @@
+import { expect } from '@playwright/test'
+
+import type { ComfyPage } from '@e2e/fixtures/ComfyPage'
+import { comfyPageFixture as test } from '@e2e/fixtures/ComfyPage'
+
+/**
+ * E2E coverage for the in-browser agent terminal (AgentFab + FoldablePanel).
+ *
+ * The panel is now a Vue-native scrollback (no xterm.js), so the tests
+ * target the plain DOM directly: the input is a `<textarea>` inside
+ * `[data-testid="agent-terminal"]`, and the scrollback lives in the same
+ * container as a list of message blocks. We exercise the deterministic
+ * shell surface — typing into the textarea runs commands directly through
+ * the runtime, which is what the LLM ends up calling via `run_shell`.
+ */
+
+async function openPanel(comfyPage: ComfyPage): Promise<void> {
+  const fab = comfyPage.page.getByTestId('agent-fab')
+  await expect(fab).toBeVisible()
+  await fab.click()
+  await expect(comfyPage.page.getByTestId('agent-panel')).toBeVisible()
+}
+
+async function readTerminalText(comfyPage: ComfyPage): Promise<string> {
+  return await comfyPage.page.getByTestId('agent-terminal').innerText()
+}
+
+async function typeAndEnter(comfyPage: ComfyPage, text: string): Promise<void> {
+  const input = comfyPage.page.getByTestId('agent-terminal').locator('textarea')
+  await input.focus()
+  await comfyPage.page.keyboard.type(text)
+  await comfyPage.page.keyboard.press('Enter')
+}
+
+test.describe('Agent terminal', { tag: ['@ui', '@agent'] }, () => {
+  test('FAB opens the panel and shows the COMFY-AI title + prompt', async ({
+    comfyPage
+  }) => {
+    await openPanel(comfyPage)
+
+    await expect(comfyPage.page.getByTestId('agent-panel-title')).toHaveText(
+      'COMFY-AI'
+    )
+    await expect.poll(() => readTerminalText(comfyPage)).toMatch(/comfy>/)
+  })
+
+  test('Clicking the FAB again closes the panel', async ({ comfyPage }) => {
+    await openPanel(comfyPage)
+    await comfyPage.page.getByTestId('agent-fab').click()
+    await expect(comfyPage.page.getByTestId('agent-panel')).toBeHidden()
+  })
+
+  test('Enter submits; help command lists built-ins', async ({ comfyPage }) => {
+    await openPanel(comfyPage)
+    await typeAndEnter(comfyPage, 'help')
+    await expect
+      .poll(() => readTerminalText(comfyPage))
+      .toMatch(/run-js|cmd-list|comfy/)
+  })
+
+  test('Shift+Enter inserts a literal newline (no submit)', async ({
+    comfyPage
+  }) => {
+    await openPanel(comfyPage)
+    const input = comfyPage.page
+      .getByTestId('agent-terminal')
+      .locator('textarea')
+    await input.focus()
+    await comfyPage.page.keyboard.type('echo one')
+    await comfyPage.page.keyboard.press('Shift+Enter')
+    await comfyPage.page.keyboard.type('echo two')
+    // Single submission should run BOTH lines as one multi-line script.
+    await comfyPage.page.keyboard.press('Enter')
+
+    const out = await readTerminalText(comfyPage)
+    expect(out).toContain('one')
+    expect(out).toContain('two')
+  })
+
+  test('coreutils: pwd / echo', async ({ comfyPage }) => {
+    await openPanel(comfyPage)
+    await typeAndEnter(comfyPage, 'pwd')
+    await expect.poll(() => readTerminalText(comfyPage)).toMatch(/^\//m)
+
+    await typeAndEnter(comfyPage, 'echo hello world')
+    await expect
+      .poll(() => readTerminalText(comfyPage))
+      .toContain('hello world')
+  })
+
+  test('comfy namespace lists subcommands', async ({ comfyPage }) => {
+    await openPanel(comfyPage)
+    await typeAndEnter(comfyPage, 'comfy')
+    await expect
+      .poll(() => readTerminalText(comfyPage))
+      .toMatch(/ComfyUI command namespace/)
+  })
+
+  test('run-js evaluates in the page scope', async ({ comfyPage }) => {
+    await openPanel(comfyPage)
+    await typeAndEnter(comfyPage, 'run-js return 1 + 2')
+    await expect.poll(() => readTerminalText(comfyPage)).toMatch(/\b3\b/)
+  })
+
+  test('graph summary reports node count for the active graph', async ({
+    comfyPage
+  }) => {
+    await openPanel(comfyPage)
+    await typeAndEnter(comfyPage, 'graph summary')
+    await expect
+      .poll(() => readTerminalText(comfyPage))
+      .toMatch(/node|count|nodes/i)
+  })
+
+  test('queue-status command returns output', async ({ comfyPage }) => {
+    await openPanel(comfyPage)
+    await typeAndEnter(comfyPage, 'queue-status')
+    await expect
+      .poll(() => readTerminalText(comfyPage))
+      .toMatch(/running|pending|queue/i)
+  })
+
+  test('active-workflow reports path / state', async ({ comfyPage }) => {
+    await openPanel(comfyPage)
+    await typeAndEnter(comfyPage, 'active-workflow')
+    await expect
+      .poll(() => readTerminalText(comfyPage))
+      .toMatch(/path|modified|persisted|none/i)
+  })
+
+  test('pipe: echo foo | wc -c emits a byte count', async ({ comfyPage }) => {
+    await openPanel(comfyPage)
+    await typeAndEnter(comfyPage, 'echo foo | wc -c')
+    // "foo\n" = 4 bytes
+    await expect.poll(() => readTerminalText(comfyPage)).toMatch(/\b4\b/)
+  })
+
+  test('unknown command surfaces an error', async ({ comfyPage }) => {
+    await openPanel(comfyPage)
+    await typeAndEnter(comfyPage, 'definitely-not-a-real-command-xyz')
+    await expect
+      .poll(() => readTerminalText(comfyPage))
+      .toMatch(/not found|unknown|no such/i)
+  })
+
+  test('Ctrl+O folds and unfolds tool blocks', async ({ comfyPage }) => {
+    await openPanel(comfyPage)
+    await typeAndEnter(comfyPage, 'graph summary')
+    // Tool blocks default to folded — body shouldn't be visible yet.
+    const panel = comfyPage.page.getByTestId('agent-panel')
+    await expect(
+      panel.locator('button:has-text("graph summary")')
+    ).toBeVisible()
+
+    // Ctrl+O expands all
+    await comfyPage.page.keyboard.press('Control+o')
+    await expect.poll(() => readTerminalText(comfyPage)).toMatch(/nodes|types/i)
+
+    // Ctrl+O folds all back — `nodes:` from the body should be hidden again.
+    await comfyPage.page.keyboard.press('Control+o')
+  })
+})

build/plugins/agentLog.ts

@@ -0,0 +1,90 @@
+import { appendFileSync, existsSync, mkdirSync } from 'fs'
+import { join } from 'path'
+import type { Plugin } from 'vite'
+
+/**
+ * Dev-only Vite plugin: accept POSTs to /__agent-log and append each
+ * JSONL line to a per-session file under ./tmp/agent-logs/.
+ *
+ * Filename: ./tmp/agent-logs/<YYYY-MM-DD>-<sessionId>.jsonl
+ *   - <sessionId> is the 8-char id assigned in the browser logger and
+ *     attached to every entry. One file per page load makes individual
+ *     conversations trivially diff-able and grep-able without sifting
+ *     through a daily mixed log.
+ *   - Entries without a sessionId fall back to '<date>-orphan.jsonl' so
+ *     unattributed lines don't get silently dropped.
+ *
+ * GET /__agent-log → returns the directory + a 1-line summary of recent
+ * session files (debugging aid).
+ *
+ * No-op in production builds (apply: 'serve'). Same origin as the Vite
+ * dev server so the browser-side logger can POST with a simple fetch().
+ */
+export function agentLogPlugin(): Plugin {
+  const LOG_DIR = join(process.cwd(), 'tmp', 'agent-logs')
+
+  return {
+    name: 'agent-log',
+    apply: 'serve',
+    configureServer(server) {
+      server.middlewares.use('/__agent-log', (req, res) => {
+        if (req.method === 'GET') {
+          res.setHeader('Content-Type', 'application/json')
+          res.end(JSON.stringify({ dir: LOG_DIR, mode: 'per-session' }))
+          return
+        }
+
+        if (req.method !== 'POST') {
+          res.statusCode = 405
+          res.end()
+          return
+        }
+
+        const chunks: Buffer[] = []
+        req.on('data', (c: Buffer) => chunks.push(c))
+        req.on('end', () => {
+          try {
+            if (!existsSync(LOG_DIR)) {
+              mkdirSync(LOG_DIR, { recursive: true })
+            }
+            const date = new Date().toISOString().slice(0, 10)
+            const body = Buffer.concat(chunks).toString('utf8')
+
+            // Group lines by sessionId so a single batch carrying multiple
+            // sessions (rare but possible) lands in the right files.
+            const buckets = new Map<string, string[]>()
+            for (const raw of body.split('\n')) {
+              const line = raw.trim()
+              if (!line) continue
+              let sessionId = 'orphan'
+              try {
+                const parsed = JSON.parse(line) as { sessionId?: string }
+                if (
+                  parsed.sessionId &&
+                  /^[A-Za-z0-9-]{1,64}$/.test(parsed.sessionId)
+                ) {
+                  sessionId = parsed.sessionId
+                }
+              } catch {
+                // Keep raw text in the orphan bucket; don't drop it.
+              }
+              const arr = buckets.get(sessionId) ?? []
+              arr.push(line)
+              buckets.set(sessionId, arr)
+            }
+
+            for (const [sessionId, lines] of buckets) {
+              const file = join(LOG_DIR, `${date}-${sessionId}.jsonl`)
+              appendFileSync(file, lines.join('\n') + '\n', 'utf8')
+            }
+            res.statusCode = 204
+            res.end()
+          } catch (err) {
+            res.statusCode = 500
+            res.end(err instanceof Error ? err.message : String(err))
+          }
+        })
+      })
+    }
+  }
+}

build/plugins/index.ts

@@ -1 +1,2 @@
+export { agentLogPlugin } from './agentLog'
 export { comfyAPIPlugin } from './comfyAPIPlugin'