[Test] Web acceptance stability

.github/workflows/44-railway-tests.yml

@@ -252,7 +252,7 @@ jobs:
 
             echo "Waiting for ${label}: ${url}"
             for attempt in $(seq 1 30); do
-              if curl -sfL -o /dev/null "$url" 2>/dev/null; then
+              if curl -sfL --max-time 10 --connect-timeout 5 -o /dev/null "$url" 2>/dev/null; then
                 echo "${label} is ready."
                 return 0
               fi

web/ee/tests/playwright/acceptance/members/index.ts

@@ -33,20 +33,6 @@ const lightFastTags = buildAcceptanceTags({
 const createInviteEmail = (scope: string) =>
     `${scope}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}@agenta.test`
 
-const waitForInviteResponse = async (page: any) => {
-    const response = await page.waitForResponse(
-        (res: any) =>
-            res.request().method() === "POST" &&
-            res.url().includes("/workspaces/") &&
-            res.url().includes("/invite?"),
-        {timeout: 15000},
-    )
-
-    if (!response.ok()) {
-        throw new Error(`Invite request failed (${response.status()}): ${await response.text()}`)
-    }
-}
-
 const waitForRemoveResponse = async (page: any) => {
     const response = await page.waitForResponse(
         (res: any) =>
@@ -64,6 +50,59 @@ const waitForRemoveResponse = async (page: any) => {
     }
 }
 
+const openInviteMembersModal = async (page: any) => {
+    const inviteButton = page.getByRole("button", {name: "Invite Members"}).first()
+    await expect(inviteButton).toBeVisible({timeout: 20000})
+    await expect(inviteButton).toBeEnabled()
+
+    const inviteModal = page.getByRole("dialog", {name: "Invite Members"})
+
+    // Use a PAGE-LEVEL (unscoped) locator for the email input.
+    //
+    // Scoping through `inviteModal.getByPlaceholder(...)` is unreliable here because:
+    //   1. InviteUsersModal is a `dynamic()` import — the form mounts AFTER the modal
+    //      wrapper becomes visible, so the dialog-scoped locator resolves to zero elements
+    //      until the JS chunk fully evaluates.
+    //   2. rc-dialog briefly UNMOUNTS content while its `animatedVisible` useEffect
+    //      settles (fires on the next frame after first render), making a dialog-scoped
+    //      locator transiently stale.
+    // Searching the entire page avoids both issues while remaining unique in practice
+    // (only one invite form is ever present at a time).
+    const emailInput = page.getByPlaceholder("member@organization.com").first()
+
+    for (let attempt = 0; attempt < 3; attempt++) {
+        // Ensure any previous dialog is closed before clicking again.
+        const alreadyOpen = await inviteModal.isVisible().catch(() => false)
+        if (!alreadyOpen) {
+            await inviteButton.click()
+        }
+
+        // Wait for the email input to become visible. This is the most reliable
+        // signal that both the modal wrapper AND its dynamic content are ready.
+        const inputAppeared = await emailInput
+            .waitFor({state: "visible", timeout: 15000})
+            .then(() => true)
+            .catch(() => false)
+
+        if (inputAppeared) {
+            return {inviteModal, emailInput}
+        }
+
+        // Form never appeared — dismiss any partial modal and retry.
+        await page.keyboard.press("Escape")
+        await page.waitForTimeout(500)
+    }
+
+    // Final assertion: surfaces a clear error if the input never appeared.
+    await expect(emailInput).toBeVisible({timeout: 15000})
+    return {inviteModal, emailInput}
+}
+
+const submitInviteMembersModal = async (inviteModal: any) => {
+    await inviteModal.locator("form").evaluate((form: HTMLFormElement) => form.requestSubmit())
+    await expect(inviteModal).not.toBeVisible({timeout: 30000})
+}
+
 /**
  * Invite a member via the EE flow (email sent) and wait for their row to appear
  * in the members table with "Invitation Pending" status.
@@ -76,23 +115,23 @@ const invitePendingMember = async (page: any, apiHelpers: any, uiHelpers: any):
     await page.goto(`${basePath}/settings`, {waitUntil: "domcontentloaded"})
     await uiHelpers.expectPath("/settings")
 
-    const inviteButton = page.getByRole("button", {name: "Invite Members"})
-    await expect(inviteButton).toBeVisible({timeout: 20000})
-    await inviteButton.click()
-
-    const inviteModal = page.getByRole("dialog", {name: "Invite Members"})
-    const emailInput = inviteModal.getByPlaceholder("member@organization.com")
+    const {inviteModal, emailInput} = await openInviteMembersModal(page)
     // Wait for the email input rather than just the dialog — the InviteUsersModal
     // is a dynamic() import, so the form body can lag behind the modal wrapper.
     // Waiting for the input guarantees the chunk has fully rendered.
-    await expect(emailInput).toBeVisible({timeout: 20000})
+    // Click before fill: rc-component/dialog briefly unmounts while animatedVisible
+    // catches up (useEffect fires after first render), which makes the locator
+    // stale. A click forces Playwright to wait for the element to be fully
+    // interactive before fill attempts to interact.
+    await emailInput.click()
     await emailInput.fill(testEmail)
 
-    await Promise.all([
-        waitForInviteResponse(page),
-        inviteModal.getByRole("button", {name: "Invite"}).click(),
-    ])
-    await expect(inviteModal).not.toBeVisible({timeout: 15000})
+    // Submit the form and wait for the modal to close as the success signal.
+    // The InviteUsersModal only closes its onSuccess callback after the API
+    // returns successfully — so modal closure == invite accepted.
+    // Waiting for the network response by URL is fragile (URL-pattern drift,
+    // timing races between listener registration and the async form submit).
+    await submitInviteMembersModal(inviteModal)
 
     // Wait for the pending row to appear in the refreshed table
     await expect(page.getByText(testEmail)).toBeVisible({timeout: 15000})
@@ -106,7 +145,8 @@ const membersTests = () => {
         "should invite a member and verify pending state",
         {tag: lightFastTags},
         async ({page, apiHelpers, uiHelpers}) => {
-            test.setTimeout(60000)
+            // 90 s: navigation + up to 3 modal-open attempts × 15 s + fill + submit + assertion
+            test.setTimeout(90000)
             const testEmail = createInviteEmail("test-member-invite")
 
             await scenarios.given("the user is authenticated", async () => {
@@ -125,13 +165,14 @@ const membersTests = () => {
             await scenarios.when(
                 "the user clicks Invite Members, fills in an email address and selects a role",
                 async () => {
-                    await page.getByRole("button", {name: "Invite Members"}).click()
-
-                    const inviteModal = page.getByRole("dialog", {name: "Invite Members"})
-                    const emailInput = inviteModal.getByPlaceholder("member@organization.com")
+                    const {inviteModal, emailInput} = await openInviteMembersModal(page)
                     // Wait for the input directly — the InviteUsersModal is a dynamic()
                     // import so the form body can lag behind the modal wrapper appearing.
-                    await expect(emailInput).toBeVisible({timeout: 20000})
+                    // Click before fill: rc-component/dialog briefly unmounts the panel
+                    // while animatedVisible settles (useEffect fires after first render),
+                    // making the locator transiently stale. Clicking first ensures the
+                    // element is fully interactive before fill runs.
+                    await emailInput.click()
                     await emailInput.fill(testEmail)
 
                     // EE renders a role selector; keep the default selection
@@ -144,11 +185,10 @@ const membersTests = () => {
 
             await scenarios.and("the user submits the invitation", async () => {
                 const inviteModal = page.getByRole("dialog", {name: "Invite Members"})
-                await Promise.all([
-                    waitForInviteResponse(page),
-                    inviteModal.getByRole("button", {name: "Invite"}).click(),
-                ])
-                await expect(inviteModal).not.toBeVisible({timeout: 15000})
+                // Submit the form and wait for the modal to dismiss as the success signal.
+                // The InviteUsersModal only closes after the API returns successfully,
+                // so modal closure is equivalent to a successful invite response.
+                await submitInviteMembersModal(inviteModal)
             })
 
             await scenarios.then(

web/ee/tests/playwright/acceptance/use-api/use-api.spec.ts

@@ -0,0 +1,4 @@
+import {test} from "@agenta/web-tests/tests/fixtures/base.fixture"
+import useApiTests from "@agenta/oss/tests/playwright/10-use-api"
+
+test.describe("Registry: use API snippets", useApiTests)

web/oss/tests/playwright/10-use-api.ts

@@ -0,0 +1,3 @@
+import useApiTests from "./acceptance/use-api"
+
+export default useApiTests

web/oss/tests/playwright/acceptance/app/test.ts

@@ -43,6 +43,11 @@ export const openCreateAppDrawerForType = async (
         .or(page.getByTestId(modalTypeTestId))
         .first()
 
+    const drawer = page
+        .getByRole("dialog")
+        .filter({has: page.getByTestId("app-create-name-input")})
+        .last()
+
     for (let attempt = 0; attempt < 3; attempt += 1) {
         for (const entryPoint of createEntryPoints) {
             if (!(await entryPoint.isVisible().catch(() => false))) continue
@@ -52,34 +57,43 @@ export const openCreateAppDrawerForType = async (
             break
         }
 
-        const opened = await typeSelector
-            .waitFor({state: "visible", timeout: 3000})
+        const typeSelectorVisible = await typeSelector
+            .waitFor({state: "visible", timeout: 4000})
             .then(() => true)
             .catch(() => false)
 
-        if (opened) {
-            // The Popover re-renders when appTemplatesQueryAtom resolves,
-            // making the item briefly unstable. force:true dispatches the
-            // click immediately without waiting for Playwright's stability
-            // check, which otherwise retries until the 60 s test timeout.
-            await typeSelector.click({force: true})
-            const drawer = page
-                .getByRole("dialog")
-                .filter({has: page.getByTestId("app-create-name-input")})
-                .last()
-            await expect(drawer).toBeVisible({timeout: 15000})
+        if (!typeSelectorVisible) {
+            await page.keyboard.press("Escape").catch(() => undefined)
+            continue
+        }
+
+        // The Popover re-renders when appTemplatesQueryAtom resolves, making
+        // the item briefly unstable. dispatchEvent('click') fires a synthetic
+        // DOM event that bypasses both Playwright's stability check AND the
+        // viewport-position check (newer Playwright no longer allows force:true
+        // to click elements outside the viewport). The drawer check below
+        // catches the rare case where the click still missed.
+        await typeSelector.dispatchEvent("click")
+
+        // Check whether the drawer opened. If the click landed on a stale
+        // element during re-render it won't appear — retry rather than throw.
+        const drawerOpened = await drawer
+            .waitFor({state: "visible", timeout: 8000})
+            .then(() => true)
+            .catch(() => false)
+
+        if (drawerOpened) {
             return drawer
         }
 
+        // Drawer didn't open — dismiss any leftover popover and try again.
         await page.keyboard.press("Escape").catch(() => undefined)
+        await page.waitForTimeout(200)
     }
 
+    // Final attempt: surfaces a clear failure if the drawer still won't open.
     await expect(typeSelector).toBeVisible({timeout: 15000})
-    await typeSelector.click()
-    const drawer = page
-        .getByRole("dialog")
-        .filter({has: page.getByTestId("app-create-name-input")})
-        .last()
+    await typeSelector.dispatchEvent("click")
     await expect(drawer).toBeVisible({timeout: 15000})
     return drawer
 }

web/oss/tests/playwright/acceptance/evaluators/tests.ts

@@ -351,6 +351,9 @@ const createHumanEvaluatorFromDrawer = async (
     await nameInput.fill(evaluatorName)
     await expect(nameInput).toHaveValue(evaluatorName)
 
+    const slugInput = drawer.locator('input[placeholder="Enter a unique slug"]').first()
+    await expect(slugInput).toHaveValue(evaluatorName, {timeout: 5000})
+
     // Fill in the feedback name (the first metric row)
     const feedbackNameInput = drawer
         .locator(`input[placeholder="${HUMAN_EVALUATOR_FEEDBACK_NAME_PLACEHOLDER}"]`)

web/oss/tests/playwright/acceptance/features/use-api.feature

@@ -0,0 +1,40 @@
+# Tests: use-api/use-api.spec.ts -> use-api/index.ts
+# RTM IDs: WEB-ACC-USEAPI-001, WEB-ACC-USEAPI-002
+# Tags: @scope:deployment @coverage:light @path:happy @speed:fast
+#
+# Implementation notes:
+# - Variant mode: navigate to /apps/{id}/variants, click "Use API" (primary button)
+#   → DeploymentsDrawer opens in mode="variant" rendering VariantUseApiContent
+#   → Fetch Prompt/Config snippet uses application_variant_ref (variant-keyed endpoint)
+#   → Invoke LLM snippet uses axios.post to the variant invocation URL
+# - Deployment mode: navigate to /apps/{id}/variants?tab=deployments&selectedEnvName=development
+#   → click "Use API" (primary button in the deployments tab header)
+#   → DeploymentsDrawer opens in mode="deployment" rendering UseApiContent
+#   → Fetch Prompt/Config snippet uses environment_ref (environment-keyed endpoint)
+#   → Invoke LLM snippet uses axios.post to the environment invocation URL
+# - Both tests assert TypeScript tab only (Python and cURL are separate language concerns).
+
+Feature: Registry Use API — TypeScript snippets
+  As a user
+  I want to view TypeScript code snippets for calling my app via API
+  So that I can integrate Agenta into my TypeScript project
+
+  Background:
+    Given the user is authenticated
+    And a completion app with at least one variant exists
+
+  @light @happy @scope:deployment @speed:fast
+  Scenario: Variant TypeScript snippet shows application_variant_ref in Fetch section
+    Given the user is on the Variants registry page
+    When the user opens the Use API drawer
+    And the user selects the TypeScript tab
+    Then the Fetch Prompt/Config section displays the variant TypeScript snippet
+    And the Invoke LLM section displays a TypeScript axios snippet
+
+  @light @happy @scope:deployment @speed:fast
+  Scenario: Deployment TypeScript snippet shows environment_ref in Fetch section
+    Given the user is on the Deployments registry page for the Development environment
+    When the user opens the Use API drawer
+    And the user selects the TypeScript tab
+    Then the Fetch Prompt/Config section displays the deployment TypeScript snippet
+    And the Invoke LLM section displays a TypeScript axios snippet

web/oss/tests/playwright/acceptance/members/index.ts

@@ -36,7 +36,8 @@ const membersTests = () => {
         "should invite a member and show the invite link modal",
         {tag: lightFastTags},
         async ({page, apiHelpers, uiHelpers}) => {
-            test.setTimeout(60000)
+            // 90 s: settings page load (~5 s) + invite API call (~15 s) + modal assertions
+            test.setTimeout(90000)
             const testEmail = `test-member-invite-${Date.now()}@agenta-e2e.test`
 
             await scenarios.given("the user is authenticated", async () => {
@@ -49,7 +50,7 @@ const membersTests = () => {
                 await uiHelpers.expectPath("/settings")
                 // The default tab is "workspace" which renders the Members section
                 await expect(page.getByRole("button", {name: "Invite Members"})).toBeVisible({
-                    timeout: 15000,
+                    timeout: 20000,
                 })
             })
 
@@ -69,26 +70,44 @@ const membersTests = () => {
 
             await scenarios.and("the user submits the invitation", async () => {
                 const inviteModal = page.getByRole("dialog", {name: "Invite Members"})
+
+                // Set up the API response listener BEFORE clicking so we don't miss the
+                // response if the API is fast.
+                const inviteResponsePromise = page
+                    .waitForResponse(
+                        (r) => /\/invite/i.test(r.url()) && r.request().method() === "POST",
+                        {timeout: 30000},
+                    )
+                    .catch(() => null)
+
                 await inviteModal.getByRole("button", {name: "Invite"}).click()
-                // Invite modal closes before link modal opens
-                await expect(inviteModal).not.toBeVisible({timeout: 15000})
+
+                // Wait for the API call to complete before asserting modal state.
+                await inviteResponsePromise
+
+                // Invite modal should close once the server responds with the invite link.
+                await expect(inviteModal).not.toBeVisible({timeout: 20000})
             })
 
             await scenarios.then(
                 "the invited user link modal appears with a shareable URL",
                 async () => {
                     const linkModal = page.getByRole("dialog", {name: "Invited user link"})
-                    await expect(linkModal).toBeVisible({timeout: 15000})
+                    await expect(linkModal).toBeVisible({timeout: 20000})
 
                     // Verify the modal shows the invited email
-                    await expect(linkModal.getByText(testEmail)).toBeVisible({timeout: 5000})
+                    await expect(linkModal.getByText(testEmail)).toBeVisible({timeout: 10000})
 
                     // Verify the invite URL is present
                     await expect(linkModal.getByText(/https?:\/\//)).toBeVisible({timeout: 5000})
 
                     // Close via the X button — "Copy & Close" calls navigator.clipboard which
                     // throws in headless CI, preventing onCancel from being called.
-                    await linkModal.locator('button[aria-label="Close"]').click()
+                    const closeButton = linkModal
+                        .locator('button[aria-label="Close"]')
+                        .or(linkModal.getByRole("button", {name: "Close"}))
+                        .first()
+                    await closeButton.click()
                     await expect(linkModal).not.toBeVisible({timeout: 10000})
                 },
             )