Improve observe tests (#592)

* Improve Observe Tests

* p1

* px

* px

* px

* px

* cleanup

* px

* px

* Prepare

Author: incorbador

.github/workflows/test-all.yml

@@ -1,6 +1,4 @@
 name: Test on React
-env:
-  BRANCH_NAME_RAW: ${{ github.head_ref || github.ref_name }}
 
 on:
   pull_request:
@@ -12,6 +10,7 @@ jobs:
   test:
     timeout-minutes: 60
     runs-on: ubuntu-24.04
+    environment: ${{ matrix.testType }}
     strategy:
       fail-fast: false
       matrix:
@@ -20,33 +19,29 @@ jobs:
             configFile: playwright.config.complete.ts
           - testType: connect
             configFile: playwright.config.connect.ts
-    env:
-      PLAYWRIGHT_NUM_CORES:       4
-      PLAYWRIGHT_TEST_URL:        ${{ secrets.PLAYWRIGHT_TEST_URL }}
 
-      PLAYWRIGHT_JWT_TOKEN:       ${{ secrets.PLAYWRIGHT_JWT_TOKEN }}
-      PLAYWRIGHT_GOOGLE_EMAIL:    ${{ secrets.PLAYWRIGHT_GOOGLE_EMAIL }}
-      PLAYWRIGHT_GOOGLE_PASSWORD: ${{ secrets.PLAYWRIGHT_GOOGLE_PASSWORD }}
+    env:
+      # ── Shared (hardcoded) ──
+      PLAYWRIGHT_NUM_CORES: 4
+      DEFAULT_CORBADO_BACKEND_API_BASIC_AUTH: ${{ secrets.DEFAULT_CORBADO_BACKEND_API_BASIC_AUTH }}
+
+      # ── "complete" environment secrets (empty on connect runners) ──
+      PLAYWRIGHT_JWT_TOKEN:          ${{ secrets.PLAYWRIGHT_JWT_TOKEN }}
+      PLAYWRIGHT_GOOGLE_EMAIL:       ${{ secrets.PLAYWRIGHT_GOOGLE_EMAIL }}
+      PLAYWRIGHT_GOOGLE_PASSWORD:    ${{ secrets.PLAYWRIGHT_GOOGLE_PASSWORD }}
       PLAYWRIGHT_GOOGLE_TOTP_SECRET: ${{ secrets.PLAYWRIGHT_GOOGLE_TOTP_SECRET }}
 
-      NEXT_PUBLIC_CORBADO_PROJECT_ID: ${{ secrets.NEXT_PUBLIC_CORBADO_PROJECT_ID }}
-      NEXT_PUBLIC_CORBADO_FRONTEND_API_URL_SUFFIX: ${{ secrets.NEXT_PUBLIC_CORBADO_FRONTEND_API_URL_SUFFIX }}
-      CORBADO_BACKEND_API_URL: ${{ secrets.CORBADO_BACKEND_API_URL }}
-      CORBADO_BACKEND_API_BASIC_AUTH:     ${{ secrets.CORBADO_BACKEND_API_BASIC_AUTH }}
-      PLAYWRIGHT_CONNECT_PROJECT_ID: ${{ secrets.PLAYWRIGHT_CONNECT_PROJECT_ID }}
-      PLAYWRIGHT_NGROK_AUTH_TOKEN:   ${{ secrets.PLAYWRIGHT_NGROK_AUTH_TOKEN }}
-      AWS_COGNITO_USER_POOL_ID:   ${{ secrets.AWS_COGNITO_USER_POOL_ID }}
-      NEXT_PUBLIC_AWS_COGNITO_USER_POOL_ID: ${{ secrets.AWS_COGNITO_USER_POOL_ID }}
-      AWS_COGNITO_CLIENT_ID:   ${{ secrets.AWS_COGNITO_CLIENT_ID }}
-      NEXT_PUBLIC_AWS_COGNITO_CLIENT_ID:   ${{ vars.NEXT_PUBLIC_AWS_COGNITO_CLIENT_ID }}
-      AWS_COGNITO_CLIENT_SECRET:   ${{ secrets.AWS_COGNITO_CLIENT_SECRET }}
-      AWS_REGION:   ${{ secrets.AWS_REGION }}
-      AWS_ACCESS_KEY_ID:   ${{ secrets.AWS_ACCESS_KEY_ID_CONNECT_PLAYGROUND }}
-      AWS_SECRET_ACCESS_KEY:   ${{ secrets.AWS_SECRET_ACCESS_KEY_CONNECT_PLAYGROUND }}
-
-      GITHUB_RUN_ID:              ${{ github.run_id }}
-      SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }}
-      GITHUB_BRANCH_NAME:         ${{ github.ref_name }}
+      # ── "connect" environment secrets (empty on complete runners) ──
+      NEXT_PUBLIC_AWS_COGNITO_USER_POOL_ID:        ${{ secrets.AWS_COGNITO_USER_POOL_ID }}
+      NEXT_PUBLIC_AWS_COGNITO_CLIENT_ID:           ${{ vars.NEXT_PUBLIC_AWS_COGNITO_CLIENT_ID }}
+      CORBADO_BACKEND_API_BASIC_AUTH:              ${{ secrets.DEFAULT_CORBADO_BACKEND_API_BASIC_AUTH }}
+      PLAYWRIGHT_NGROK_AUTH_TOKEN:                 ${{ secrets.PLAYWRIGHT_NGROK_AUTH_TOKEN }}
+      AWS_COGNITO_USER_POOL_ID:                    ${{ secrets.AWS_COGNITO_USER_POOL_ID }}
+      AWS_COGNITO_CLIENT_ID:                       ${{ secrets.AWS_COGNITO_CLIENT_ID }}
+      AWS_COGNITO_CLIENT_SECRET:                   ${{ secrets.AWS_COGNITO_CLIENT_SECRET }}
+      AWS_REGION:                                  ${{ secrets.AWS_REGION }}
+      AWS_ACCESS_KEY_ID:                           ${{ secrets.COGNITO_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY:                       ${{ secrets.COGNITO_AWS_SECRET_ACCESS_KEY }}
 
     steps:
       - name: Checkout code
@@ -59,7 +54,7 @@ jobs:
 
       - name: Cache node modules
         id: cache-npm
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cache-node-modules
         with:
@@ -86,23 +81,22 @@ jobs:
 
       - name: Get installed Playwright version
         id: playwright-version
-        run: echo "::set-output name=version::$(yarn why --json @playwright/test | grep -h 'workspace:.' | jq --raw-output '.children[].locator' | sed -e 's/@playwright\/test@.*://')"
+        run: echo "version=$(yarn why --json @playwright/test | grep -h 'workspace:.' | jq --raw-output '.children[].locator' | sed -e 's/@playwright\/test@.*://')" >> "$GITHUB_OUTPUT"
 
       - if: ${{ steps.cache-npm.outputs.cache-hit != 'true' }}
         name: List the state of node modules
         continue-on-error: true
-        run: |
-          cd packages/tests-e2e
-          npm list
+        working-directory: packages/tests-e2e
+        run: npm list
 
       - name: Install dependencies
+        working-directory: packages/tests-e2e
         run: |
-          cd packages/tests-e2e
           npm install
           sudo apt-get update
           sudo apt-get install -y libxml2-utils
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         id: playwright-cache
         with:
           path: '~/.cache/ms-playwright'
@@ -112,26 +106,24 @@ jobs:
 
       - name: Install Playwright's dependencies
         if: steps.playwright-cache.outputs.cache-hit != 'true'
-        run: |
-          cd packages/tests-e2e
-          npx playwright install --with-deps
+        working-directory: packages/tests-e2e
+        run: npx playwright install --with-deps
 
-      - name: Create a log stream on AWS CloudWatch
+      - name: Create CloudWatch log stream
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ vars.CLOUDWATCH_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDWATCH_AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: ${{ secrets.AWS_REGION }}
         run: |
           TIMESTAMP=$(date +%s000)
-          RANDOM_SUFFIX=$(head /dev/urandom | tr -dc a-z0-9 | head -c 6)
+          RANDOM_SUFFIX=$(head /dev/urandom | tr -dc 'a-z0-9' | head -c 6)
           LOG_STREAM_NAME="javascript-${TIMESTAMP}-${RANDOM_SUFFIX}"
-          echo "LOG_STREAM_NAME=$LOG_STREAM_NAME" >> $GITHUB_ENV
+          echo "LOG_STREAM_NAME=$LOG_STREAM_NAME" >> "$GITHUB_ENV"
           aws logs create-log-stream --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME"
 
       - name: Run ${{ matrix.testType }} tests for react
         working-directory: packages/tests-e2e
-        run: |
-          npx playwright test --config=${{ matrix.configFile }}
+        run: npx playwright test --config=${{ matrix.configFile }}
 
       - uses: actions/upload-artifact@v4
         if: always()
@@ -140,21 +132,14 @@ jobs:
           path: packages/tests-e2e/playwright-report/
           retention-days: 30
 
-      - name: Send logs to AWS CloudWatch
+      - name: Send results to CloudWatch
         if: always()
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ vars.CLOUDWATCH_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDWATCH_AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: ${{ secrets.AWS_REGION }}
         run: |
-          cd packages/tests-e2e
-          TESTS=$(xmllint --xpath 'string(/testsuites/@tests)' test-results/results.xml)
-          FAILURES=$(xmllint --xpath 'string(/testsuites/@failures)' test-results/results.xml)
-          SKIPPED=$(xmllint --xpath 'string(/testsuites/@skipped)' test-results/results.xml)
-          ERRORS=$(xmllint --xpath 'string(/testsuites/@errors)' test-results/results.xml)
-          TIME=$(xmllint --xpath 'string(/testsuites/@time)' test-results/results.xml)
-          PASSED=$((TESTS - FAILURES - ERRORS - SKIPPED))
-          FAILED=$((FAILURES + ERRORS))
-          TIMESTAMP=$(date +%s000)
-          LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"{\\\"application\\\":\\\"${{ matrix.testType }}\\\",\\\"platform\\\":\\\"react\\\",\\\"run_type\\\":\\\"commitly\\\",\\\"execution_time\\\":$TIME,\\\"passed\\\":$PASSED,\\\"failed\\\":$FAILED,\\\"link\\\":\\\"https://github.com/corbado/javascript/actions/runs/${GITHUB_RUN_ID}\\\"}\"}]"
-          aws logs put-log-events --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME" --log-events "$LOG_EVENT_JSON"
+          bash scripts/cloudwatch-log.sh \
+            "${{ matrix.testType }}" \
+            "react" \
+            "commitly"

.github/workflows/test.yml

@@ -30,4 +30,4 @@ packages/tests-e2e/.env.connect.local
 
 # scripts
 scripts/*.sh
-.env*.local
+!scripts/cloudwatch-log.sh

.gitignore

@@ -1,5 +1,7 @@
+
 DEVELOPERPANEL_API_URL=https://console.cloud.corbado-staging.io
-CORBADO_BACKEND_API_URL=https://backendapi.cloud.corbado-staging.io
-CORBADO_BACKEND_API_BASIC_AUTH=
 FRONTEND_API_URL_SUFFIX=frontendapi.cloud.corbado-staging.io
-PLAYWRIGHT_COMPLETE_OBSERVE_PROJECT_ID=
+
+NEXT_PUBLIC_DEFAULT_CORBADO_PROJECT_ID=pro-4505536417832482032
+NEXT_PUBLIC_CORBADO_FRONTEND_API_URL_SUFFIX=frontendapi.cloud.corbado-staging.io
+CORBADO_BACKEND_API_URL=https://backendapi.cloud.corbado-staging.io

packages/tests-e2e/.env.complete.ci

@@ -1,3 +1,5 @@
+DEVELOPERPANEL_API_URL=https://console.cloud.corbado-staging.io
+
+NEXT_PUBLIC_CORBADO_PROJECT_ID=pro-4641896612641727554
+NEXT_PUBLIC_CORBADO_FRONTEND_API_URL_SUFFIX=frontendapi.cloud.corbado-staging.io
 CORBADO_BACKEND_API_URL=https://backendapi.cloud.corbado-staging.io
-#DEVELOPERPANEL_API_URL=https://console.cloud.corbado-staging.io
-#FRONTEND_API_URL_SUFFIX=frontendapi.cloud.corbado-staging.io

packages/tests-e2e/.env.complete.example

@@ -1,63 +1,74 @@
-# `tests-e2e`
+# tests-e2e
 
-This package currently contains all the end-to-end tests for testing the React package. We plan to extend the tests for other packages of Corbado JavaScript Library.
+End-to-end tests for the Corbado JavaScript SDK, using Playwright.
 
-## Running Tests Locally
+## Overview
 
-Playground is locally run within Playwright for both Complete and Connect tests. For reference look at `utils/playground.ts`.
+### Playground applications
 
-This means that tests can simply be run with a single command. The command depends on whether you want to run it headless or with UI.
+Each SDK has a corresponding **playground application** in `playground/`:
 
-### Headless
+| Playground                 | SDK                            | Framework      |
+| -------------------------- | ------------------------------ | -------------- |
+| `playground/react`         | `@corbado/react`               | Vite + React   |
+| `playground/web-js`        | `@corbado/web-js`              | Vite (vanilla) |
+| `playground/web-js-script` | `@corbado/web-js` (script tag) | Vite (vanilla) |
+| `playground/connect-next`  | `@corbado/connect-react`       | Next.js        |
 
-```
-cd packages/tests-e2e
-npm run e2e:complete
-npm run e2e:connect
-```
+Playgrounds serve two purposes:
 
-### With UI
+1. **Manual testing** — each playground can be started locally (`npm run dev`) and is deployed automatically to Vercel on every push. This gives a quick way to verify SDK behaviour in a real browser without writing tests.
+2. **Automated testing** — Playwright tests run _on top of_ these playgrounds. The `globalSetup` in each test suite installs dependencies, builds the playground, and spawns it on a random port before tests start. No manual playground setup is needed.
 
-```
+### Test suites
+
+Because multiple playgrounds can expose the same SDK surface, a single Playwright suite can be **reused across different playground apps**. For example, the three Complete playgrounds (`react`, `web-js`, `web-js-script`) all render the same auth UI component, so they share the same test suite — only the playground that gets spawned changes.
+
+There are two test suites:
+
+- **Complete** — tests the full Corbado auth UI component (signup, login, passkey list, social login, observe). By default runs against the React playground; in the nightly workflow it also runs against `web-js` and `web-js-script`.
+- **Connect** — tests the Corbado Connect passkey integration (login, append, network blocking). Runs against the Next.js playground that talks to AWS Cognito.
+
+Each suite has its own Playwright config (`playwright.config.complete.ts` / `playwright.config.connect.ts`) and env files.
+
+## Running locally
+
+### 1. Set up env files
+
+Copy the example files and fill in the secrets:
+
+```bash
 cd packages/tests-e2e
-npm run e2e:complete:ui
-npm run e2e:connect:ui
+cp .env.complete.example .env.complete.local
+cp .env.connect.example .env.connect.local
 ```
 
-## Generating JWT Token
+Edit `.env.complete.local` and `.env.connect.local` with the required values. See the [Environment variables](#environment-variables) section below.
 
-The following is the script used to generate the JWT token.
+### 2. Run tests
 
 ```bash
-#!/bin/bash
-
-PRIVATE_KEY_PATH="private.pem"
-KID="pki-..."
-ISS="https://auth.corbado-dev.com"
-USER_ID="usr-..."
-USER_NAME="systemtest"
-USER_EMAIL="anders.choi+systemtest@corbado.com"
-
-NOW=$( date +%s )
-IAT=${NOW}
-EXP=$((${NOW} +  3000000)) # after around 34 days
-NBF=$((${NOW} -  10)) # 10 seconds ago
-HEADER_RAW='{"alg":"RS256","kid":"'"${KID}"'","typ":"JWT"}'
-HEADER=$( echo  -n "${HEADER_RAW}" |  openssl base64 |  tr  -d '=' |  tr '/+' '_-' |  tr  -d '\n' )
-PAYLOAD_RAW='{"iss":"'"${ISS}"'","sub":"'"${USER_ID}"'","exp":'"${EXP}"',"nbf":'"${NBF}"',"iat":'"${IAT}"',"name":"'"${USER_NAME}"'","orig":"'"${USER_EMAIL}"'","email":"'"${USER_EMAIL}"'","version":2}'
-PAYLOAD=$( echo  -n "${PAYLOAD_RAW}" |  openssl base64 |  tr  -d '=' |  tr '/+' '_-' |  tr  -d '\n' )
-HEADER_PAYLOAD="${HEADER}.${PAYLOAD}"
-SIGNATURE=$( openssl dgst -sha256  -sign ${PRIVATE_KEY_PATH} <(echo  -n "${HEADER_PAYLOAD}") |  openssl base64 |  tr  -d '=' |  tr '/+' '_-' |  tr  -d '\n' )
-JWT="${HEADER_PAYLOAD}.${SIGNATURE}"
-
-echo  $JWT
+cd packages/tests-e2e
+
+# Headless
+npm run e2e:complete
+npm run e2e:connect
+
+# With Playwright UI
+npm run e2e:complete:ui
+npm run e2e:connect:ui
 ```
 
-The first segment of the script contains all the information that must be extracted from the backend deployment.
+## Running in CI
+
+### Workflows
 
-As evident in the script, the token is valid for around 34 days. The token is intended to be renewed monthly using this script. For rewnewal, it is not necessary to extract all the information in the first segment.
+| Workflow       | Trigger                            | What it does                                                                                              |
+| -------------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------- |
+| `test.yml`     | PR + push to develop               | Runs Complete and Connect tests for **react** platform only (matrix)                                      |
+| `test-all.yml` | Nightly schedule (3x/day) + manual | Runs Complete tests for **react, web-js, web-js-script** + Connect tests for **react** (matrix, parallel) |
 
-## Authoring rules (connect2)
+## Authoring rules (connect)
 
 - Knowledge about page structure lives in `/models` only. Scenarios call model methods, not raw selectors.
 - All passkey authenticator interactions flow through `VirtualAuthenticator`.

packages/tests-e2e/.env.connect.ci

@@ -66,6 +66,7 @@ export default defineConfig({
     // ],
     ['html'],
     ['junit', { outputFile: 'test-results/results.xml' }],
+    ['./src/complete/utils/log-reporter.ts'],
   ],
   timeout: totalTimeout, // default: 30000ms
   expect: {
@@ -74,7 +75,6 @@ export default defineConfig({
   use: {
     actionTimeout: operationTimeout, // default: none
     navigationTimeout: operationTimeout, // default: none
-    // baseURL: process.env.PLAYWRIGHT_TEST_URL,
     screenshot: 'only-on-failure',
     video: 'retain-on-failure',
     trace: 'retain-on-failure',
@@ -92,10 +92,14 @@ export default defineConfig({
       name: 'passkey-list-general',
       testMatch: ['scenarios/passkey-list-general/*.ts'],
     },
-    //{
-    //  name: 'observe',
-    //  testMatch: ['scenarios/observe/*.ts'],
-    //},
+    ...(!process.env.CI
+      ? [
+          {
+            name: 'observe',
+            testMatch: ['scenarios/observe/*.ts'],
+          },
+        ]
+      : []),
   ],
   globalSetup: 'src/complete/utils/playground.ts',
 });

packages/tests-e2e/.env.connect.example

@@ -37,7 +37,7 @@ export class CorbadoAuthModel {
   async load(projectId: string, port: number, passkeySupport?: boolean, hashCode?: string) {
     this.projectId = projectId;
 
-    let url = `${process.env.PLAYWRIGHT_TEST_URL}:${port.toString()}/${this.projectId}/auth`;
+    let url = `http://localhost:${port.toString()}/${this.projectId}/auth`;
     if (hashCode) {
       url += `#${hashCode}`;
     }