test-all.yml

name: Test-all
run-name: All tests with extension build ${{ inputs.CLIENTS_BRANCH }} by @${{ github.actor }}
on:
  push:
    branches:
      - "main"
  pull_request:
  workflow_dispatch:
    inputs:
      CLIENTS_BRANCH:
        default: "main"
        description: "clients branch of browser build to use"
        required: true
        type: string
      REMOTE_VAULT_CONFIG_MATCH:
        description: "URL to an endpoint returning a vault configuration (will default to US prod configuration)"
        required: false
        type: string
      origin_issue:
        description: "PR number in bitwarden/clients to post feedback to"
        required: false
        type: string
jobs:
  build-and-test:
    name: Build and test
    runs-on: ubuntu-24.04
    permissions:
      id-token: write
      contents: read
      packages: read
      pull-requests: write
    outputs:
      send_pr_feedback: ${{ steps.set-send-pr-feedback.outputs.send_pr_feedback }}
    steps:
      - name: Send PR feedback check
        id: set-send-pr-feedback
        env:
          _SEND_PR_FEEDBACK: ${{ (inputs.origin_issue || false) && (vars.ENABLE_PR_FEEDBACK || false) }}
        run: |
          echo "send_pr_feedback=$_SEND_PR_FEEDBACK" >> "$GITHUB_OUTPUT"

      - name: Log in to Azure
        uses: bitwarden/gh-actions/azure-login@main
        if: steps.set-send-pr-feedback.outputs.send_pr_feedback == 'true'
        with:
          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
          client_id: ${{ secrets.AZURE_CLIENT_ID }}

      - name: Get Azure Key Vault secrets
        id: get-kv-secrets
        uses: bitwarden/gh-actions/get-keyvault-secrets@main
        if: steps.set-send-pr-feedback.outputs.send_pr_feedback == 'true'
        with:
          keyvault: gh-org-bitwarden
          secrets: "BW-GHAPP-ID,BW-GHAPP-KEY"

      - name: Log out from Azure
        uses: bitwarden/gh-actions/azure-logout@main
        if: steps.set-send-pr-feedback.outputs.send_pr_feedback == 'true'

      - name: Generate GH App token
        id: app-token
        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
        if: steps.set-send-pr-feedback.outputs.send_pr_feedback == 'true'
        with:
          app-id: ${{ steps.get-kv-secrets.outputs.BW-GHAPP-ID }}
          private-key: ${{ steps.get-kv-secrets.outputs.BW-GHAPP-KEY }}
          owner: bitwarden
          repositories: clients

      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Setup Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          cache: "npm"
          cache-dependency-path: "**/package-lock.json"
          node-version: "24"

      - name: Create dotenv file
        uses: ./.github/actions/setup-env-file
        with:
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          BW_DB_PORT: ${{ vars.BW_DB_PORT }}
          BW_DB_PROVIDER: ${{ vars.BW_DB_PROVIDER }}
          BW_DB_SERVER: ${{ vars.BW_DB_SERVER }}
          BW_DOMAIN: ${{ vars.BW_DOMAIN }}
          BW_ENABLE_SSL: ${{ vars.BW_ENABLE_SSL }}
          REMOTE_VAULT_CONFIG_MATCH: ${{ inputs.REMOTE_VAULT_CONFIG_MATCH || vars.BW_REMOTE_VAULT_CONFIG_MATCH }}
          BW_SSL_CERT: ${{ vars.BW_SSL_CERT }}
          BW_SSL_KEY: ${{ vars.BW_SSL_KEY }}
          CI: true
          CLI_SERVE_HOST: ${{ vars.CLI_SERVE_HOST }}
          CLI_SERVE_PORT: ${{ vars.CLI_SERVE_PORT }}
          EXTENSION_BUILD_PATH: ${{ vars.EXTENSION_BUILD_PATH }}
          PAGES_HOST_INSECURE_PORT: ${{ vars.PAGES_HOST_INSECURE_PORT }}
          PAGES_HOST_PORT: ${{ vars.PAGES_HOST_PORT }}
          PAGES_HOST: ${{ vars.PAGES_HOST }}
          VAULT_HOST_INSECURE_PORT: ${{ vars.VAULT_HOST_INSECURE_PORT }}
          VAULT_HOST_PORT: ${{ vars.VAULT_HOST_PORT }}
          VAULT_HOST_URL: ${{ vars.VAULT_HOST_URL }}

      - name: Download extension artifact
        id: get-build-artifact
        uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          workflow: build-browser.yml
          workflow_conclusion: ""
          branch: ${{ inputs.CLIENTS_BRANCH || 'main' }}
          name: ^dist-chrome-MV3-\w{7}\.zip$
          name_is_regexp: true
          repo: bitwarden/clients
          if_no_artifact_found: fail
          skip_unpack: true

      - name: Set no PR feedback when missing build artifact
        if: ${{ failure() && steps.get-build-artifact.conclusion != 'success' }}
        run: |
          echo "send_pr_feedback=false" > "$GITHUB_OUTPUT"

      - name: Unzip extension artifact
        run: |
          unzip -o "*dist-chrome-*.zip.zip"
          unzip -o "dist-chrome*.zip" -d build

      - name: Generate and install certs
        run: |
          npm run setup:ssl
          sudo apt-get install -y libnss3-tools
          . .env
          mkdir -p "$HOME/.pki/nssdb"
          certutil -d "$HOME/.pki/nssdb" -N --empty-password
          certutil -d "sql:$HOME/.pki/nssdb" -A -t "CP,CP," -n TestAutomationSSL -i "./$BW_SSL_CERT"

      - name: Install Bitwarden CLI
        run: npm install -g @bitwarden/cli@2026.6.0

      # FIXME Temporary workaround for PM-39403: server >= 2026.6.1 returns responses
      # that node-fetch v2 treats as a premature close (ERR_STREAM_PREMATURE_CLOSE),
      # even though the response body is fully delivered. Neutralize the two
      # `new Error('Premature close')` emission sites in the CLI's bundled
      # node-fetch so seed:vault:ciphers can proceed. Remove this step once
      # PM-39403 ships and the lite image is rebuilt with the fix.
      - name: Patch @bitwarden/cli node-fetch (PM-39403 workaround)
        run: |
          NF=$(npm root -g)/@bitwarden/cli/node_modules/node-fetch/lib/index.js
          BEFORE=$(grep -c "Premature close" "$NF" || echo 0)
          # Each match is followed by `err.code = ...` and an emit/callback
          # line; delete the 3-line body, leaving the surrounding `if` empty.
          sed -i "/new Error('Premature close')/,+2d" "$NF"
          AFTER=$(grep -c "Premature close" "$NF" || echo 0)
          echo "node-fetch patch: removed $((BEFORE - AFTER)) of $BEFORE match(es)"
          if [ "$AFTER" != "0" ]; then
            echo "WARNING: not all Premature-close references removed; @bitwarden/cli's node-fetch may have changed shape"
          fi

      - name: Install project dependencies
        run: |
          npm ci
          npx playwright install --with-deps chromium

      - name: Match feature flags file to remote config
        run: |
          touch flags.json
          npm run setup:flags

      - name: Build and start the test vault
        run: docker compose up -d --build --remove-orphans --wait --wait-timeout 60

      - name: Wait for admin DB migrations to complete
        run: |
          echo "Waiting for admin DB migrations..."
          timeout 120 sh -c '
            until docker compose exec -T bitwarden sh -c "grep -q \"Migration successful\" /etc/bitwarden/logs/admin-*.log 2>/dev/null"; do
              sleep 2
            done
          '
          echo "Migrations complete."

      - name: Generate crypto values
        run: npm run setup:crypto

      - name: Setup the vault test account
        run: npm run seed:vault:account

      - name: Setup the vault test account ciphers
        run: npm run seed:vault:ciphers

      - name: Download test site build
        uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          workflow: build.yml
          workflow_conclusion: ""
          branch: main
          name: build-files
          path: test-site
          repo: bitwarden/test-the-web
          if_no_artifact_found: fail

      - name: Copy over certs and install test site dependencies
        run: |
          cp ssl.crt test-site/api/
          cp ssl.key test-site/api/
          cd test-site
          npm ci

      - name: Run all tests
        run: npm run test:static:ci

      - name: Update job summary
        if: always()
        run: echo "$(<./test-summary/test-summary.md)" >> "$GITHUB_STEP_SUMMARY"

      - name: Upload results as artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: always()
        with:
          name: test-summary
          path: |
            ./flags.json
            ./test-summary
            ./tests-out/videos
            ./tests-out/screenshots

      - name: Remove past BIT status comments on originating issue
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        if: (success() || (failure() && steps.get-build-artifact.conclusion == 'success')) && steps.set-send-pr-feedback.outputs.send_pr_feedback == 'true'
        with:
          github-token: ${{ steps.app-token.outputs.token }}
          script: |
            // Note: should match the first line of `message` in the communication steps
            const workflowCommentTag = '<!-- comment_tag: test-all -->';

            const owner = 'bitwarden';

            const issueComments = await github.rest.issues.listComments({
              issue_number: Number(context.payload.inputs?.origin_issue),
              owner: owner,
              repo: 'clients',
            });

            for (const comment of issueComments.data || []) {
              const shouldDeleteComment =
                // Do not delete comments that were not automated
                !!comment.performed_via_github_app &&

                // Do not delete user comments
                comment.user.type === 'Bot' &&

                // Do not delete edited comments
                comment.created_at === comment.updated_at &&

                // Only delete comments from this workflow
                comment.body.trim().startsWith(workflowCommentTag);

              if (shouldDeleteComment) {
                await github.rest.issues.deleteComment({
                  comment_id: comment.id,
                  owner: owner,
                  repo: 'clients',
                });
              }
            }

      - name: Communicate BIT failure on originating issue
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        if: failure() && steps.get-build-artifact.conclusion == 'success' && steps.set-send-pr-feedback.outputs.send_pr_feedback == 'true'
        env:
          REMOTE_VAULT_CONFIG: ${{ vars.BW_REMOTE_VAULT_CONFIG_MATCH }}
        with:
          github-token: ${{ steps.app-token.outputs.token }}
          script: |
            const owner = 'bitwarden';
            const runURL = `https://github.com/${owner}/browser-interactions-testing/actions/runs/${context.runId}`;
            const configURL = new URL(process.env.REMOTE_VAULT_CONFIG);
            const message = `
            <!-- comment_tag: test-all -->
            #### ⚠️ Changes in this PR impact the Autofill experience of the browser client ⚠️

            [BIT](https://github.com/${owner}/browser-interactions-testing) has tested the core experience with these changes and **the feature flag configuration used by \`${configURL.hostname}\`**

            > [!CAUTION]
            > Unfortunately, one or more of these tests failed. 😞
            >
            > Please resolve the failure before merging; reach out to \`@bitwarden/team-autofill-dev\` if you'd like help.

            You can view the detailed results of the tests [here](${runURL}).
            `;

            github.rest.issues.createComment({
              issue_number: Number(context.payload.inputs?.origin_issue),
              owner: owner,
              repo: 'clients',
              body: message
            });

      - name: Communicate BIT success on originating issue
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        if: success() && steps.set-send-pr-feedback.outputs.send_pr_feedback == 'true'
        env:
          REMOTE_VAULT_CONFIG: ${{ vars.BW_REMOTE_VAULT_CONFIG_MATCH }}
        with:
          github-token: ${{ steps.app-token.outputs.token }}
          script: |
            const owner = 'bitwarden';
            const runURL = `https://github.com/${owner}/browser-interactions-testing/actions/runs/${context.runId}`;
            const configURL = new URL(process.env.REMOTE_VAULT_CONFIG);
            const message = `
            <!-- comment_tag: test-all -->
            #### Changes in this PR impact the Autofill experience of the browser client

            [BIT](https://github.com/${owner}/browser-interactions-testing) has tested the core experience with these changes and <ins>the feature flag configuration used by \`${configURL.hostname}\`</ins>.

            ✅ Fortunately, [these BIT tests have passed](${runURL})! 🎉
            `;

            github.rest.issues.createComment({
              issue_number: Number(context.payload.inputs?.origin_issue),
              owner: owner,
              repo: 'clients',
              body: message
            });