boot-tests-nightly.yml

name: Boot tests

on:
  schedule:
    - cron: '5 0 * * 1-5' # working days at midnight
  workflow_dispatch:
    inputs:
      pr_number:
        description: 'Pull Request number to run tests against'
        required: false
        type: number
      slack_channel_id:
        description: 'Slack channel ID for notifications (enables notification jobs on manual runs)'
        required: false
        type: string
      run_id:
        description: 'Run ID to re-analyze (skips boot tests, downloads existing artifact)'
        required: false
        type: number
      rerun_of:
        description: 'Original run ID (set automatically when retrying flaky failures)'
        required: false
        type: number
  pull_request:
    types: [opened, reopened, synchronize]
    paths:
      - 'playwright/BootTests/**' # Only run on PRs that change a file in BootTests directory
  merge_group:

jobs:
  boot-tests:
    if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.run_id != '') }}
    runs-on:
      - codebuild-image-builder-frontend-${{ github.run_id }}-${{ github.run_attempt }}
      - instance-size:large
      - buildspec-override:true
    permissions:
      pull-requests: read

    steps:
      - name: Get PR details if triggered by workflow_dispatch with pr_number
        id: get-pr-details
        if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_number != ''
        run: |
          PR_INFO=$(gh pr view "$PR_NUMBER" --json headRefOid,url)
          echo "pr_head_sha=$(echo "$PR_INFO" | jq -r '.headRefOid')" >> $GITHUB_ENV
          echo "pr_url=$(echo "$PR_INFO" | jq -r '.url')" >> $GITHUB_ENV
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ github.event.inputs.pr_number }}

      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ env.pr_head_sha || github.ref }}

      - name: Setup Node.js with cached dependencies
        uses: ./.github/actions/setup-node-cached

      - name: Install Playwright
        run: npx playwright install chromium --only-shell

      # This prevents an error related to minimum watchers when running the front-end and playwright
      - name: Increase file watchers limit
        run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p

      - name: Update /etc/hosts
        run: sudo npm run patch:hosts

      - name: Start front-end server
        run: |
          npm run start:federated &
          npx wait-on http://localhost:8003/apps/image-builder/

      - name: Run testing proxy
        run: docker run -d --network=host -e HTTPS_PROXY=$RH_PROXY_URL -v "$(pwd)/config:/config:ro,Z" --name frontend-development-proxy quay.io/redhat-user-workloads/hcc-platex-services-tenant/frontend-development-proxy

      - name: Run front-end Playwright tests
        env:
          BASE_URL: https://stage.foo.redhat.com:1337
        run: |
          CURRENTS_PROJECT_ID=CNrla0 CURRENTS_RECORD_KEY=$CURRENTS_RECORD_KEY npx playwright test playwright/BootTests

      - name: Store front-end Test report
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: playwright-report
          path: playwright-report/
          retention-days: 10

      - name: Store test results JSON
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-results-json
          path: test-results.json
          retention-days: 10

  # Compute which Slack channels to notify based on trigger type.
  # Scheduled runs notify both channels; manual runs with slack_channel_id notify only that channel.
  prepare-channels:
    runs-on: ubuntu-latest
    if: >-
      github.event_name == 'schedule' ||
      (github.event_name == 'workflow_dispatch' && github.event.inputs.slack_channel_id != '') ||
      (github.event_name == 'workflow_dispatch' && github.event.inputs.rerun_of != '')
    permissions: {}
    outputs:
      channels: ${{ steps.set.outputs.channels }}
    steps:
      - name: Build channel list
        id: set
        run: |
          if [ -n "$MANUAL" ]; then
            echo 'channels=[{"id":"'"$MANUAL"'","name":"manual"}]' >> "$GITHUB_OUTPUT"
          else
            echo 'channels=[{"id":"'"$MAIN"'","name":"main"},{"id":"'"$FRONTEND"'","name":"frontend"}]' >> "$GITHUB_OUTPUT"
          fi
        env:
          MANUAL: ${{ github.event.inputs.slack_channel_id }}
          MAIN: ${{ vars.SLACK_CHANNEL_ID }}
          FRONTEND: ${{ vars.SLACK_FRONTEND_CHANNEL_ID }}

  # Post success/failure notifications to Slack.
  # Isolated from Claude analysis — secrets here are NOT accessible to the analyze-failures job.
  # Uses a matrix to post to all target channels without duplicating steps.
  notify-slack:
    runs-on: ubuntu-latest
    needs: [boot-tests, prepare-channels]
    if: >-
      always() &&
      needs.prepare-channels.result == 'success'
    permissions:
      actions: read
    strategy:
      fail-fast: false
      matrix:
        channel: ${{ fromJSON(needs.prepare-channels.outputs.channels) }}

    steps:
      - name: Notify Slack on success
        if: ${{ needs.boot-tests.result == 'success' && github.event.inputs.rerun_of == '' }}
        uses: slackapi/slack-github-action@v2.1.1
        with:
          method: chat.postMessage
          token: ${{ secrets.SLACK_BOT_TOKEN }}
          payload: |
            channel: ${{ matrix.channel.id }}
            attachments:
              - color: "#2EB67D"
                blocks:
                  - type: section
                    text:
                      type: mrkdwn
                      text: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/ | :stars: Nightly boot test run>: *SUCCESS* :partymeow:"

      - name: Notify Slack on failure
        id: slack-failure
        if: ${{ (needs.boot-tests.result == 'failure' || github.event.inputs.run_id != '') && github.event.inputs.rerun_of == '' }}
        uses: slackapi/slack-github-action@v2.1.1
        with:
          method: chat.postMessage
          token: ${{ secrets.SLACK_BOT_TOKEN }}
          payload: |
            channel: ${{ matrix.channel.id }}
            attachments:
              - color: "#E01E5A"
                blocks:
                  - type: section
                    text:
                      type: mrkdwn
                      text: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/ | :stars: Nightly boot test run>: *FAILED* :big-sad:"

      - name: Save Slack message ts
        if: ${{ (needs.boot-tests.result == 'failure' || github.event.inputs.run_id != '') && github.event.inputs.rerun_of == '' }}
        run: echo "${{ steps.slack-failure.outputs.ts }}" > slack-ts.txt

      - name: Upload Slack ts artifact
        if: ${{ (needs.boot-tests.result == 'failure' || github.event.inputs.run_id != '') && github.event.inputs.rerun_of == '' }}
        uses: actions/upload-artifact@v4
        with:
          name: slack-ts-${{ matrix.channel.name }}
          path: slack-ts.txt
          overwrite: true

      # On reruns, retrieve the original message ts from the original run
      - name: Download Slack ts artifact
        if: ${{ github.event.inputs.rerun_of != '' }}
        uses: actions/download-artifact@v4
        with:
          name: slack-ts-${{ matrix.channel.name }}
          run-id: ${{ github.event.inputs.rerun_of }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
        continue-on-error: true

      - name: Read Slack message ts
        id: slack-ts
        run: |
          if [ -f slack-ts.txt ]; then
            echo "ts=$(cat slack-ts.txt)" >> "$GITHUB_OUTPUT"
          fi

      - name: Update original message on retry success
        if: ${{ needs.boot-tests.result == 'success' && github.event.inputs.rerun_of != '' && steps.slack-ts.outputs.ts }}
        uses: slackapi/slack-github-action@v2.1.1
        with:
          method: chat.update
          token: ${{ secrets.SLACK_BOT_TOKEN }}
          payload: |
            channel: ${{ matrix.channel.id }}
            ts: ${{ steps.slack-ts.outputs.ts }}
            attachments:
              - color: "#2EB67D"
                blocks:
                  - type: section
                    text:
                      type: mrkdwn
                      text: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/ | :stars: Nightly boot test run>: ~*FAILED*~ *PASSED on retry* :partymeow:"

  # Analyze test failures with Claude Code (via Vertex AI) — only on first run, not reruns
  # SECURITY: This job only has Vertex AI secrets — no Slack tokens, no other credentials.
  # Claude runs here and cannot access secrets from other jobs.
  analyze-failures:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    needs: boot-tests
    if: >-
      always() &&
      (needs.boot-tests.result == 'failure' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_id != '')) &&
      github.event.inputs.rerun_of == '' &&
      (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && (github.event.inputs.slack_channel_id != '' || github.event.inputs.run_id != '')))
    permissions:
      contents: read
      actions: read

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Download test results JSON
        uses: actions/download-artifact@v4
        with:
          name: test-results-json
          path: .
          run-id: ${{ github.event.inputs.run_id || github.run_id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
        continue-on-error: true # Fallback for runs before JSON reporter was added

      - name: Download playwright report (fallback)
        if: ${{ hashFiles('test-results.json') == '' }}
        uses: actions/download-artifact@v4
        with:
          name: playwright-report
          path: playwright-report/
          run-id: ${{ github.event.inputs.run_id || github.run_id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
        continue-on-error: true

      - name: Download boot-tests job log
        run: |
          JOB_ID=$(gh api "repos/${{ github.repository }}/actions/runs/${RUN_ID}/jobs" \
            --jq '.jobs[] | select(.name == "boot-tests") | .id')
          if [ -n "$JOB_ID" ]; then
            gh api "repos/${{ github.repository }}/actions/jobs/${JOB_ID}/logs" > ci-logs.txt
          else
            echo "Could not find boot-tests job" > ci-logs.txt
          fi
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          RUN_ID: ${{ github.event.inputs.run_id || github.run_id }}
        continue-on-error: true

      - name: Install Claude Code
        run: npm install -g @anthropic-ai/claude-code

      - name: Set up GCP credentials
        run: |
          echo '${{ secrets.ANTHROPIC_VERTEX_CREDENTIALS }}' > /tmp/gcp-credentials.json

      - name: Analyze test failures
        run: |
          mkdir -p boot-test-reports
          cat boot-test-instructions.md > /tmp/prompt.md
          echo "" >> /tmp/prompt.md
          echo "The GitHub Actions run URL is: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/" >> /tmp/prompt.md
          echo "This is attempt ${{ github.run_attempt }}." >> /tmp/prompt.md
          cat /tmp/prompt.md | claude --print --verbose \
            --output-format stream-json \
            --model "${MODEL:-claude-sonnet-4-6}" \
            --allowedTools "Read,Glob,Grep,Write" \
            --disallowedTools "Bash,Agent,WebFetch,WebSearch,Edit,NotebookEdit"
        env:
          CLAUDE_CODE_USE_VERTEX: "1"
          CLOUD_ML_REGION: us-east5
          ANTHROPIC_VERTEX_PROJECT_ID: ${{ secrets.ANTHROPIC_VERTEX_PROJECT_ID }}
          GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp-credentials.json

      - name: Upload analysis reports
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: reports
          path: boot-test-reports/
          retention-days: 10

  # Post Claude's analysis to Slack and trigger rerun for flaky tests (first run),
  # or post a short "retry also failed" thread reply (rerun).
  # Uses the same channel matrix as notify-slack to avoid step duplication.
  # NOTE: When dispatched with run_id only (no slack_channel_id), this job is
  # intentionally skipped — analyze-failures still runs and uploads reports as
  # artifacts, but nothing posts them. Use this mode to re-analyze a past run
  # and download the results manually.
  post-analysis:
    runs-on: ubuntu-latest
    needs: [boot-tests, analyze-failures, notify-slack, prepare-channels]
    if: >-
      always() &&
      needs.prepare-channels.result == 'success' &&
      (needs.boot-tests.result == 'failure' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_id != '')) &&
      (needs.analyze-failures.result == 'success' || needs.analyze-failures.result == 'skipped' || needs.analyze-failures.result == 'failure')
    permissions:
      actions: write
    strategy:
      fail-fast: false
      matrix:
        channel: ${{ fromJSON(needs.prepare-channels.outputs.channels) }}

    steps:
      # --- First run: post Claude's analysis and trigger rerun if flake ---
      - name: Download analysis reports
        if: ${{ github.event.inputs.rerun_of == '' && needs.analyze-failures.result == 'success' }}
        uses: actions/download-artifact@v4
        with:
          name: reports
          path: boot-test-reports/

      - name: Download Slack ts artifact
        uses: actions/download-artifact@v4
        with:
          name: slack-ts-${{ matrix.channel.name }}
          run-id: ${{ github.event.inputs.rerun_of || github.run_id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
        continue-on-error: true

      - name: Read Slack message ts
        id: slack-ts
        run: |
          if [ -f slack-ts.txt ]; then
            echo "ts=$(cat slack-ts.txt)" >> "$GITHUB_OUTPUT"
          fi

      - name: Build analysis payload
        id: analysis-payload
        if: ${{ github.event.inputs.rerun_of == '' && needs.analyze-failures.result == 'success' && steps.slack-ts.outputs.ts }}
        run: |
          if [ ! -f boot-test-reports/latest.md ]; then
            echo "::warning::Claude did not produce boot-test-reports/latest.md"
            exit 0
          fi
          # Truncate to stay under Slack's 40k character limit
          if [ "$(wc -c < boot-test-reports/latest.md)" -gt 39000 ]; then
            head -c 39000 boot-test-reports/latest.md > /tmp/truncated.md
            printf '\n\n:scissors: Report truncated — full report available in workflow artifacts.' >> /tmp/truncated.md
            mv /tmp/truncated.md boot-test-reports/latest.md
          fi
          {
            echo "json<<PAYLOAD_EOF"
            jq -n \
              --arg channel "$SLACK_CHANNEL_ID" \
              --arg thread_ts "$THREAD_TS" \
              --rawfile text boot-test-reports/latest.md \
              '{channel: $channel, thread_ts: $thread_ts, text: $text}'
            echo "PAYLOAD_EOF"
          } >> "$GITHUB_OUTPUT"
        env:
          SLACK_CHANNEL_ID: ${{ matrix.channel.id }}
          THREAD_TS: ${{ steps.slack-ts.outputs.ts }}

      - name: Post analysis to Slack
        if: ${{ steps.analysis-payload.outputs.json }}
        uses: slackapi/slack-github-action@v2.1.1
        with:
          method: chat.postMessage
          token: ${{ secrets.SLACK_BOT_TOKEN }}
          payload: ${{ steps.analysis-payload.outputs.json }}

      - name: Read classification
        id: classification
        if: ${{ github.event.inputs.rerun_of == '' && needs.analyze-failures.result == 'success' }}
        run: |
          CLASSIFICATION=$(cat boot-test-reports/classification.txt 2>/dev/null || echo "UNKNOWN")
          CLASSIFICATION=$(echo "$CLASSIFICATION" | tr -d '[:space:]' | tr '[:lower:]' '[:upper:]')
          echo "value=$CLASSIFICATION" >> "$GITHUB_OUTPUT"
          if [ "$CLASSIFICATION" = "FLAKE" ] || [ "$CLASSIFICATION" = "INFRASTRUCTURE" ]; then
            echo "retriable=true" >> "$GITHUB_OUTPUT"
          fi

      - name: Notify analysis failure
        if: ${{ github.event.inputs.rerun_of == '' && needs.analyze-failures.result == 'failure' && steps.slack-ts.outputs.ts }}
        uses: slackapi/slack-github-action@v2.1.1
        with:
          method: chat.postMessage
          token: ${{ secrets.SLACK_BOT_TOKEN }}
          payload: |
            channel: ${{ matrix.channel.id }}
            thread_ts: ${{ steps.slack-ts.outputs.ts }}
            text: ":warning: Automated analysis failed — could not classify this failure. <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/|View run>"

      # Only trigger rerun from one matrix instance to avoid duplicates
      - name: Trigger rerun
        id: rerun
        if: ${{ steps.classification.outputs.retriable && github.event.inputs.run_id == '' && github.event.inputs.rerun_of == '' && (matrix.channel.name == 'main' || matrix.channel.name == 'manual') }}
        run: |
          echo "Classified as $CLASSIFICATION — triggering rerun"
          gh workflow run boot-tests-nightly.yml \
            -f rerun_of=${{ github.run_id }} \
            --repo ${{ github.repository }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          CLASSIFICATION: ${{ steps.classification.outputs.value }}

      - name: Update original message — rerun in progress
        if: ${{ steps.rerun.outcome == 'success' && steps.slack-ts.outputs.ts }}
        uses: slackapi/slack-github-action@v2.1.1
        with:
          method: chat.update
          token: ${{ secrets.SLACK_BOT_TOKEN }}
          payload: |
            channel: ${{ matrix.channel.id }}
            ts: ${{ steps.slack-ts.outputs.ts }}
            attachments:
              - color: "#1445f7"
                blocks:
                  - type: section
                    text:
                      type: mrkdwn
                      text: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/ | :stars: Nightly boot test run>: *FAILED* :big-sad: :arrows_counterclockwise:"

      # --- Rerun: post a short thread reply ---
      - name: Post retry failure to Slack
        if: ${{ github.event.inputs.rerun_of != '' && steps.slack-ts.outputs.ts }}
        uses: slackapi/slack-github-action@v2.1.1
        with:
          method: chat.postMessage
          token: ${{ secrets.SLACK_BOT_TOKEN }}
          payload: |
            channel: ${{ matrix.channel.id }}
            thread_ts: ${{ steps.slack-ts.outputs.ts }}
            text: ":x: Retry also failed. <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/|View run>"

      - name: Revert original message on retry failure
        if: ${{ github.event.inputs.rerun_of != '' && steps.slack-ts.outputs.ts }}
        uses: slackapi/slack-github-action@v2.1.1
        with:
          method: chat.update
          token: ${{ secrets.SLACK_BOT_TOKEN }}
          payload: |
            channel: ${{ matrix.channel.id }}
            ts: ${{ steps.slack-ts.outputs.ts }}
            attachments:
              - color: "#E01E5A"
                blocks:
                  - type: section
                    text:
                      type: mrkdwn
                      text: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/ | :stars: Nightly boot test run>: *FAILED* :big-sad:"