test-all.yml

name: Test on All Platforms
env:
  BRANCH_NAME_RAW: ${{ github.head_ref || github.ref_name }}

on:
  workflow_dispatch:
  schedule:
    - cron: '10 5 * * *'  #  6:10 AM CET / 7:10 AM CEST
    - cron: '50 11 * * *' # 12:50 PM CET / 1:50 PM CEST
    - cron: '30 18 * * *' #  7:30 PM CET / 8:30 PM CEST

jobs:
  test:
    timeout-minutes: 60
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Node.js
        uses: actions/setup-node@v4

      - name: Get installed Playwright version
        id: playwright-version
        run: echo "::set-output name=version::$(yarn why --json @playwright/test | grep -h 'workspace:.' | jq --raw-output '.children[].locator' | sed -e 's/@playwright\/test@.*://')"

      - name: Cache node modules
        id: cache-npm
        uses: actions/cache@v3
        env:
          cache-name: cache-node-modules
        with:
          path: |
            ~/.npm
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-build-${{ env.cache-name }}-
            ${{ runner.os }}-build-
            ${{ runner.os }}-

      - if: ${{ steps.cache-npm.outputs.cache-hit != 'true' }}
        name: List the state of node modules
        continue-on-error: true
        run: |
          cd packages/tests-e2e
          npm list

      - name: Install Dependencies
        run: |
          cd packages/tests-e2e
          npm install
          sudo apt-get update
          sudo apt-get install -y libxml2-utils

      - uses: actions/cache@v3
        id: playwright-cache
        with:
          path: '~/.cache/ms-playwright'
          key: '${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}'
          restore-keys: |
            ${{ runner.os }}-playwright-

      - name: Install Playwright's dependencies
        if: steps.playwright-cache.outputs.cache-hit != 'true'
        run: |
          cd packages/tests-e2e
          npx playwright install --with-deps

      - name: Create a log stream on AWS CloudWatch
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
        run: |
          TIMESTAMP=$(date +%s000)
          echo "LOG_STREAM_NAME=javascript-$TIMESTAMP" >> $GITHUB_ENV
          aws logs create-log-stream --log-group-name "test-results-board" --log-stream-name "javascript-$TIMESTAMP"

      - name: Run Complete tests for react
        run: |
          cd packages/tests-e2e
          BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-')
          export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.react.playground.corbado.io"
          set +e
          npx playwright test --config=playwright.config.complete.ts
          EXIT_CODE=$?
          if [ $EXIT_CODE -ne 0 ]; then
            echo "COMPLETE_REACT_FAILED=true" >> $GITHUB_ENV
          fi
        env:
          PLAYWRIGHT_NUM_CORES: 4
          PLAYWRIGHT_JWT_TOKEN: ${{ secrets.PLAYWRIGHT_JWT_TOKEN }}
          PLAYWRIGHT_GOOGLE_EMAIL: ${{ secrets.PLAYWRIGHT_GOOGLE_EMAIL }}
          PLAYWRIGHT_GOOGLE_PASSWORD: ${{ secrets.PLAYWRIGHT_GOOGLE_PASSWORD }}
          PLAYWRIGHT_GOOGLE_TOTP_SECRET: ${{ secrets.PLAYWRIGHT_GOOGLE_TOTP_SECRET }}
          GITHUB_RUN_ID: ${{ github.run_id }}
          SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }}
          GITHUB_BRANCH_NAME: ${{ env.BRANCH_NAME_RAW }}
        continue-on-error: true

      - uses: actions/upload-artifact@v4
        with:
          name: playwright-report-complete-react
          path: packages/tests-e2e/playwright-report/
          retention-days: 30

      - name: Send logs to AWS CloudWatch
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
        run: |
          cd packages/tests-e2e
          TESTS=$(xmllint --xpath 'string(/testsuites/@tests)' test-results/results.xml)
          FAILURES=$(xmllint --xpath 'string(/testsuites/@failures)' test-results/results.xml)
          SKIPPED=$(xmllint --xpath 'string(/testsuites/@skipped)' test-results/results.xml)
          ERRORS=$(xmllint --xpath 'string(/testsuites/@errors)' test-results/results.xml)
          TIME=$(xmllint --xpath 'string(/testsuites/@time)' test-results/results.xml)
          PASSED=$((TESTS - FAILURES - ERRORS - SKIPPED))
          FAILED=$((FAILURES + ERRORS))
          TIMESTAMP=$(date +%s000)
          LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"{\\\"application\\\":\\\"complete\\\",\\\"platform\\\":\\\"react\\\",\\\"run_type\\\":\\\"nightly\\\",\\\"execution_time\\\":$TIME,\\\"passed\\\":$PASSED,\\\"failed\\\":$FAILED,\\\"link\\\":\\\"https://github.com/corbado/javascript/actions/runs/${GITHUB_RUN_ID}\\\"}\"}]"
          aws logs put-log-events --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME" --log-events "$LOG_EVENT_JSON"

      - name: Run Complete tests for web-js
        run: |
          cd packages/tests-e2e
          BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-')
          export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.web-js.playground.corbado.io"
          set +e
          npx playwright test --config=playwright.config.complete.ts
          EXIT_CODE=$?
          if [ $EXIT_CODE -ne 0 ]; then
            echo "COMPLETE_WEBJS_FAILED=true" >> $GITHUB_ENV
          fi
        env:
          PLAYWRIGHT_NUM_CORES: 4
          PLAYWRIGHT_JWT_TOKEN: ${{ secrets.PLAYWRIGHT_JWT_TOKEN }}
          PLAYWRIGHT_GOOGLE_EMAIL: ${{ secrets.PLAYWRIGHT_GOOGLE_EMAIL }}
          PLAYWRIGHT_GOOGLE_PASSWORD: ${{ secrets.PLAYWRIGHT_GOOGLE_PASSWORD }}
          PLAYWRIGHT_GOOGLE_TOTP_SECRET: ${{ secrets.PLAYWRIGHT_GOOGLE_TOTP_SECRET }}
          GITHUB_RUN_ID: ${{ github.run_id }}
          SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }}
          GITHUB_BRANCH_NAME: ${{ env.BRANCH_NAME_RAW }}
        continue-on-error: true

      - uses: actions/upload-artifact@v4
        with:
          name: playwright-report-complete-web-js
          path: packages/tests-e2e/playwright-report/
          retention-days: 30

      - name: Send logs to AWS CloudWatch
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
        run: |
          cd packages/tests-e2e
          TESTS=$(xmllint --xpath 'string(/testsuites/@tests)' test-results/results.xml)
          FAILURES=$(xmllint --xpath 'string(/testsuites/@failures)' test-results/results.xml)
          SKIPPED=$(xmllint --xpath 'string(/testsuites/@skipped)' test-results/results.xml)
          ERRORS=$(xmllint --xpath 'string(/testsuites/@errors)' test-results/results.xml)
          TIME=$(xmllint --xpath 'string(/testsuites/@time)' test-results/results.xml)
          PASSED=$((TESTS - FAILURES - ERRORS - SKIPPED))
          FAILED=$((FAILURES + ERRORS))
          TIMESTAMP=$(date +%s000)
          LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"{\\\"application\\\":\\\"complete\\\",\\\"platform\\\":\\\"web-js\\\",\\\"run_type\\\":\\\"nightly\\\",\\\"execution_time\\\":$TIME,\\\"passed\\\":$PASSED,\\\"failed\\\":$FAILED,\\\"link\\\":\\\"https://github.com/corbado/javascript/actions/runs/${GITHUB_RUN_ID}\\\"}\"}]"
          aws logs put-log-events --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME" --log-events "$LOG_EVENT_JSON"

      - name: Run Complete tests for web-js-script
        run: |
          cd packages/tests-e2e
          BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-')
          export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.web-js-script.playground.corbado.io"
          set +e
          npx playwright test --config=playwright.config.complete.ts
          EXIT_CODE=$?
          if [ $EXIT_CODE -ne 0 ]; then
            echo "COMPLETE_WEBJSSCRIPT_FAILED=true" >> $GITHUB_ENV
          fi
        env:
          PLAYWRIGHT_NUM_CORES: 4
          PLAYWRIGHT_JWT_TOKEN: ${{ secrets.PLAYWRIGHT_JWT_TOKEN }}
          PLAYWRIGHT_GOOGLE_EMAIL: ${{ secrets.PLAYWRIGHT_GOOGLE_EMAIL }}
          PLAYWRIGHT_GOOGLE_PASSWORD: ${{ secrets.PLAYWRIGHT_GOOGLE_PASSWORD }}
          PLAYWRIGHT_GOOGLE_TOTP_SECRET: ${{ secrets.PLAYWRIGHT_GOOGLE_TOTP_SECRET }}
          GITHUB_RUN_ID: ${{ github.run_id }}
          SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }}
          GITHUB_BRANCH_NAME: ${{ env.BRANCH_NAME_RAW }}
        continue-on-error: true

      - uses: actions/upload-artifact@v4
        with:
          name: playwright-report-complete-web-js-script
          path: packages/tests-e2e/playwright-report/
          retention-days: 30

      - name: Send logs to AWS CloudWatch
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
        run: |
          cd packages/tests-e2e
          TESTS=$(xmllint --xpath 'string(/testsuites/@tests)' test-results/results.xml)
          FAILURES=$(xmllint --xpath 'string(/testsuites/@failures)' test-results/results.xml)
          SKIPPED=$(xmllint --xpath 'string(/testsuites/@skipped)' test-results/results.xml)
          ERRORS=$(xmllint --xpath 'string(/testsuites/@errors)' test-results/results.xml)
          TIME=$(xmllint --xpath 'string(/testsuites/@time)' test-results/results.xml)
          PASSED=$((TESTS - FAILURES - ERRORS - SKIPPED))
          FAILED=$((FAILURES + ERRORS))
          TIMESTAMP=$(date +%s000)
          LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"{\\\"application\\\":\\\"complete\\\",\\\"platform\\\":\\\"web-js-script\\\",\\\"run_type\\\":\\\"nightly\\\",\\\"execution_time\\\":$TIME,\\\"passed\\\":$PASSED,\\\"failed\\\":$FAILED,\\\"link\\\":\\\"https://github.com/corbado/javascript/actions/runs/${GITHUB_RUN_ID}\\\"}\"}]"
          aws logs put-log-events --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME" --log-events "$LOG_EVENT_JSON"

      - name: Run Connect tests for react
        run: |
          cd packages/tests-e2e
          BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-')
          export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.connect-next.playground.corbado.io"
          set +e
          npx playwright test --config=playwright.config.connect.ts
          EXIT_CODE=$?
          if [ $EXIT_CODE -ne 0 ]; then
            echo "CONNECT_REACT_FAILED=true" >> $GITHUB_ENV
          fi
        env:
          PLAYWRIGHT_NUM_CORES: 4
          BACKEND_API_BASIC_AUTH: ${{ secrets.BACKEND_API_BASIC_AUTH }}
          PLAYWRIGHT_CONNECT_PROJECT_ID: ${{ secrets.PLAYWRIGHT_CONNECT_PROJECT_ID }}
          PLAYWRIGHT_NGROK_AUTH_TOKEN: ${{ secrets.PLAYWRIGHT_NGROK_AUTH_TOKEN }}
          GITHUB_RUN_ID: ${{ github.run_id }}
          SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }}
          GITHUB_BRANCH_NAME: ${{ env.BRANCH_NAME_RAW }}
        continue-on-error: true

      - uses: actions/upload-artifact@v4
        with:
          name: playwright-report-connect-react
          path: packages/tests-e2e/playwright-report/
          retention-days: 30

      - name: Send logs to AWS CloudWatch
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
        run: |
          cd packages/tests-e2e
          TESTS=$(xmllint --xpath 'string(/testsuites/@tests)' test-results/results.xml)
          FAILURES=$(xmllint --xpath 'string(/testsuites/@failures)' test-results/results.xml)
          SKIPPED=$(xmllint --xpath 'string(/testsuites/@skipped)' test-results/results.xml)
          ERRORS=$(xmllint --xpath 'string(/testsuites/@errors)' test-results/results.xml)
          TIME=$(xmllint --xpath 'string(/testsuites/@time)' test-results/results.xml)
          PASSED=$((TESTS - FAILURES - ERRORS - SKIPPED))
          FAILED=$((FAILURES + ERRORS))
          TIMESTAMP=$(date +%s000)
          LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"{\\\"application\\\":\\\"connect\\\",\\\"platform\\\":\\\"react\\\",\\\"run_type\\\":\\\"nightly\\\",\\\"execution_time\\\":$TIME,\\\"passed\\\":$PASSED,\\\"failed\\\":$FAILED,\\\"link\\\":\\\"https://github.com/corbado/javascript/actions/runs/${GITHUB_RUN_ID}\\\"}\"}]"
          aws logs put-log-events --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME" --log-events "$LOG_EVENT_JSON"

      - name: Aggregate results
        run: |
          FAILED_STEPS=""
          if [ "${COMPLETE_REACT_FAILED}" = "true" ]; then
            FAILED_STEPS="${FAILED_STEPS} Complete-React"
          fi
          if [ "${COMPLETE_WEBJS_FAILED}" = "true" ]; then
            FAILED_STEPS="${FAILED_STEPS} Complete-WebJs"
          fi
          if [ "${COMPLETE_WEBJSSCRIPT_FAILED}" = "true" ]; then
            FAILED_STEPS="${FAILED_STEPS} Complete-WebJsScript"
          fi
          if [ "${CONNECT_REACT_FAILED}" = "true" ]; then
            FAILED_STEPS="${FAILED_STEPS} Connect-React"
          fi

          if [ -n "$FAILED_STEPS" ]; then
            echo "The following test steps have failed: $FAILED_STEPS"
            exit 1
          else
            echo "All tests passed."
          fi