matrix complete and connect tests

Author: aehnh

.github/workflows/test.yml

@@ -12,6 +12,23 @@ jobs:
   test:
     timeout-minutes: 60
     runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - testType: complete
+            configFile: playwright.config.complete.ts
+            extraEnv: |
+              PLAYWRIGHT_JWT_TOKEN: ${{ secrets.PLAYWRIGHT_JWT_TOKEN }}
+              PLAYWRIGHT_GOOGLE_EMAIL: ${{ secrets.PLAYWRIGHT_GOOGLE_EMAIL }}
+              PLAYWRIGHT_GOOGLE_PASSWORD: ${{ secrets.PLAYWRIGHT_GOOGLE_PASSWORD }}
+              PLAYWRIGHT_GOOGLE_TOTP_SECRET: ${{ secrets.PLAYWRIGHT_GOOGLE_TOTP_SECRET }}
+          - testType: connect
+            configFile: playwright.config.connect.ts
+            extraEnv: |
+              BACKEND_API_BASIC_AUTH: ${{ secrets.BACKEND_API_BASIC_AUTH }}
+              PLAYWRIGHT_CONNECT_PROJECT_ID: ${{ secrets.PLAYWRIGHT_CONNECT_PROJECT_ID }}
+              PLAYWRIGHT_NGROK_AUTH_TOKEN: ${{ secrets.PLAYWRIGHT_NGROK_AUTH_TOKEN }}
 
     steps:
       - name: Checkout code
@@ -90,78 +107,26 @@ jobs:
           echo "LOG_STREAM_NAME=javascript-$TIMESTAMP" >> $GITHUB_ENV
           aws logs create-log-stream --log-group-name "test-results-board" --log-stream-name "javascript-$TIMESTAMP"
 
-      - name: Run Complete tests for react
-        run: |
-          cd packages/tests-e2e
-          set +e
-          npx playwright test --config=playwright.config.complete.ts
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "COMPLETE_REACT_FAILED=true" >> $GITHUB_ENV
-          fi
-        env:
+      - name: Run ${{ matrix.testType }} tests for react
+        working-directory: packages/tests-e2e
+        env: |
           PLAYWRIGHT_NUM_CORES: 4
-          PLAYWRIGHT_JWT_TOKEN: ${{ secrets.PLAYWRIGHT_JWT_TOKEN }}
-          PLAYWRIGHT_GOOGLE_EMAIL: ${{ secrets.PLAYWRIGHT_GOOGLE_EMAIL }}
-          PLAYWRIGHT_GOOGLE_PASSWORD: ${{ secrets.PLAYWRIGHT_GOOGLE_PASSWORD }}
-          PLAYWRIGHT_GOOGLE_TOTP_SECRET: ${{ secrets.PLAYWRIGHT_GOOGLE_TOTP_SECRET }}
           GITHUB_RUN_ID: ${{ github.run_id }}
           SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }}
           GITHUB_BRANCH_NAME: ${{ github.ref_name }}
-        continue-on-error: true
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: playwright-report-complete-react
-          path: packages/tests-e2e/playwright-report/
-          retention-days: 30
-
-      - name: Send logs to AWS CloudWatch
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_REGION: ${{ secrets.AWS_REGION }}
+          ${{ matrix.extraEnv }}
         run: |
-          cd packages/tests-e2e
-          TESTS=$(xmllint --xpath 'string(/testsuites/@tests)' test-results/results.xml)
-          FAILURES=$(xmllint --xpath 'string(/testsuites/@failures)' test-results/results.xml)
-          SKIPPED=$(xmllint --xpath 'string(/testsuites/@skipped)' test-results/results.xml)
-          ERRORS=$(xmllint --xpath 'string(/testsuites/@errors)' test-results/results.xml)
-          TIME=$(xmllint --xpath 'string(/testsuites/@time)' test-results/results.xml)
-          PASSED=$((TESTS - FAILURES - ERRORS - SKIPPED))
-          FAILED=$((FAILURES + ERRORS))
-          TIMESTAMP=$(date +%s000)
-          LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"{\\\"application\\\":\\\"complete\\\",\\\"platform\\\":\\\"react\\\",\\\"run_type\\\":\\\"commitly\\\",\\\"execution_time\\\":$TIME,\\\"passed\\\":$PASSED,\\\"failed\\\":$FAILED,\\\"link\\\":\\\"https://github.com/corbado/javascript/actions/runs/${GITHUB_RUN_ID}\\\"}\"}]"
-          aws logs put-log-events --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME" --log-events "$LOG_EVENT_JSON"
-
-      - name: Run Connect tests for react
-        run: |
-          cd packages/tests-e2e
-          BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-')
-          export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.connect-next.playground.corbado.io"
-          set +e
-          npx playwright test --config=playwright.config.connect.ts
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "CONNECT_REACT_FAILED=true" >> $GITHUB_ENV
-          fi
-        env:
-          PLAYWRIGHT_NUM_CORES: 4
-          BACKEND_API_BASIC_AUTH: ${{ secrets.BACKEND_API_BASIC_AUTH }}
-          PLAYWRIGHT_CONNECT_PROJECT_ID: ${{ secrets.PLAYWRIGHT_CONNECT_PROJECT_ID }}
-          PLAYWRIGHT_NGROK_AUTH_TOKEN: ${{ secrets.PLAYWRIGHT_NGROK_AUTH_TOKEN }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }}
-          GITHUB_BRANCH_NAME: ${{ env.BRANCH_NAME_RAW }}
-        continue-on-error: true
+          npx playwright test --config=${{ matrix.configFile }}
 
       - uses: actions/upload-artifact@v4
+        if: always()
         with:
-          name: playwright-report-connect-react
+          name: playwright-report-${{ matrix.testType }}-react
           path: packages/tests-e2e/playwright-report/
           retention-days: 30
 
       - name: Send logs to AWS CloudWatch
+        if: always()
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -176,24 +141,5 @@ jobs:
           PASSED=$((TESTS - FAILURES - ERRORS - SKIPPED))
           FAILED=$((FAILURES + ERRORS))
           TIMESTAMP=$(date +%s000)
-          LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"{\\\"application\\\":\\\"connect\\\",\\\"platform\\\":\\\"react\\\",\\\"run_type\\\":\\\"commitly\\\",\\\"execution_time\\\":$TIME,\\\"passed\\\":$PASSED,\\\"failed\\\":$FAILED,\\\"link\\\":\\\"https://github.com/corbado/javascript/actions/runs/${GITHUB_RUN_ID}\\\"}\"}]"
+          LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"{\\\"application\\\":\\\"${{ matrix.testType }}\\\",\\\"platform\\\":\\\"react\\\",\\\"run_type\\\":\\\"commitly\\\",\\\"execution_time\\\":$TIME,\\\"passed\\\":$PASSED,\\\"failed\\\":$FAILED,\\\"link\\\":\\\"https://github.com/corbado/javascript/actions/runs/${GITHUB_RUN_ID}\\\"}\"}]"
           aws logs put-log-events --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME" --log-events "$LOG_EVENT_JSON"
-
-      - name: Aggregate results
-        run: |
-          FAILED_STEPS=""
-          if [ "${COMPLETE_REACT_FAILED}" = "true" ]; then
-            FAILED_STEPS="${FAILED_STEPS} Complete-React"
-          fi
-          if [ "${COMPLETE_WEBJS_FAILED}" = "true" ]; then
-            FAILED_STEPS="${FAILED_STEPS} Complete-WebJs"
-          fi
-          if [ "${COMPLETE_WEBJSSCRIPT_FAILED}" = "true" ]; then
-            FAILED_STEPS="${FAILED_STEPS} Complete-WebJsScript"
-          fi
-          if [ -n "$FAILED_STEPS" ]; then
-            echo "The following test steps have failed: $FAILED_STEPS"
-            exit 1
-          else
-            echo "All tests passed."
-          fi