diff --git a/.github/workflows/e2e_tests_update.yml b/.github/workflows/e2e_tests_update.yml new file mode 100644 index 0000000000..4abd29b747 --- /dev/null +++ b/.github/workflows/e2e_tests_update.yml @@ -0,0 +1,401 @@ +name: "🎳 End2end with update" +on: + pull_request: + branches: + - master + - release_3_* + # allow to run manually + workflow_dispatch: + +env: + NODE_VERSION: "22" + PHP_VERSION: "8.3" + PG_POSTGIS_VERSION: "15-3" + QGIS_SERVER_VERSION: "3.40" + UPDATE_PROJECTS: "FALSE" + +jobs: + end2end: + # name: "E2E Update QGIS ${{ env.QGIS_SERVER_VERSION }} PG ${{ env.PG_POSTGIS_VERSION }} PHP ${{ env.PHP_VERSION }}" + permissions: + issues: write + pull-requests: write + runs-on: ubuntu-latest + defaults: + run: + working-directory: tests + strategy: + fail-fast: false + env: + PLAYWRIGHT_FORCE_TTY: true + PLAYWRIGHT_LIST_PRINT_STEPS: true + PLAYWRIGHT_JSON_OUTPUT_DIR: ${{ github.workspace }}/tests/end2end/playwright-report + PLAYWRIGHT_OPTIONS: --project=end2end + FORCE_COLOR: true + steps: + + - name: Login to Docker Hub + if: ${{ github.secret_source == 'Actions' }} + uses: docker/login-action@v4 + env: + ACTIONS_STEP_DEBUG: true + with: + username: "3liz" + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Checkout 3.9.6 + uses: actions/checkout@v6 + with: + ref: '3.9.6' + + - name: Make environment and show Lizmap versions + env: + PHP_VERSION: ${{ env.PHP_VERSION }} + LZMPOSTGISVERSION: ${{ env.PG_POSTGIS_VERSION }} + LZMQGSRVVERSION: ${{ env.QGIS_SERVER_VERSION }} + run: | + mkdir -p ${{env.PLAYWRIGHT_JSON_OUTPUT_DIR}} + make env + cat .env + + - name: Read environment file and set variables + uses: cosq-network/dotenv-loader@v1.0.2 + with: + # Somehow, the working-directory is not taken into account + env-file: tests/.env + + # For testing only + - name: Update all QGIS projects to QGIS Desktop ${{ env.QGIS_SERVER_VERSION }} by opening them and rewriting them + if: | + env.UPDATE_PROJECTS == 'TRUE' + run: | + make upgrade-projects + git status + git diff qgis-projects/tests/ + + - name: Run docker compose + run: | + make env + docker compose version + echo "Building image" + docker compose build --quiet + echo "Starting docker stack - wait for healthy states" + docker compose up pgsql -d --quiet-pull + docker compose up redis -d --quiet-pull + docker compose up map -d --quiet-pull + docker compose up lizmap -d --quiet-pull + docker compose ps + + - name: Install QGIS server plugins + run: make build-plugins + + - name: Show QGIS server environment + run: | + make show-qgis-server-versions + + - name: Load SQL data + run: | + cd qgis-projects/tests + ./load_sql.sh + + - name: Add hosts to /etc/hosts + run: | + sudo echo "127.0.0.1 othersite.local" | sudo tee -a /etc/hosts + + - name: "Setup Node ${{ env.NODE_VERSION }}" + uses: actions/setup-node@v6 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: ${{ github.workspace }}/package.json + + # checkout HEAD + - name: Checkout HEAD + uses: actions/checkout@v6 + with: + clean: false + + - name: Make environment and show Lizmap versions + env: + PHP_VERSION: ${{ env.PHP_VERSION }} + LZMPOSTGISVERSION: ${{ env.PG_POSTGIS_VERSION }} + LZMQGSRVVERSION: ${{ env.QGIS_SERVER_VERSION }} + run: | + mkdir -p ${{env.PLAYWRIGHT_JSON_OUTPUT_DIR}} + make env + cat .env + + - name: Read environment file and set variables + uses: cosq-network/dotenv-loader@v1.0.2 + with: + # Somehow, the working-directory is not taken into account + env-file: tests/.env + + - name: Run docker compose + run: | + make env + docker compose version + echo "Building image" + docker compose build --quiet + echo "Starting docker stack - wait for healthy states" + docker compose up -d --quiet-pull + docker compose ps + + - name: Install QGIS server plugins + run: make build-plugins + + - name: Show QGIS server environment + run: | + make show-qgis-server-versions + + - name: Check about QGIS Server status + id: qgis-server-status + run: | + curl \ + --user 'admin:admin' \ + --retry 30 \ + --retry-delay 5 \ + -N \ + "http://localhost:8130/index.php/view/app/metadata" \ + -o /tmp/test-qgis-server.json + + cat /tmp/test-qgis-server.json | jq .qgis_server_info + qgis_info=$(cat /tmp/test-qgis-server.json | jq --raw-output '.qgis_server_info.error') + echo $qgis_info + if [[ "$qgis_info" != "null" ]]; then + echo "QGIS Server is not well configured" + echo "::warning QGIS Server was not up" + exit 1 + else + echo "JSON metadata OK about QGIS Server" + fi + + - name: Check about updated files after a build (PHP or JS) + run: | + if [[ -z $(git status --porcelain -uno) ]]; then + echo "No updated files 👍" + else + echo "Updated files" + git status + echo "::warning Git status is not clean" + # exit 1 + fi + + - name: Npm run build + working-directory: ./ + run: | + rm -rf node_modules/* + npm ci + npm run build + + - name: Run BATS tests + working-directory: ./ + run: | + npm run bats + + - name: Get Playwright version + id: playwright-version + run: | + VERSION=$(node -p "require('@playwright/test/package.json').version") + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Cache Playwright browsers + uses: actions/cache@v4 + id: playwright-cache + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ steps.playwright-version.outputs.version }} + + - name: Install Playwright with browsers + if: steps.playwright-cache.outputs.cache-hit != 'true' + working-directory: tests/end2end + run: | + npx playwright install --with-deps chromium + + - name: Install Playwright dependencies + if: steps.playwright-cache.outputs.cache-hit == 'true' + working-directory: tests/end2end + run: | + npx playwright install-deps + + - name: Run Playwright tests tagged "@requests" and "@readonly" + id: test-playwright-requests + working-directory: tests/end2end + env: + CRTF_JSON_FILE: playwright-tagged-requests.json + PLAYWRIGHT_JSON_OUTPUT_NAME: tests-results-requests.json + run: | + npx playwright test --grep "(?=.*@requests)(?=.*@readonly)" ${{ env.PLAYWRIGHT_OPTIONS }} + + - name: Run Playwright tests tagged "@readonly" + id: test-playwright-read-only + working-directory: tests/end2end + env: + CRTF_JSON_FILE: playwright-tagged-readonly.json + PLAYWRIGHT_JSON_OUTPUT_NAME: tests-results-readonly.json + if: | + success() || + steps.test-playwright-requests.outcome != 'success' + run: | + npx playwright test --grep @readonly --grep-invert @requests ${{ env.PLAYWRIGHT_OPTIONS }} + + - name: Prepare the database diff from Playwright "@readonly" tests + if: | + success() || + steps.test-playwright-requests.outcome != 'success' || + steps.test-playwright-read-only.outcome != 'success' + run: | + ./lizmap-ctl dump-pgsql-data + git diff --exit-code qgis-projects/tests/tests_dataset_data.sql + db_diff=$? + echo "db_diff=${db_diff}" >> "$GITHUB_OUTPUT" + if [[ -z ${db_diff} ]]; then + echo "No updated files 👍" + else + echo "Updated files" + git diff qgis-projects/tests/tests_dataset_data.sql > qgis-projects/tests/tests_dataset_data.patch + git restore qgis-projects/tests/tests_dataset_data.sql + echo "::warning Git status is not clean after running tests about the DB" + exit 0 + fi + + - name: Upload DB results + if: | + success() || + steps.test-playwright-requests.outcome != 'success' || + steps.test-playwright-read-only.outcome != 'success' + uses: actions/upload-artifact@master + with: + name: ${{ matrix.name }}-DB-diff-read-only + if-no-files-found: 'ignore' + path: | + ${{ github.workspace }}/tests/qgis-projects/tests/tests_dataset.patch + + - name: Check about updated files after read only tests + if: | + success() || + steps.test-playwright-requests.outcome != 'success' || + steps.test-playwright-read-only.outcome != 'success' + run: | + if [[ -z $(git status --porcelain -uno) ]]; then + echo "No updated files 👍" + else + echo "Updated files" + git status + echo "::warning Git status is not clean after running tests about tracked files" + fi + + - name: Run Playwright tests tagged neither "@readonly" nor "@write" + id: test-playwright-no-tag + working-directory: tests/end2end + env: + CRTF_JSON_FILE: playwright-no-tag.json + PLAYWRIGHT_JSON_OUTPUT_NAME: tests-results-no-tag.json + if: | + steps.qgis-server-status.outcome == 'success' && + ( success() || + steps.test-playwright-requests.outcome != 'success' || + steps.test-playwright-read-only.outcome != 'success' ) + run: | + npx playwright test --workers 1 --grep-invert "(?=.*@write|.*@readonly)" ${{ env.PLAYWRIGHT_OPTIONS }} + + - name: Run Playwright tests tagged "@write" + id: test-playwright-write + working-directory: tests/end2end + env: + CRTF_JSON_FILE: playwright-tagged-write.json + PLAYWRIGHT_JSON_OUTPUT_NAME: tests-results-write.json + if: | + steps.qgis-server-status.outcome == 'success' && + ( success() || + steps.test-playwright-requests.outcome != 'success' || + steps.test-playwright-read-only.outcome != 'success' || + steps.test-playwright-no-tag.outcome != 'success' ) + run: | + npx playwright test --workers 1 --grep @write ${{ env.PLAYWRIGHT_OPTIONS }} + + - name: Debug + if: always() + working-directory: tests + run: | + find . -type d -name "test-results" + echo $GITHUB_WORKSPACE + echo ${{ github.workspace }} + echo ${{ github.workspace }}/tests/end2end/test-results/ + ls -l $GITHUB_WORKSPACE/tests/end2end/test-results/ + ls ${{ github.workspace }}/tests/end2end/test-results/ + + - name: Send screenshots if necessary about all Playwright tests, if one failed + if: | + failure() && + ( steps.test-playwright-requests.outcome != 'success' || + steps.test-playwright-read-only.outcome != 'success' || + steps.test-playwright-no-tag.outcome != 'success' || + steps.test-playwright-write.outcome != 'success' ) + uses: actions/upload-artifact@master + with: + name: ${{ matrix.name }}-screenshots-readonly + if-no-files-found: 'ignore' + path: | + ${{ github.workspace }}/tests/end2end/test-results/ + + - name: Publish test report + continue-on-error: true + if: | + success() || + steps.test-playwright-requests.outcome != 'success' || + steps.test-playwright-read-only.outcome != 'success' || + steps.test-playwright-no-tag.outcome != 'success' || + steps.test-playwright-write.outcome != 'success' + uses: ctrf-io/github-test-reporter@v1.0.28 + env: + GITHUB_TOKEN: ${{ secrets.BOT_HUB_TOKEN || github.token }} + with: + report-path: './tests/end2end/ctrf/*.json' + summary-report: true + flaky-report: true + flaky-rate-report: true + summary: true + title: All Playwright tests ${{ matrix.NAME }} + failed-report: true + # pull-request-report: true + pull-request: true + # annotate: false + update-comment: true + overwrite-comment: false + comment-tag: '${{ github.workflow }}-${{ github.job }}' + + - name: Upload test results + if: | + success() || + steps.test-playwright-requests.outcome != 'success' || + steps.test-playwright-read-only.outcome != 'success' || + steps.test-playwright-no-tag.outcome != 'success' || + steps.test-playwright-write.outcome != 'success' + uses: actions/upload-artifact@master + with: + if-no-files-found: 'ignore' + name: ${{ matrix.name }}-playwright-report + path: ${{ github.workspace }}/tests/end2end/playwright-report + + - name: Export some logs to files + if: always() + run: | + mkdir -p /tmp/e2e/lwc + mkdir -p /tmp/e2e/docker + docker logs lizmap${{ env.LZMBRANCH }}_test_qgis &> /tmp/e2e/docker/qgis-server.log | true + docker logs lizmap${{ env.LZMBRANCH }}_test_php &> /tmp/e2e/docker/php.log | true + docker logs lizmap${{ env.LZMBRANCH }}_test_nginx &> /tmp/e2e/docker/nginx.log | true + cp -r ../lizmap/var/log /tmp/e2e/lwc/ + + - name: Upload all logs as artifact + uses: actions/upload-artifact@master + if: always() + with: + name: ${{ matrix.name }}-E2E-all-logs + path: | + /tmp/e2e/ + + - name: Database diff check after readonly tests + run: + exit ${{ env.db_diff }}