GH Actions: test against different versions of all CS dependencies

jrfnl · jrfnl · commit 01c27cbe3767 · 2023-12-13T12:17:44.000+01:00
As things were, when testing against "dev" versions, only PHPCS "dev" was used, while WPCS now has more dependencies, i.e. PHPCSUtils and PHPCSExtra.

This commit updates the workflows to take that into account better.
It also improves the readability of the workflows by using multi-line command in a few places.

Notes:
* This renames the `phpcs_version` matrix variable to `dependencies` in all workflows to make it clear this is not just about PHPCS itself, but about all dependencies.
* The CS check for the own codebase is always run against "dev" versions of all dependencies as an early detection system for upstream issues.
* The "Ruleset" checks are run against low/stable/dev.
* The fixer conflict check is run against dev only, as conflicts detected cannot be fixed anymore in already released versions.
* The "quick check" now only runs against low/stable.
* The "test" run run against both low/stable by default and now has some extra builds against "dev" versions.
    - These extra builds could be set to "continue on error", but I've chosen not to do so as if any issues are detected, they should be fixed asap as they will impact WPCS sooner rather than later.
    - Code coverage will now be run against low/high PHP with low/stable dependencies and no longer against dev.
* Also note that, while the "quick test"/"test" runs shouldn't need PHPCSExtra, as that is a dependency used only in the ruleset, not in our sniff code, I've still included PHPCSExtra in the upgrade/downgrade block for consistency.

These changes do mean that the "required build"/branch protection will need to be updated (again) before the PR can be merged.
I will do so once the PR has been approved.
diff --git a/.github/workflows/basic-qa.yml b/.github/workflows/basic-qa.yml
@@ -12,6 +12,11 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  PHPCS_DEV: 'dev-master'
+  UTILS_DEV: 'dev-develop'
+  EXTRA_DEV: 'dev-develop'
+
 jobs:
   # Check code style of sniffs, rulesets and XML documentation.
   # Check that all sniffs are feature complete.
@@ -37,9 +42,13 @@ jobs:
       - name: Validate the composer.json file
         run: composer validate --no-check-all --strict
 
-      # Using PHPCS `master` as an early detection system for bugs upstream.
-      - name: Set PHPCS version
-        run: composer require squizlabs/php_codesniffer:"dev-master" --no-update --no-scripts --no-interaction
+      # Using dev versions of the dependencies as an early detection system for bugs upstream.
+      - name: "Composer: set PHPCS dependencies (dev)"
+        run: >
+          composer require --no-update --no-scripts --no-interaction
+          squizlabs/php_codesniffer:"${{ env.PHPCS_DEV }}"
+          phpcsstandards/phpcsutils:"${{ env.UTILS_DEV }}"
+          phpcsstandards/phpcsextra:"${{ env.EXTRA_DEV }}"
 
       - name: Install Composer dependencies
         uses: ramsey/composer-install@v2
@@ -93,15 +102,15 @@ jobs:
 
   # Makes sure the rulesets don't throw unexpected errors or warnings.
   # This workflow needs to be run against a high PHP version to prevent triggering the syntax error check.
-  # It also needs to be run against all PHPCS versions WPCS is tested against.
+  # It also needs to be run against all dependency versions WPCS is tested against.
   ruleset-tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         php: [ 'latest' ]
-        phpcs_version: [ 'lowest', 'dev-master' ]
+        dependencies: [ 'lowest', 'stable', 'dev' ]
 
-    name: "Ruleset test: PHP ${{ matrix.php }} on PHPCS ${{ matrix.phpcs_version }}"
+    name: "Ruleset test: PHP ${{ matrix.php }} on PHPCS ${{ matrix.dependencies }}"
 
     steps:
       - name: Checkout repository
@@ -115,9 +124,13 @@ jobs:
           ini-values: error_reporting = E_ALL & ~E_DEPRECATED
           coverage: none
 
-      - name: "Set PHPCS version (master)"
-        if: ${{ matrix.phpcs_version != 'lowest' }}
-        run: composer require squizlabs/php_codesniffer:"${{ matrix.phpcs_version }}" --no-update --no-scripts --no-interaction
+      - name: "Composer: set PHPCS dependencies for tests (dev)"
+        if: ${{ matrix.dependencies == 'dev' }}
+        run: >
+          composer require --no-update --no-scripts --no-interaction
+          squizlabs/php_codesniffer:"${{ env.PHPCS_DEV }}"
+          phpcsstandards/phpcsutils:"${{ env.UTILS_DEV }}"
+          phpcsstandards/phpcsextra:"${{ env.EXTRA_DEV }}"
 
       - name: Install Composer dependencies
         uses: ramsey/composer-install@v2
@@ -126,8 +139,13 @@ jobs:
           # Bust the cache at least once a month - output format: YYYY-MM.
           custom-cache-suffix: $(date -u "+%Y-%m")
 
-      - name: "Set PHPCS version (lowest)"
-        run: composer update squizlabs/php_codesniffer --prefer-lowest --no-scripts --no-interaction
+      - name: "Composer: downgrade PHPCS dependencies for tests (lowest)"
+        if: ${{ matrix.dependencies == 'lowest' }}
+        run: >
+          composer update --prefer-lowest --no-scripts --no-interaction
+          squizlabs/php_codesniffer
+          phpcsstandards/phpcsutils
+          phpcsstandards/phpcsextra
 
       - name: Test the WordPress-Core ruleset
         run: $(pwd)/vendor/bin/phpcs -ps ./Tests/RulesetCheck/class-ruleset-test.inc --standard=WordPress-Core
@@ -150,8 +168,9 @@ jobs:
       # Test for fixer conflicts by running the auto-fixers of the complete WPCS over the test case files.
       # This is not an exhaustive test, but should give an early indication for typical fixer conflicts.
       # If only fixable errors are found, the exit code will be 1, which can be interpreted as success.
+      # This check is only run against "dev" as conflicts can not be fixed for already released versions.
       - name: Test for fixer conflicts (fixes expected)
-        if: ${{ matrix.phpcs_version == 'dev-master' }}
+        if: ${{ matrix.dependencies == 'dev' }}
         id: phpcbf
         continue-on-error: true
         run: |
diff --git a/.github/workflows/quicktest.yml b/.github/workflows/quicktest.yml
@@ -24,48 +24,39 @@ jobs:
     strategy:
       matrix:
         php: [ '5.4', 'latest' ]
-        phpcs_version: [ 'lowest', 'dev-master' ]
+        dependencies: [ 'lowest', 'stable' ]
 
-    name: QTest - PHP ${{ matrix.php }} on PHPCS ${{ matrix.phpcs_version }}
+    name: QTest - PHP ${{ matrix.php }} on PHPCS ${{ matrix.dependencies }}
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      # On stable PHPCS versions, allow for PHP deprecation notices.
-      # Unit tests don't need to fail on those for stable releases where those issues won't get fixed anymore.
-      - name: Setup ini config
-        id: set_ini
-        run: |
-          if [ "${{ matrix.phpcs_version }}" != "dev-master" ]; then
-            echo 'PHP_INI=error_reporting=E_ALL & ~E_DEPRECATED, display_errors=On' >> $GITHUB_OUTPUT
-          else
-            echo 'PHP_INI=error_reporting=-1, display_errors=On' >> $GITHUB_OUTPUT
-          fi
-
       - name: Set up PHP
         uses: shivammathur/setup-php@v2
         with:
           php-version: ${{ matrix.php }}
-          ini-values: ${{ steps.set_ini.outputs.PHP_INI }}
+          # With stable PHPCS dependencies, allow for PHP deprecation notices.
+          # Unit tests don't need to fail on those for stable releases where those issues won't get fixed anymore.
+          ini-values: error_reporting=-1, display_errors=On
           coverage:  ${{ github.ref_name == 'develop' && 'xdebug' || 'none' }}
 
-      - name: "Set PHPCS version (master)"
-        if: ${{ matrix.phpcs_version != 'lowest' }}
-        run: composer require squizlabs/php_codesniffer:"${{ matrix.phpcs_version }}" --no-update --no-scripts --no-interaction
-
       - name: Install Composer dependencies
         uses: ramsey/composer-install@v2
         with:
           # Bust the cache at least once a month - output format: YYYY-MM.
           custom-cache-suffix: $(date -u "+%Y-%m")
 
-      - name: "Set PHPCS version (lowest)"
-        if: ${{ matrix.phpcs_version == 'lowest' }}
-        run: composer update squizlabs/php_codesniffer --prefer-lowest --ignore-platform-req=php+ --no-scripts --no-interaction
+      - name: "Composer: downgrade PHPCS dependencies for tests (lowest)"
+        if: ${{ matrix.dependencies == 'lowest' }}
+        run: >
+          composer update --prefer-lowest --no-scripts --no-interaction
+          squizlabs/php_codesniffer
+          phpcsstandards/phpcsutils
+          phpcsstandards/phpcsextra
 
       - name: Lint PHP files against parse errors
-        if: ${{ matrix.phpcs_version == 'dev-master' }}
+        if: ${{ matrix.dependencies == 'stable' }}
         run: composer lint -- --checkstyle
 
       - name: Run the unit tests without code coverage
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -14,60 +14,83 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  PHPCS_DEV: 'dev-master'
+  UTILS_DEV: 'dev-develop'
+  EXTRA_DEV: 'dev-develop'
+
 jobs:
   # Runs the test suite against all supported branches and combinations.
-  # Linting is performed on all jobs run against PHPCS `dev-master`.
+  # Linting is performed on all jobs run with dependencies `stable`.
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         php: [ '5.5', '5.6', '7.0', '7.1', '7.2', '7.3', '7.4', '8.0', '8.1', '8.2', '8.4' ]
-        phpcs_version: [ 'lowest', 'dev-master' ]
+        dependencies: [ 'lowest', 'stable' ]
         extensions: [ '' ]
         coverage: [false]
 
         include:
           - php: '7.4'
-            phpcs_version: 'dev-master'
+            dependencies: 'stable'
             extensions: ':mbstring' # = Disable Mbstring.
             coverage: true # Make sure coverage is recorded for this too.
 
           # Run code coverage builds against high/low PHP and high/low PHPCS.
           - php: '5.4'
-            phpcs_version: 'dev-master'
+            dependencies: 'stable'
             extensions: ''
             coverage: true
           - php: '5.4'
-            phpcs_version: 'lowest'
+            dependencies: 'lowest'
             extensions: ''
             coverage: true
           - php: '8.3'
-            phpcs_version: 'dev-master'
+            dependencies: 'stable'
             extensions: ''
             coverage: true
           - php: '8.3'
-            phpcs_version: 'lowest'
+            dependencies: 'lowest'
             extensions: ''
             coverage: true
 
+          # Test against dev versions of all dependencies with select PHP versions for early detection of issues.
+          - php: '5.4'
+            dependencies: 'dev'
+            extensions: ''
+            coverage: false
+          - php: '7.0'
+            dependencies: 'dev'
+            extensions: ''
+            coverage: false
+          - php: '7.4'
+            dependencies: 'dev'
+            extensions: ''
+            coverage: false
+          - php: '8.3'
+            dependencies: 'dev'
+            extensions: ''
+            coverage: false
+
           # Add extra build to test against PHPCS 4.
           #- php: '7.4'
-          #  phpcs_version: '4.0.x-dev as 3.9.99'
+          #  dependencies: '4.0.x-dev as 3.9.99'
 
-    name: PHP ${{ matrix.php }} on PHPCS ${{ matrix.phpcs_version }}
+    name: PHP ${{ matrix.php }} on PHPCS ${{ matrix.dependencies }}
 
     continue-on-error: ${{ matrix.php == '8.4' }}
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      # On stable PHPCS versions, allow for PHP deprecation notices.
+      # With stable PHPCS dependencies, allow for PHP deprecation notices.
       # Unit tests don't need to fail on those for stable releases where those issues won't get fixed anymore.
       - name: Setup ini config
         id: set_ini
         run: |
-          if [ "${{ matrix.phpcs_version }}" != "dev-master" ]; then
+          if [ "${{ matrix.dependencies }}" != "dev" ]; then
             echo 'PHP_INI=error_reporting=E_ALL & ~E_DEPRECATED, display_errors=On' >> $GITHUB_OUTPUT
           else
             echo 'PHP_INI=error_reporting=-1, display_errors=On' >> $GITHUB_OUTPUT
@@ -81,22 +104,30 @@ jobs:
           coverage: ${{ matrix.coverage && 'xdebug' || 'none' }}
           tools: cs2pr
 
-      - name: "Set PHPCS version (master)"
-        if: ${{ matrix.phpcs_version != 'lowest' }}
-        run: composer require squizlabs/php_codesniffer:"${{ matrix.phpcs_version }}" --no-update --no-scripts --no-interaction
+      - name: "Composer: set PHPCS dependencies for tests (dev)"
+        if: ${{ matrix.dependencies == 'dev' }}
+        run: >
+          composer require --no-update --no-scripts --no-interaction
+          squizlabs/php_codesniffer:"${{ env.PHPCS_DEV }}"
+          phpcsstandards/phpcsutils:"${{ env.UTILS_DEV }}"
+          phpcsstandards/phpcsextra:"${{ env.EXTRA_DEV }}"
 
       - name: Install Composer dependencies
         uses: ramsey/composer-install@v2
         with:
           # Bust the cache at least once a month - output format: YYYY-MM.
           custom-cache-suffix: $(date -u "+%Y-%m")
 
-      - name: "Set PHPCS version (lowest)"
-        if: ${{ matrix.phpcs_version == 'lowest' }}
-        run: composer update squizlabs/php_codesniffer --prefer-lowest --ignore-platform-req=php+ --no-scripts --no-interaction
+      - name: "Composer: downgrade PHPCS dependencies for tests (lowest)"
+        if: ${{ ! startsWith( matrix.php, '8' ) && matrix.dependencies == 'lowest' }}
+        run: >
+          composer update --prefer-lowest --no-scripts --no-interaction
+          squizlabs/php_codesniffer
+          phpcsstandards/phpcsutils
+          phpcsstandards/phpcsextra
 
       - name: Lint PHP files against parse errors
-        if: ${{ matrix.phpcs_version == 'dev-master' }}
+        if: ${{ matrix.dependencies == 'stable' }}
         run: composer lint -- --checkstyle | cs2pr
 
       - name: Run the unit tests without code coverage
