Define polyfilled T_* constants from Tokenizer as int

.github/CONTRIBUTING.md

@@ -399,6 +399,8 @@ To run the tests specific to the use of `PHP_CODESNIFFER_CBF === true`:
 
 ### Writing End-to-End Tests
 
+#### Bashunit
+
 Bash-based end-to-end tests can be written using the [Bashunit](https://bashunit.typeddevs.com/) test tooling using version 0.26.0 or higher.
 
 To install bashunit, follow the [installation guide](https://bashunit.typeddevs.com/installation).
@@ -413,6 +415,20 @@ You can then run the bashunit tests on Linux/Mac/WSL, like so:
 When writing end-to-end tests, please use fixtures for the "files under scan" to make the tests stable.
 These fixtures can be placed in the `tests/EndToEndBash/Fixtures` subdirectory.
 
+#### phpt
+
+PHP-based end-to-end tests can be written using the `phpt` format. This is the format that PHP uses for its own tests. We use PHPUnit to run these tests.
+
+```bash
+vendor/bin/phpunit -c phpunit-e2e.xml.dist
+```
+
+The following resources may be helpful when writing `phpt`-style tests:
+
+* <https://www.phpinternalsbook.com/tests/phpt_file_structure.html>
+* <https://php.github.io/php-src/miscellaneous/writing-tests.html>
+* <https://dev.to/moxio/start-testing-with-phpt-tests-in-phpunit-2jpg>
+* <https://github.com/sebastianbergmann/phpunit-documentation-english/issues/302>
 
 ### Submitting Your Pull Request
 

.github/workflows/end-to-end-tests.yml

@@ -15,22 +15,21 @@ on:
   # Allow manually triggering the workflow.
   workflow_dispatch:
 
+# Cancels all previous workflow runs for the same branch that have not yet completed.
+concurrency:
+  # The concurrency group contains the workflow name and the branch name.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   bash-tests:
-    # Cancels all previous runs of this particular job for the same branch that have not yet completed.
-    concurrency:
-      # The concurrency group contains the workflow name, job name, job index and the branch name.
-      group: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index }}-${{ github.ref }}
-      cancel-in-progress: true
-
     runs-on: 'ubuntu-latest'
 
     strategy:
       matrix:
         php: ['7.2', '7.3', '7.4', '8.0', '8.1', '8.2', '8.3', '8.4', '8.5', 'nightly']
 
-    # yamllint disable-line rule:line-length
-    name: "E2E PHP: ${{ matrix.php }}"
+    name: "E2E Bash PHP: ${{ matrix.php }}"
 
     continue-on-error: ${{ matrix.php == 'nightly' }}
 
@@ -43,13 +42,6 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Install PHP
-        uses: shivammathur/setup-php@44454db4f0199b8b9685a5d763dc37cbf79108e1 # 2.36.0
-        with:
-          php-version: ${{ matrix.php }}
-          ini-values: "error_reporting=-1, display_errors=On, display_startup_errors=On"
-          coverage: none
-
       - name: "Install bashunit"
         shell: bash
         run: |
@@ -60,3 +52,46 @@ jobs:
       - name: "Run bashunit tests"
         shell: bash
         run: "./lib/bashunit -p tests/EndToEndBash"
+
+  phpt-tests:
+    runs-on: 'ubuntu-latest'
+
+    strategy:
+      matrix:
+        php: ['7.2', '7.3', '7.4', '8.0', '8.1', '8.2', '8.3', '8.4', '8.5', 'nightly']
+
+    name: "E2E phpt PHP: ${{ matrix.php }}"
+
+    continue-on-error: ${{ matrix.php == 'nightly' }}
+
+    steps:
+      - name: Prepare git to leave line endings alone
+        run: git config --global core.autocrlf input
+
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Install PHP
+        uses: shivammathur/setup-php@44454db4f0199b8b9685a5d763dc37cbf79108e1 # 2.36.0
+        with:
+          php-version: ${{ matrix.php }}
+          ini-values: "error_reporting=-1, display_errors=On, display_startup_errors=On"
+          coverage: none
+
+      # Install dependencies and handle caching in one go.
+      # @link https://github.com/marketplace/actions/install-php-dependencies-with-composer
+      - name: Install Composer dependencies
+        uses: "ramsey/composer-install@3cf229dc2919194e9e36783941438d17239e8520" # 3.1.1
+        with:
+          composer-options: ${{ matrix.php == 'nightly' && '--ignore-platform-req=php+' || '' }}
+          custom-cache-suffix: $(date -u "+%Y-%m")
+
+      - name: 'PHPUnit: run end-to-end tests (phpt)'
+        # We have included '--no-coverage' here to make sure that we do not run with coverage enabled
+        # by mistake, if the Xdebug extension gets loaded in future. These tests are not compatible
+        # with how PHPUnit runs when coverage is enabled. This is because PHPUnit pre-loads all PHP
+        # files as part of its coverage initialisation process, and these tests specifically test the
+        # code which is used during the PHP_CodeSniffer bootstrap step.
+        run: php "vendor/bin/phpunit" -c phpunit-e2e.xml.dist --no-coverage

phpunit-e2e.xml.dist

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<phpunit
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/9.2/phpunit.xsd"
+    backupGlobals="true"
+    beStrictAboutOutputDuringTests="true"
+    beStrictAboutTestsThatDoNotTestAnything="true"
+    >
+    <testsuites>
+        <testsuite name="End2EndPhpt">
+            <directory suffix=".phpt">tests/EndToEndPhpt/</directory>
+        </testsuite>
+    </testsuites>
+</phpunit>

src/Util/Tokens.php

@@ -10,6 +10,7 @@
 
 namespace PHP_CodeSniffer\Util;
 
+// PHPCS native tokens.
 define('T_NONE', 'PHPCS_T_NONE');
 define('T_OPEN_CURLY_BRACKET', 'PHPCS_T_OPEN_CURLY_BRACKET');
 define('T_CLOSE_CURLY_BRACKET', 'PHPCS_T_CLOSE_CURLY_BRACKET');
@@ -70,84 +71,6 @@
 define('T_TYPE_OPEN_PARENTHESIS', 'PHPCS_T_TYPE_OPEN_PARENTHESIS');
 define('T_TYPE_CLOSE_PARENTHESIS', 'PHPCS_T_TYPE_CLOSE_PARENTHESIS');
 
-/*
- * {@internal IMPORTANT: all PHP native polyfilled tokens MUST be added to the
- * `PHP_CodeSniffer\Tests\Core\Util\Tokens\TokenNameTest::dataPolyfilledPHPNativeTokens()` test method!}
- */
-
-// Some PHP 7.4 tokens, replicated for lower versions.
-if (defined('T_COALESCE_EQUAL') === false) {
-    define('T_COALESCE_EQUAL', 'PHPCS_T_COALESCE_EQUAL');
-}
-
-if (defined('T_BAD_CHARACTER') === false) {
-    define('T_BAD_CHARACTER', 'PHPCS_T_BAD_CHARACTER');
-}
-
-if (defined('T_FN') === false) {
-    define('T_FN', 'PHPCS_T_FN');
-}
-
-// Some PHP 8.0 tokens, replicated for lower versions.
-if (defined('T_NULLSAFE_OBJECT_OPERATOR') === false) {
-    define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR');
-}
-
-if (defined('T_NAME_QUALIFIED') === false) {
-    define('T_NAME_QUALIFIED', 'PHPCS_T_NAME_QUALIFIED');
-}
-
-if (defined('T_NAME_FULLY_QUALIFIED') === false) {
-    define('T_NAME_FULLY_QUALIFIED', 'PHPCS_T_NAME_FULLY_QUALIFIED');
-}
-
-if (defined('T_NAME_RELATIVE') === false) {
-    define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE');
-}
-
-if (defined('T_MATCH') === false) {
-    define('T_MATCH', 'PHPCS_T_MATCH');
-}
-
-if (defined('T_ATTRIBUTE') === false) {
-    define('T_ATTRIBUTE', 'PHPCS_T_ATTRIBUTE');
-}
-
-// Some PHP 8.1 tokens, replicated for lower versions.
-if (defined('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG') === false) {
-    define('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG', 'PHPCS_T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG');
-}
-
-if (defined('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG') === false) {
-    define('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG', 'PHPCS_T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG');
-}
-
-if (defined('T_READONLY') === false) {
-    define('T_READONLY', 'PHPCS_T_READONLY');
-}
-
-if (defined('T_ENUM') === false) {
-    define('T_ENUM', 'PHPCS_T_ENUM');
-}
-
-// Some PHP 8.4 tokens, replicated for lower versions.
-if (defined('T_PUBLIC_SET') === false) {
-    define('T_PUBLIC_SET', 'PHPCS_T_PUBLIC_SET');
-}
-
-if (defined('T_PROTECTED_SET') === false) {
-    define('T_PROTECTED_SET', 'PHPCS_T_PROTECTED_SET');
-}
-
-if (defined('T_PRIVATE_SET') === false) {
-    define('T_PRIVATE_SET', 'PHPCS_T_PRIVATE_SET');
-}
-
-// Some PHP 8.5 tokens, replicated for lower versions.
-if (defined('T_VOID_CAST') === false) {
-    define('T_VOID_CAST', 'PHPCS_T_VOID_CAST');
-}
-
 // Tokens used for parsing doc blocks.
 define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
 define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');
@@ -163,6 +86,8 @@
 define('T_PHPCS_IGNORE', 'PHPCS_T_PHPCS_IGNORE');
 define('T_PHPCS_IGNORE_FILE', 'PHPCS_T_PHPCS_IGNORE_FILE');
 
+Tokens::polyfillTokenizerConstants();
+
 final class Tokens
 {
 
@@ -612,6 +537,13 @@ final class Tokens
         T_YIELD_FROM   => T_YIELD_FROM,
     ];
 
+    /**
+     * Mapping table for polyfilled constants.
+     *
+     * @var array<int, string>
+     */
+    private static $polyfillMappingTable = [];
+
     /**
      * The token weightings.
      *
@@ -943,12 +875,12 @@ final class Tokens
      */
     public static function tokenName($token)
     {
-        if (is_string($token) === false) {
-            // PHP-supplied token name.
-            return token_name($token);
+        if (is_string($token) === true) {
+            // PHPCS native token.
+            return substr($token, 6);
         }
 
-        return substr($token, 6);
+        return (self::$polyfillMappingTable[$token] ?? token_name($token));
     }
 
 
@@ -991,4 +923,99 @@ public static function getHighestWeightedToken(array $tokens)
 
         return $highestType;
     }
+
+
+    /**
+     * Polyfill tokenizer (T_*) constants.
+     *
+     * {@internal IMPORTANT: all PHP native polyfilled tokens MUST be added to the
+     * `PHP_CodeSniffer\Tests\Core\Util\Tokens\TokenNameTest::dataPolyfilledPHPNativeTokens()` test method!}
+     *
+     * @return void
+     */
+    public static function polyfillTokenizerConstants(): void
+    {
+        // Ideally this would be a private class constant. We cannot do that
+        // here as the constants that we are polyfilling in this method are
+        // used in some of the class constants for this class. If we reference
+        // any class constants or properties before this method has fully run,
+        // PHP will intitialise the class, leading to warnings about undefined
+        // T_* constants.
+        $tokensToPolyfill = [
+            // PHP 7.4 native tokens.
+            'T_BAD_CHARACTER',
+            'T_COALESCE_EQUAL',
+            'T_FN',
+
+            // PHP 8.0 native tokens.
+            'T_ATTRIBUTE',
+            'T_MATCH',
+            'T_NAME_FULLY_QUALIFIED',
+            'T_NAME_QUALIFIED',
+            'T_NAME_RELATIVE',
+            'T_NULLSAFE_OBJECT_OPERATOR',
+
+            // PHP 8.1 native tokens.
+            'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG',
+            'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG',
+            'T_ENUM',
+            'T_READONLY',
+
+            // PHP 8.4 native tokens.
+            'T_PRIVATE_SET',
+            'T_PROTECTED_SET',
+            'T_PUBLIC_SET',
+
+            // PHP 8.5 native tokens.
+            'T_VOID_CAST',
+        ];
+
+        // <https://www.php.net/manual/en/tokens.php>
+        // The PHP manual suggests "using big numbers like 10000" for
+        // polyfilled T_* constants. We have arbitrarily chosen to start our
+        // numbering scheme from 135_000.
+        $nextTokenNumber = 135000;
+
+        // This variable is necessary to avoid collisions with any other
+        // libraries which also polyfill T_* constants.
+        // array_flip()/isset() because in_array() is slow.
+        $allDefinedConstants = get_defined_constants(true);
+        $existingConstants   = array_flip($allDefinedConstants['tokenizer']);
+        foreach (($allDefinedConstants['user'] ?? []) as $k => $v) {
+            if (isset($k[2]) === false || $k[0] !== 'T' || $k[1] !== '_') {
+                // We only care about T_* constants.
+                continue;
+            }
+
+            if (isset($existingConstants[$v]) === true) {
+                throw new \Exception("Externally polyfilled tokenizer constant value collision detected! $k has the same value as {$existingConstants[$v]}");
+            }
+
+            $existingConstants[$v] = $k;
+        }
+
+        $polyfillMappingTable = [];
+
+        foreach ($tokensToPolyfill as $tokenName) {
+            if (isset($allDefinedConstants['tokenizer'][$tokenName]) === true) {
+                // This is a PHP native token, which is already defined by PHP.
+                continue;
+            }
+
+            if (defined($tokenName) === false) {
+                while (isset($existingConstants[$nextTokenNumber]) === true) {
+                    $nextTokenNumber++;
+                }
+
+                define($tokenName, $nextTokenNumber);
+                $existingConstants[$nextTokenNumber] = $tokenName;
+            }
+
+            $polyfillMappingTable[constant($tokenName)] = $tokenName;
+        }
+
+        // Be careful to not reference this class anywhere in this method until
+        // *after* all constants have been polyfilled.
+        self::$polyfillMappingTable = $polyfillMappingTable;
+    }
 }

tests/Core/Util/Tokens/TokenNameTest.php

@@ -16,6 +16,7 @@
  * Tests for the \PHP_CodeSniffer\Util\Tokens::tokenName() method.
  *
  * @covers \PHP_CodeSniffer\Util\Tokens::tokenName
+ * @covers \PHP_CodeSniffer\Util\Tokens::polyfillTokenizerConstants
  */
 final class TokenNameTest extends TestCase
 {

tests/EndToEndPhpt/Util/Tokens/polyfillTokenizerConstants-collision-php.phpt

@@ -0,0 +1,18 @@
+--TEST--
+Detect when the value of a polyfilled PHP token collides with a value already used by an existing internal PHP token.
+--SKIPIF--
+<?php
+if (version_compare(PHP_VERSION, '8.4', '>=') === true) {
+    echo 'skip because tokens used in this test already exist in PHP 8.4 so we cannot test polyfilling them', PHP_EOL;
+}
+--FILE--
+<?php
+define('T_PUBLIC_SET', T_STRING);
+require 'src/Util/Tokens.php';
+--EXPECTF--
+Fatal error: Uncaught Exception: Externally polyfilled tokenizer constant value collision detected! T_PUBLIC_SET has the same value as T_STRING in %s:%d
+Stack trace:
+#0 %s(%d): PHP_CodeSniffer\Util\Tokens::polyfillTokenizerConstants()
+#1 %s(%d): require('%s')
+#2 {main}
+  thrown in %s on line %d