diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 36e467efa7..4a8d1bf072 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -399,6 +399,8 @@ To run the tests specific to the use of `PHP_CODESNIFFER_CBF === true`: ### Writing End-to-End Tests +#### Bashunit + Bash-based end-to-end tests can be written using the [Bashunit](https://bashunit.typeddevs.com/) test tooling using version 0.26.0 or higher. To install bashunit, follow the [installation guide](https://bashunit.typeddevs.com/installation). @@ -413,6 +415,20 @@ You can then run the bashunit tests on Linux/Mac/WSL, like so: When writing end-to-end tests, please use fixtures for the "files under scan" to make the tests stable. These fixtures can be placed in the `tests/EndToEndBash/Fixtures` subdirectory. +#### phpt + +PHP-based end-to-end tests can be written using the `phpt` format. This is the format that PHP uses for its own tests. We use PHPUnit to run these tests. + +```bash +vendor/bin/phpunit -c phpunit-e2e.xml.dist +``` + +The following resources may be helpful when writing `phpt`-style tests: + +* +* +* +* ### Submitting Your Pull Request diff --git a/.github/workflows/end-to-end-tests.yml b/.github/workflows/end-to-end-tests.yml index d81c5e8737..2e685e6007 100644 --- a/.github/workflows/end-to-end-tests.yml +++ b/.github/workflows/end-to-end-tests.yml @@ -15,22 +15,21 @@ on: # Allow manually triggering the workflow. workflow_dispatch: +# Cancels all previous workflow runs for the same branch that have not yet completed. +concurrency: + # The concurrency group contains the workflow name and the branch name. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: bash-tests: - # Cancels all previous runs of this particular job for the same branch that have not yet completed. - concurrency: - # The concurrency group contains the workflow name, job name, job index and the branch name. - group: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index }}-${{ github.ref }} - cancel-in-progress: true - runs-on: 'ubuntu-latest' strategy: matrix: php: ['7.2', '7.3', '7.4', '8.0', '8.1', '8.2', '8.3', '8.4', '8.5', 'nightly'] - # yamllint disable-line rule:line-length - name: "E2E PHP: ${{ matrix.php }}" + name: "E2E Bash PHP: ${{ matrix.php }}" continue-on-error: ${{ matrix.php == 'nightly' }} @@ -43,13 +42,6 @@ jobs: with: persist-credentials: false - - name: Install PHP - uses: shivammathur/setup-php@44454db4f0199b8b9685a5d763dc37cbf79108e1 # 2.36.0 - with: - php-version: ${{ matrix.php }} - ini-values: "error_reporting=-1, display_errors=On, display_startup_errors=On" - coverage: none - - name: "Install bashunit" shell: bash run: | @@ -60,3 +52,46 @@ jobs: - name: "Run bashunit tests" shell: bash run: "./lib/bashunit -p tests/EndToEndBash" + + phpt-tests: + runs-on: 'ubuntu-latest' + + strategy: + matrix: + php: ['7.2', '7.3', '7.4', '8.0', '8.1', '8.2', '8.3', '8.4', '8.5', 'nightly'] + + name: "E2E phpt PHP: ${{ matrix.php }}" + + continue-on-error: ${{ matrix.php == 'nightly' }} + + steps: + - name: Prepare git to leave line endings alone + run: git config --global core.autocrlf input + + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install PHP + uses: shivammathur/setup-php@44454db4f0199b8b9685a5d763dc37cbf79108e1 # 2.36.0 + with: + php-version: ${{ matrix.php }} + ini-values: "error_reporting=-1, display_errors=On, display_startup_errors=On" + coverage: none + + # Install dependencies and handle caching in one go. + # @link https://github.com/marketplace/actions/install-php-dependencies-with-composer + - name: Install Composer dependencies + uses: "ramsey/composer-install@3cf229dc2919194e9e36783941438d17239e8520" # 3.1.1 + with: + composer-options: ${{ matrix.php == 'nightly' && '--ignore-platform-req=php+' || '' }} + custom-cache-suffix: $(date -u "+%Y-%m") + + - name: 'PHPUnit: run end-to-end tests (phpt)' + # We have included '--no-coverage' here to make sure that we do not run with coverage enabled + # by mistake, if the Xdebug extension gets loaded in future. These tests are not compatible + # with how PHPUnit runs when coverage is enabled. This is because PHPUnit pre-loads all PHP + # files as part of its coverage initialisation process, and these tests specifically test the + # code which is used during the PHP_CodeSniffer bootstrap step. + run: php "vendor/bin/phpunit" -c phpunit-e2e.xml.dist --no-coverage diff --git a/phpunit-e2e.xml.dist b/phpunit-e2e.xml.dist new file mode 100644 index 0000000000..2e989c9084 --- /dev/null +++ b/phpunit-e2e.xml.dist @@ -0,0 +1,14 @@ + + + + + tests/EndToEndPhpt/ + + + diff --git a/src/Util/Tokens.php b/src/Util/Tokens.php index 62c918c31b..f7ca28a94b 100644 --- a/src/Util/Tokens.php +++ b/src/Util/Tokens.php @@ -10,6 +10,7 @@ namespace PHP_CodeSniffer\Util; +// PHPCS native tokens. define('T_NONE', 'PHPCS_T_NONE'); define('T_OPEN_CURLY_BRACKET', 'PHPCS_T_OPEN_CURLY_BRACKET'); define('T_CLOSE_CURLY_BRACKET', 'PHPCS_T_CLOSE_CURLY_BRACKET'); @@ -70,84 +71,6 @@ define('T_TYPE_OPEN_PARENTHESIS', 'PHPCS_T_TYPE_OPEN_PARENTHESIS'); define('T_TYPE_CLOSE_PARENTHESIS', 'PHPCS_T_TYPE_CLOSE_PARENTHESIS'); -/* - * {@internal IMPORTANT: all PHP native polyfilled tokens MUST be added to the - * `PHP_CodeSniffer\Tests\Core\Util\Tokens\TokenNameTest::dataPolyfilledPHPNativeTokens()` test method!} - */ - -// Some PHP 7.4 tokens, replicated for lower versions. -if (defined('T_COALESCE_EQUAL') === false) { - define('T_COALESCE_EQUAL', 'PHPCS_T_COALESCE_EQUAL'); -} - -if (defined('T_BAD_CHARACTER') === false) { - define('T_BAD_CHARACTER', 'PHPCS_T_BAD_CHARACTER'); -} - -if (defined('T_FN') === false) { - define('T_FN', 'PHPCS_T_FN'); -} - -// Some PHP 8.0 tokens, replicated for lower versions. -if (defined('T_NULLSAFE_OBJECT_OPERATOR') === false) { - define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR'); -} - -if (defined('T_NAME_QUALIFIED') === false) { - define('T_NAME_QUALIFIED', 'PHPCS_T_NAME_QUALIFIED'); -} - -if (defined('T_NAME_FULLY_QUALIFIED') === false) { - define('T_NAME_FULLY_QUALIFIED', 'PHPCS_T_NAME_FULLY_QUALIFIED'); -} - -if (defined('T_NAME_RELATIVE') === false) { - define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE'); -} - -if (defined('T_MATCH') === false) { - define('T_MATCH', 'PHPCS_T_MATCH'); -} - -if (defined('T_ATTRIBUTE') === false) { - define('T_ATTRIBUTE', 'PHPCS_T_ATTRIBUTE'); -} - -// Some PHP 8.1 tokens, replicated for lower versions. -if (defined('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG') === false) { - define('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG', 'PHPCS_T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG'); -} - -if (defined('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG') === false) { - define('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG', 'PHPCS_T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG'); -} - -if (defined('T_READONLY') === false) { - define('T_READONLY', 'PHPCS_T_READONLY'); -} - -if (defined('T_ENUM') === false) { - define('T_ENUM', 'PHPCS_T_ENUM'); -} - -// Some PHP 8.4 tokens, replicated for lower versions. -if (defined('T_PUBLIC_SET') === false) { - define('T_PUBLIC_SET', 'PHPCS_T_PUBLIC_SET'); -} - -if (defined('T_PROTECTED_SET') === false) { - define('T_PROTECTED_SET', 'PHPCS_T_PROTECTED_SET'); -} - -if (defined('T_PRIVATE_SET') === false) { - define('T_PRIVATE_SET', 'PHPCS_T_PRIVATE_SET'); -} - -// Some PHP 8.5 tokens, replicated for lower versions. -if (defined('T_VOID_CAST') === false) { - define('T_VOID_CAST', 'PHPCS_T_VOID_CAST'); -} - // Tokens used for parsing doc blocks. define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR'); define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE'); @@ -163,6 +86,8 @@ define('T_PHPCS_IGNORE', 'PHPCS_T_PHPCS_IGNORE'); define('T_PHPCS_IGNORE_FILE', 'PHPCS_T_PHPCS_IGNORE_FILE'); +Tokens::polyfillTokenizerConstants(); + final class Tokens { @@ -612,6 +537,13 @@ final class Tokens T_YIELD_FROM => T_YIELD_FROM, ]; + /** + * Mapping table for polyfilled constants. + * + * @var array + */ + private static $polyfillMappingTable = []; + /** * The token weightings. * @@ -943,12 +875,12 @@ final class Tokens */ public static function tokenName($token) { - if (is_string($token) === false) { - // PHP-supplied token name. - return token_name($token); + if (is_string($token) === true) { + // PHPCS native token. + return substr($token, 6); } - return substr($token, 6); + return (self::$polyfillMappingTable[$token] ?? token_name($token)); } @@ -991,4 +923,99 @@ public static function getHighestWeightedToken(array $tokens) return $highestType; } + + + /** + * Polyfill tokenizer (T_*) constants. + * + * {@internal IMPORTANT: all PHP native polyfilled tokens MUST be added to the + * `PHP_CodeSniffer\Tests\Core\Util\Tokens\TokenNameTest::dataPolyfilledPHPNativeTokens()` test method!} + * + * @return void + */ + public static function polyfillTokenizerConstants(): void + { + // Ideally this would be a private class constant. We cannot do that + // here as the constants that we are polyfilling in this method are + // used in some of the class constants for this class. If we reference + // any class constants or properties before this method has fully run, + // PHP will intitialise the class, leading to warnings about undefined + // T_* constants. + $tokensToPolyfill = [ + // PHP 7.4 native tokens. + 'T_BAD_CHARACTER', + 'T_COALESCE_EQUAL', + 'T_FN', + + // PHP 8.0 native tokens. + 'T_ATTRIBUTE', + 'T_MATCH', + 'T_NAME_FULLY_QUALIFIED', + 'T_NAME_QUALIFIED', + 'T_NAME_RELATIVE', + 'T_NULLSAFE_OBJECT_OPERATOR', + + // PHP 8.1 native tokens. + 'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG', + 'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG', + 'T_ENUM', + 'T_READONLY', + + // PHP 8.4 native tokens. + 'T_PRIVATE_SET', + 'T_PROTECTED_SET', + 'T_PUBLIC_SET', + + // PHP 8.5 native tokens. + 'T_VOID_CAST', + ]; + + // + // The PHP manual suggests "using big numbers like 10000" for + // polyfilled T_* constants. We have arbitrarily chosen to start our + // numbering scheme from 135_000. + $nextTokenNumber = 135000; + + // This variable is necessary to avoid collisions with any other + // libraries which also polyfill T_* constants. + // array_flip()/isset() because in_array() is slow. + $allDefinedConstants = get_defined_constants(true); + $existingConstants = array_flip($allDefinedConstants['tokenizer']); + foreach (($allDefinedConstants['user'] ?? []) as $k => $v) { + if (isset($k[2]) === false || $k[0] !== 'T' || $k[1] !== '_') { + // We only care about T_* constants. + continue; + } + + if (isset($existingConstants[$v]) === true) { + throw new \Exception("Externally polyfilled tokenizer constant value collision detected! $k has the same value as {$existingConstants[$v]}"); + } + + $existingConstants[$v] = $k; + } + + $polyfillMappingTable = []; + + foreach ($tokensToPolyfill as $tokenName) { + if (isset($allDefinedConstants['tokenizer'][$tokenName]) === true) { + // This is a PHP native token, which is already defined by PHP. + continue; + } + + if (defined($tokenName) === false) { + while (isset($existingConstants[$nextTokenNumber]) === true) { + $nextTokenNumber++; + } + + define($tokenName, $nextTokenNumber); + $existingConstants[$nextTokenNumber] = $tokenName; + } + + $polyfillMappingTable[constant($tokenName)] = $tokenName; + } + + // Be careful to not reference this class anywhere in this method until + // *after* all constants have been polyfilled. + self::$polyfillMappingTable = $polyfillMappingTable; + } } diff --git a/tests/Core/Util/Tokens/TokenNameTest.php b/tests/Core/Util/Tokens/TokenNameTest.php index 43198266d1..5c5689f546 100644 --- a/tests/Core/Util/Tokens/TokenNameTest.php +++ b/tests/Core/Util/Tokens/TokenNameTest.php @@ -16,6 +16,7 @@ * Tests for the \PHP_CodeSniffer\Util\Tokens::tokenName() method. * * @covers \PHP_CodeSniffer\Util\Tokens::tokenName + * @covers \PHP_CodeSniffer\Util\Tokens::polyfillTokenizerConstants */ final class TokenNameTest extends TestCase { diff --git a/tests/EndToEndPhpt/Util/Tokens/polyfillTokenizerConstants-collision-php.phpt b/tests/EndToEndPhpt/Util/Tokens/polyfillTokenizerConstants-collision-php.phpt new file mode 100644 index 0000000000..031ab29126 --- /dev/null +++ b/tests/EndToEndPhpt/Util/Tokens/polyfillTokenizerConstants-collision-php.phpt @@ -0,0 +1,18 @@ +--TEST-- +Detect when the value of a polyfilled PHP token collides with a value already used by an existing internal PHP token. +--SKIPIF-- +=') === true) { + echo 'skip because tokens used in this test already exist in PHP 8.4 so we cannot test polyfilling them', PHP_EOL; +} +--FILE-- +=') === true) { + echo 'skip because tokens used in this test already exist in PHP 8.4 so we cannot test polyfilling them', PHP_EOL; +} +--FILE-- +=') === true) { + echo 'skip because tokens used in this test already exist in PHP 8.4 so we cannot test polyfilling them', PHP_EOL; +} +--FILE-- + 135000) { + echo 'Success.', PHP_EOL; +} else { + echo 'Failure - ', T_PRIVATE_SET, PHP_EOL; +} +--EXPECT-- +Success. diff --git a/tests/EndToEndPhpt/Util/Tokens/polyfillTokenizerConstants-skip-names.phpt b/tests/EndToEndPhpt/Util/Tokens/polyfillTokenizerConstants-skip-names.phpt new file mode 100644 index 0000000000..429090a0dd --- /dev/null +++ b/tests/EndToEndPhpt/Util/Tokens/polyfillTokenizerConstants-skip-names.phpt @@ -0,0 +1,17 @@ +--TEST-- +Value collision with a non-tokenizer constant should not cause an error. +--SKIPIF-- +=') === true) { + echo 'skip because tokens used in this test already exist in PHP 8.4 so we cannot test polyfilling them', PHP_EOL; +} +--FILE-- +