e2e-tests.yml

# https://help.github.com/en/categories/automating-your-workflow-with-github-actions

name: "E2E Tests"

on:
  pull_request:
    paths-ignore:
      - 'compiler/**'
      - 'apigen/**'
      - 'changelog-generator/**'
      - 'issue-bot/**'
  push:
    branches:
      - "2.2.x"
    paths-ignore:
      - 'compiler/**'
      - 'apigen/**'
      - 'changelog-generator/**'
      - 'issue-bot/**'

concurrency:
  group: e2e-${{ github.head_ref || github.run_id }} # will be canceled on subsequent pushes in pull requests but not branches
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  result-cache-e2e-tests:
    name: "Result cache E2E tests"
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        include:
          - script: |
              cd e2e/result-cache-1
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              patch -b src/Bar.php < patch-1.patch
              cat baseline-1.neon > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              mv src/Bar.php.orig src/Bar.php
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
          - script: |
              cd e2e/result-cache-2
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              patch -b src/Bar.php < patch-1.patch
              cat baseline-1.neon > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              mv src/Bar.php.orig src/Bar.php
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
          - script: |
              cd e2e/result-cache-3
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              patch -b src/Baz.php < patch-1.patch
              cat baseline-1.neon > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              mv src/Baz.php.orig src/Baz.php
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
          - script: |
              cd e2e/result-cache-4
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              patch -b src/Bar.php < patch-1.patch
              cat baseline-1.neon > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              mv src/Bar.php.orig src/Bar.php
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
          - script: |
              cd e2e/result-cache-5
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              patch -b src/Baz.php < patch-1.patch
              cat baseline-1.neon > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              mv src/Baz.php.orig src/Baz.php
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
          - script: |
              cd e2e/result-cache-6
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              patch -b src/Baz.php < patch-1.patch
              cat baseline-1.neon > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              mv src/Baz.php.orig src/Baz.php
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
          - script: |
              cd e2e/result-cache-7
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              patch -b src/Bar.php < patch-1.patch
              cat baseline-1.neon > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              mv src/Bar.php.orig src/Bar.php
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
          - script: |
              cd e2e/bug10449
              ../../bin/phpstan analyze
              git apply patch.diff
              rm phpstan-baseline.neon
              mv after-phpstan-baseline.neon phpstan-baseline.neon
              ../../bin/phpstan analyze -vvv
          - script: |
              cd e2e/bug10449b
              ../../bin/phpstan analyze
              git apply patch.diff
              rm phpstan-baseline.neon
              mv after-phpstan-baseline.neon phpstan-baseline.neon
              ../../bin/phpstan analyze -vvv
          - script: |
              cd e2e/bug-9622
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              patch -b src/Foo.php < patch-1.patch
              cat baseline-1.neon > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              mv src/Foo.php.orig src/Foo.php
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
          - script: |
              cd e2e/bug-9622-trait
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              patch -b src/Foo.php < patch-1.patch
              cat baseline-1.neon > phpstan-baseline.neon
              ../../bin/phpstan -vvv
              mv src/Foo.php.orig src/Foo.php
              echo -n > phpstan-baseline.neon
              ../../bin/phpstan -vvv
          - script: |
              cd e2e/env-parameter
              export PHPSTAN_RESULT_CACHE_PATH=/some/path
              ACTUAL=$(../../bin/phpstan dump-parameters -c phpstan.neon --json -l 9 | jq --raw-output '.resultCachePath')
              [[ "$ACTUAL" == "/some/path" ]];
          - script: |
              cd e2e/result-cache-8
              composer install
              ../../bin/phpstan
              echo -en '\n' >> build/CustomRule.php
              OUTPUT=$(../../bin/phpstan analyze 2>&1 || true)
              echo "$OUTPUT"
              ../bashunit -a contains 'Result cache might not behave correctly' "$OUTPUT"
              ../bashunit -a contains 'ResultCache8E2E\CustomRule' "$OUTPUT"
          - script: |
              cd e2e/env-int-key
              env 1=1 ../../bin/phpstan analyse test.php
          - script: |
              cd e2e/result-cache-scanned
              ../../bin/phpstan
              patch -b src/Generated/Foo.php < patch.patch
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan analyse -vv --error-format raw")
              echo "$OUTPUT"
              ../bashunit -a contains 'Result cache not used because the metadata do not match: projectConfig, scannedFiles' "$OUTPUT"
              ../bashunit -a contains 'Instantiated class ResultCacheE2EGenerated\Foo not found.' "$OUTPUT"
          - script: |
              cd e2e/result-cache-traits
              ../../bin/phpstan analyse
              patch -b src/FooTrait.php < renameFooTraitMethod.patch
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan analyse -vv --error-format raw")
              echo "$OUTPUT"
              ../bashunit -a contains 'ClassMentioningClassUsingBarTrait.php:10:Call to an undefined method ResultCacheE2ETraits\ClassUsingBarTrait::doFooTrait(). [identifier=method.notFound]' "$OUTPUT"
          - script: |
              cd e2e/editor-mode
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan analyse -vv --error-format raw")
              echo "$OUTPUT"
              ../bashunit -a contains 'Bar.php:10:Parameter #1 $s of method EditorModeE2E\Bar::requireString() expects string, int given. [identifier=argument.type]' "$OUTPUT"
              ../bashunit -a contains 'Foo.php:10:Method EditorModeE2E\Foo::doFoo() should return int but returns string. [identifier=return.type]' "$OUTPUT"
              ../bashunit -a contains 'Result cache is saved.' "$OUTPUT"

              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan analyse -vv --error-format raw --tmp-file differentFoo.php --instead-of src/Foo.php")
              echo "$OUTPUT"
              ../bashunit -a contains 'Foo.php:10:Method EditorModeE2E\Foo::doFoo() should return float but returns string. [identifier=return.type]' "$OUTPUT"
              ../bashunit -a not_contains 'Foo.php:10:Method EditorModeE2E\Foo::doFoo() should return int but returns string. [identifier=return.type]' "$OUTPUT"
              ../bashunit -a not_contains 'differentFoo.php' "$OUTPUT"
              ../bashunit -a contains 'Bar.php:10:Parameter #1 $s of method EditorModeE2E\Bar::requireString() expects string, float given. [identifier=argument.type]' "$OUTPUT"
              ../bashunit -a contains 'Result cache restored. 2 files will be reanalysed.' "$OUTPUT"
              ../bashunit -a contains 'Result cache was not saved because of --tmp-file and --instead-of CLI options passed (editor mode).' "$OUTPUT"
          - script: |
              cd e2e/trait-caching
              ../../bin/phpstan analyze --no-progress --level 8 --error-format raw data/
              ../../bin/phpstan analyze --no-progress --level 8 --error-format raw data/
          - script: |
              cd e2e/trait-caching
              ../../bin/phpstan analyze --no-progress --level 8 --error-format raw data/
              patch -b data/TraitOne.php < TraitOne.patch
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan analyze --no-progress --level 8 --error-format raw data/")
              echo "$OUTPUT"
              ../bashunit -a line_count 2 "$OUTPUT"
              ../bashunit -a matches "Note: Using configuration file .+phpstan.neon." "$OUTPUT"
              ../bashunit -a contains 'Method TraitsCachingIssue\TestClassUsingTrait::doBar() should return stdClass but returns Exception.' "$OUTPUT"
          - script: |
              cd e2e/trait-caching
              ../../bin/phpstan analyze --no-progress --level 8 --error-format raw data/
              patch -b data/TraitTwo.php < TraitTwo.patch
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan analyze --no-progress --level 8 --error-format raw data/")
              echo "$OUTPUT"
              ../bashunit -a line_count 2 "$OUTPUT"
              ../bashunit -a matches "Note: Using configuration file .+phpstan.neon." "$OUTPUT"
              ../bashunit -a contains 'Method class@anonymous/TestClassUsingTrait.php:20::doBar() should return stdClass but returns Exception.' "$OUTPUT"
          - script: |
              cd e2e/trait-caching
              ../../bin/phpstan analyze --no-progress --level 8 --error-format raw data/
              patch -b data/TraitOne.php < TraitOne.patch
              patch -b data/TraitTwo.php < TraitTwo.patch
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan analyze --no-progress --level 8 --error-format raw data/")
              echo "$OUTPUT"
              ../bashunit -a line_count 3 "$OUTPUT"
              ../bashunit -a matches "Note: Using configuration file .+phpstan.neon." "$OUTPUT"
              ../bashunit -a contains 'Method TraitsCachingIssue\TestClassUsingTrait::doBar() should return stdClass but returns Exception.' "$OUTPUT"
              ../bashunit -a contains 'Method class@anonymous/TestClassUsingTrait.php:20::doBar() should return stdClass but returns Exception.' "$OUTPUT"
          - script: |
              cd e2e/bad-exclude-paths
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan analyse -c ignore.neon")
              echo "$OUTPUT"
              ../bashunit -a contains 'Invalid entry in ignoreErrors' "$OUTPUT"
              ../bashunit -a contains 'tests" is neither a directory, nor a file path, nor a fnmatch pattern.' "$OUTPUT"
          - script: |
              cd e2e/bad-exclude-paths
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan analyse -c phpneon.php")
              echo "$OUTPUT"
              ../bashunit -a contains 'Invalid entry in ignoreErrors' "$OUTPUT"
              ../bashunit -a contains '"src/test.php" is neither a directory, nor a file path, nor a fnmatch pattern.' "$OUTPUT"
          - script: |
              cd e2e/bad-exclude-paths
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan analyse -c excludePaths.neon")
              echo "$OUTPUT"
              ../bashunit -a contains 'Invalid entry in excludePaths' "$OUTPUT"
              ../bashunit -a contains 'tests" is neither a directory, nor a file path, nor a fnmatch pattern.' "$OUTPUT"
          - script: |
              cd e2e/bad-exclude-paths
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan analyse -c phpneon2.php")
              echo "$OUTPUT"
              ../bashunit -a contains 'Invalid entry in excludePaths' "$OUTPUT"
              ../bashunit -a contains '"src/test.php" is neither a directory, nor a file path, nor a fnmatch pattern.' "$OUTPUT"
          - script: |
              cd e2e/bad-exclude-paths
              OUTPUT=$(../../bin/phpstan analyse -c ignoreNonexistentExcludePath.neon)
              echo "$OUTPUT"
          - script: |
              cd e2e/bad-exclude-paths
              cp -r tmp-node-modules node_modules
              OUTPUT=$(../../bin/phpstan analyse -c ignoreNonexistentExcludePath.neon)
              echo "$OUTPUT"
          - script: |
              cd e2e/bad-exclude-paths
              OUTPUT=$(../../bin/phpstan analyse -c ignoreReportUnmatchedFalse.neon)
              echo "$OUTPUT"
          - script: |
              cd e2e/bug-11826
              composer install
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan")
              echo "$OUTPUT"
              ../bashunit -a contains 'Child process error (exit code 255): PHP Fatal error' "$OUTPUT"
              ../bashunit -a contains 'Result is incomplete because of severe errors.' "$OUTPUT"
          - script: |
              cd e2e/bug-11857
              composer install
              ../../bin/phpstan
          - script: |
              cd e2e/in-trait
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan --error-format=raw")
              ../bashunit -a contains 'FooTrait.php:10:Strict comparison using === between int<0, max> and false will always evaluate to false.' "$OUTPUT"
              ../bashunit -a contains 'FooTrait.php (in context of class E2EInTrait\Bar):18:Strict comparison using === between E2EInTrait\Bar and null will always evaluate to false.' "$OUTPUT"
              ../bashunit -a contains 'FooTrait.php (in context of class E2EInTrait\Foo):18:Strict comparison using === between E2EInTrait\Foo and null will always evaluate to false.' "$OUTPUT"
          - script: |
              cd e2e/result-cache-meta-extension
              composer install
              ../../bin/phpstan -vvv
              ../../bin/phpstan -vvv --fail-without-result-cache
              echo 'modified-hash' > hash.txt
              OUTPUT=$(../bashunit -a exit_code "2" "../../bin/phpstan -vvv --fail-without-result-cache")
              echo "$OUTPUT"
              ../bashunit -a matches "Note: Using configuration file .+phpstan.neon." "$OUTPUT"
              ../bashunit -a contains 'Result cache not used because the metadata do not match: metaExtensions' "$OUTPUT"
          - script: |
              cd e2e/bug-12606
              export CONFIGTEST=test
              ../../bin/phpstan
          - script: |
              cd e2e/ignore-error-extension
              composer install
              ../../bin/phpstan
          - script: |
              cd e2e/bug-10483
              ../../bin/phpstan
          - script: |
              cd e2e/stub-files-errors
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan")
              echo "$OUTPUT"
              ../bashunit -a contains 'Method StubFiles\Foo::doFoo() has no return type specified.' "$OUTPUT"
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan")
              echo "$OUTPUT"
              ../bashunit -a contains 'Method StubFiles\Foo::doFoo() has no return type specified.' "$OUTPUT"
          - script: |
              cd e2e/nested-trait-use
              ../../bin/phpstan analyze
          - script: |
              cd e2e/report-ignores-without-comments
              OUTPUT=$(../bashunit -a exit_code "1" "../../bin/phpstan --error-format=raw")
              echo "$OUTPUT"
              ../bashunit -a contains 'test.php:4:Ignore with identifier variable.undefined has no comment.' "$OUTPUT"
              ../bashunit -a contains 'test.php:4:Variable $undefined might not be defined.' "$OUTPUT"
              ../bashunit -a not_contains 'test.php:7' "$OUTPUT"
              ../bashunit -a contains 'test.php:10:Ignoring all errors on a line is not allowed.' "$OUTPUT"
              ../bashunit -a contains 'test.php:10:Variable $yetAnotherUndefined might not be defined.' "$OUTPUT"
              ../bashunit -a contains 'test.php:12:Ignoring all errors on a line is not allowed.' "$OUTPUT"
              ../bashunit -a contains 'test.php:12:Variable $yetYetAnotherUndefined might not be defined.' "$OUTPUT"

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2
        with:
          egress-policy: audit

      - name: "Checkout"
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1

      - name: "Install PHP"
        uses: "shivammathur/setup-php@44454db4f0199b8b9685a5d763dc37cbf79108e1" # v2
        with:
          coverage: "none"
          php-version: "8.2"
          extensions: mbstring
          ini-values: memory_limit=256M

      - uses: "ramsey/composer-install@3cf229dc2919194e9e36783941438d17239e8520" # v3

      - name: "Patch PHPStan"
        run: "patch src/Analyser/Error.php < e2e/PHPStanErrorPatch.patch"

      - name: "Install bashunit"
        run: "curl -s https://bashunit.typeddevs.com/install.sh | bash -s e2e/ 0.22.0"

      - name: "Test"
        run: "${{ matrix.script }}"

  e2e-tests:
    name: "E2E tests"
    runs-on: "ubuntu-latest"
    timeout-minutes: 60

    strategy:
      fail-fast: false
      matrix:
        include:
          - script: "bin/phpstan analyse -l 8 -a tests/e2e/data/timecop.php -c tests/e2e/data/empty.neon tests/e2e/data/timecop.php"
            tools: "pecl"
            extensions: "timecop-beta"
          - script: "bin/phpstan analyse -l 8 -a tests/e2e/data/soap.php -c tests/e2e/data/empty.neon tests/e2e/data/soap.php"
            extensions: "soap"
          - script: "bin/phpstan analyse -l 8 -a tests/e2e/data/soap.php -c tests/e2e/data/empty.neon tests/e2e/data/soap.php"
            extensions: ""
          - script: "bin/phpstan analyse -l 8 -c tests/e2e/data/empty.neon tests/e2e/data/curl_setopt.php"
            extensions: ":curl"
          - script: "bin/phpstan analyse -l 8 tests/e2e/anon-class/Granularity.php"
            extensions: ""
          - script: "bin/phpstan analyse -l 8 e2e/phpstan-phpunit-190/test.php -c e2e/phpstan-phpunit-190/test.neon"
            extensions: ""
          - script: "bin/phpstan analyse e2e/only-files-not-analysed-trait/src -c e2e/only-files-not-analysed-trait/ignore.neon"
            extensions: ""
          - script: "bin/phpstan analyse e2e/only-files-not-analysed-trait/src/Foo.php e2e/only-files-not-analysed-trait/src/BarTrait.php -c e2e/only-files-not-analysed-trait/no-ignore.neon"
            extensions: ""
          - script: |
              cd e2e/baseline-uninit-prop-trait
              ../../bin/phpstan analyse --debug --configuration test-no-baseline.neon --generate-baseline test-baseline.neon
              ../../bin/phpstan analyse --debug --configuration test.neon
          - script: |
              cd e2e/baseline-uninit-prop-trait
              ../../bin/phpstan analyse --configuration test-no-baseline.neon --generate-baseline test-baseline.neon
              ../../bin/phpstan analyse --configuration test.neon
          - script: |
              cd e2e/discussion-11362
              composer install
              ../../bin/phpstan
          - script: |
              cd e2e/bug-11819
              ../../bin/phpstan
          - script: |
              cd e2e/bug-14305
              ../../bin/phpstan
          - script: |
              cd e2e/composer-and-phpstan-version-config
              composer install --ignore-platform-reqs
              ../../bin/phpstan analyze test.php --level=0
          - script: |
              cd e2e/composer-max-version
              composer install
              ../../bin/phpstan analyze test.php --level=0
          - script: |
              cd e2e/composer-min-max-version
              composer install
              ../../bin/phpstan analyze test.php --level=0
          - script: |
              cd e2e/composer-min-open-end-version
              composer install
              ../../bin/phpstan analyze test.php --level=0
          - script: |
              cd e2e/composer-min-version-v5
              composer install --ignore-platform-reqs
              ../../bin/phpstan analyze test.php --level=0
          - script: |
              cd e2e/composer-min-version-v7
              composer install --ignore-platform-reqs
              ../../bin/phpstan analyze test.php --level=0
          - script: |
              cd e2e/composer-min-version
              composer install
              ../../bin/phpstan analyze test.php --level=0
          - script: |
              cd e2e/composer-no-versions
              composer install
              ../../bin/phpstan analyze test.php --level=0
          - script: |
              cd e2e/composer-version-config-invalid
              OUTPUT=$(../bashunit -a exit_code "1" ../../bin/phpstan)
              echo "$OUTPUT"
              ../bashunit -a contains 'Invalid configuration' "$OUTPUT"
              ../bashunit -a contains 'Invalid PHP version range: phpVersion.max should be greater or equal to phpVersion.min.' "$OUTPUT"
          - script: |
              cd e2e/composer-version-config-patch
              composer install --ignore-platform-reqs
              ../../bin/phpstan analyze test.php --level=0
          - script: |
              cd e2e/composer-version-config
              composer install
              ../../bin/phpstan analyze test.php --level=0
          - script: |
              cd e2e/bug13425
              timeout 15 ../bashunit -a exit_code "1" "../../bin/phpstan analyze src/ plugins/"
          - script: |
              cd e2e/bug-14036
              composer install
              ../../bin/phpstan analyze
          - script: |
              cd e2e/vvv-with-debug
              OUTPUT=$(../../bin/phpstan analyze -l 0 -vvv --debug test.php 2>&1)
              echo "$OUTPUT"
              ../bashunit -a not_contains 'Parallel processing scheduler' "$OUTPUT"
          - script: |
              cd e2e/vvv-with-debug
              OUTPUT=$(../../bin/phpstan analyze -l 0 -vvv --debug --configuration parallel.neon test.php 2>&1)
              echo "$OUTPUT"
              ../bashunit -a not_contains 'Parallel processing scheduler' "$OUTPUT"
          - script: |
              cd e2e/vvv-with-debug
              OUTPUT=$(../../bin/phpstan analyze -l 0 -vvv test.php 2>&1)
              echo "$OUTPUT"
              ../bashunit -a contains 'Parallel processing scheduler' "$OUTPUT"
          - script: |
              cd e2e/bug-14093
              # Use a relative sys_temp_dir so the value starts with a letter (not /), just like Windows
              # paths starting with a drive letter. If the sys_temp_dir value isn't correctly quoted,
              # PHP evaluates ~ as a bitwise NOT rather than a literal character, truncating the path.
              mkdir tmp~1
              php -d "sys_temp_dir='tmp~1'" ../../bin/phpstan analyze
              if [ -d tmp ]; then
                echo "FAIL: tmp was created by a worker, meaning sys_temp_dir='tmp~1' was incorrectly evaluated to 'tmp'"
                exit 1
              fi

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2
        with:
          egress-policy: audit

      - name: "Checkout"
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1

      - name: "Install PHP"
        uses: "shivammathur/setup-php@44454db4f0199b8b9685a5d763dc37cbf79108e1" # v2
        with:
          coverage: "none"
          php-version: "8.2"
          tools: ${{ matrix.tools }}
          extensions: ${{ matrix.extensions }}

      - uses: "ramsey/composer-install@3cf229dc2919194e9e36783941438d17239e8520" # v3

      - name: "Install bashunit"
        run: "curl -s https://bashunit.typeddevs.com/install.sh | bash -s e2e/ 0.22.0"

      - name: "Test"
        run: ${{ matrix.script }}