Repeated test execution using --repeat CLI option and #[Repeat] attribute

.php-cs-fixer.dist.php

@@ -24,6 +24,9 @@
     ->notName('TestAttributeOnHookMethodsTest.php')
     // InvokableConstraintAndPipeOperatorTest.php uses PHP 8.5 syntax
     ->notName('InvokableConstraintAndPipeOperatorTest.php')
+    // NoReturnTypeTest.php and RepeatAttributeWithoutReturnTypeTest.php must have a method without a return type declaration
+    ->notPath('repeat/_files/NoReturnTypeTest.php')
+    ->notPath('repeat/_files/RepeatAttributeWithoutReturnTypeTest.php')
     // FirstPartyClass.php and PhpDeprecationTest.php must not use declare(strict_types=1);
     ->notPath('error-handler/_files/php-deprecation/src/FirstPartyClass.php')
     ->notPath('error-handler/_files/php-deprecation/tests/PhpDeprecationTest.php')

phpunit.xml

@@ -37,6 +37,7 @@
             <directory suffix=".phpt">tests/end-to-end/migration</directory>
             <directory suffix=".phpt">tests/end-to-end/phpt</directory>
             <directory suffix=".phpt">tests/end-to-end/regression</directory>
+            <directory suffix=".phpt">tests/end-to-end/repeat</directory>
             <directory suffix=".phpt">tests/end-to-end/test-doubles/generator</directory>
             <directory suffix=".phpt">tests/end-to-end/test-doubles/runtime</directory>
             <directory suffix=".phpt">tests/end-to-end/testdox</directory>
@@ -46,6 +47,7 @@
             <exclude>tests/end-to-end/groups-from-configuration/_files</exclude>
             <exclude>tests/end-to-end/logging/_files</exclude>
             <exclude>tests/end-to-end/migration/_files</exclude>
+            <exclude>tests/end-to-end/repeat/_files</exclude>
             <exclude>tests/end-to-end/self-direct-indirect/_files</exclude>
             <exclude>tests/end-to-end/testdox/_files</exclude>
         </testsuite>

src/Event/Events/TestSuite/Finished.php

@@ -47,11 +47,31 @@ public function testSuite(): TestSuite
      */
     public function asString(): string
     {
+        $prefix = 'Test Suite Finished';
+        $unit   = 'test';
+
+        if ($this->testSuite->isForTestMethodWithDataProvider()) {
+            $prefix = 'Test Suite for Test Method with Data Provider Finished';
+            $unit   = 'data set';
+        } elseif ($this->testSuite->isForRepeatedTestMethod()) {
+            $prefix = 'Test Suite for Repeated Test Method Finished';
+            $unit   = 'repetition';
+        }
+
+        $count  = $this->testSuite->count();
+        $plural = '';
+
+        if ($count !== 1) {
+            $plural = 's';
+        }
+
         return sprintf(
-            'Test Suite Finished (%s, %d test%s)',
+            '%s (%s, %d %s%s)',
+            $prefix,
             $this->testSuite->name(),
-            $this->testSuite->count(),
-            $this->testSuite->count() !== 1 ? 's' : '',
+            $count,
+            $unit,
+            $plural,
         );
     }
 }

src/Event/Events/TestSuite/Started.php

@@ -47,11 +47,31 @@ public function testSuite(): TestSuite
      */
     public function asString(): string
     {
+        $prefix = 'Test Suite Started';
+        $unit   = 'test';
+
+        if ($this->testSuite->isForTestMethodWithDataProvider()) {
+            $prefix = 'Test Suite for Test Method with Data Provider Started';
+            $unit   = 'data set';
+        } elseif ($this->testSuite->isForRepeatedTestMethod()) {
+            $prefix = 'Test Suite for Repeated Test Method Started';
+            $unit   = 'repetition';
+        }
+
+        $count  = $this->testSuite->count();
+        $plural = '';
+
+        if ($count !== 1) {
+            $plural = 's';
+        }
+
         return sprintf(
-            'Test Suite Started (%s, %d test%s)',
+            '%s (%s, %d %s%s)',
+            $prefix,
             $this->testSuite->name(),
-            $this->testSuite->count(),
-            $this->testSuite->count() !== 1 ? 's' : '',
+            $count,
+            $unit,
+            $plural,
         );
     }
 }

src/Event/Value/Test/TestMethod.php

@@ -39,24 +39,38 @@
     private MetadataCollection $metadata;
     private TestDataCollection $testData;
 
+    /**
+     * @var positive-int
+     */
+    private int $repetition;
+
+    /**
+     * @var positive-int
+     */
+    private int $totalRepetitions;
+
     /**
      * @param class-string     $className
      * @param non-empty-string $methodName
      * @param non-empty-string $file
      * @param non-negative-int $line
+     * @param positive-int     $repetition
+     * @param positive-int     $totalRepetitions
      *
      * @internal This method is not covered by the backward compatibility promise for PHPUnit
      */
-    public function __construct(string $className, string $methodName, string $file, int $line, TestDox $testDox, MetadataCollection $metadata, TestDataCollection $testData)
+    public function __construct(string $className, string $methodName, string $file, int $line, TestDox $testDox, MetadataCollection $metadata, TestDataCollection $testData, int $repetition = 1, int $totalRepetitions = 1)
     {
         parent::__construct($file);
 
-        $this->className  = $className;
-        $this->methodName = $methodName;
-        $this->line       = $line;
-        $this->testDox    = $testDox;
-        $this->metadata   = $metadata;
-        $this->testData   = $testData;
+        $this->className        = $className;
+        $this->methodName       = $methodName;
+        $this->line             = $line;
+        $this->testDox          = $testDox;
+        $this->metadata         = $metadata;
+        $this->testData         = $testData;
+        $this->repetition       = $repetition;
+        $this->totalRepetitions = $totalRepetitions;
     }
 
     /**
@@ -98,6 +112,27 @@ public function testData(): TestDataCollection
         return $this->testData;
     }
 
+    /**
+     * @return positive-int
+     */
+    public function repetition(): int
+    {
+        return $this->repetition;
+    }
+
+    /**
+     * @return positive-int
+     */
+    public function totalRepetitions(): int
+    {
+        return $this->totalRepetitions;
+    }
+
+    public function isRepeated(): bool
+    {
+        return $this->totalRepetitions > 1;
+    }
+
     public function isTestMethod(): true
     {
         return true;
@@ -114,6 +149,14 @@ public function id(): string
             $buffer .= '#' . $this->testData->dataFromDataProvider()->dataSetName();
         }
 
+        if ($this->totalRepetitions > 1) {
+            $buffer .= sprintf(
+                ' (repetition %d of %d)',
+                $this->repetition,
+                $this->totalRepetitions,
+            );
+        }
+
         return $buffer;
     }
 
@@ -130,24 +173,32 @@ public function nameWithClass(): string
      */
     public function name(): string
     {
-        if (!$this->testData->hasDataFromDataProvider()) {
-            return $this->methodName;
+        $name = $this->methodName;
+
+        if ($this->testData->hasDataFromDataProvider()) {
+            $dataSetName = $this->testData->dataFromDataProvider()->dataSetName();
+
+            if (is_int($dataSetName)) {
+                $name .= sprintf(
+                    ' with data set #%d',
+                    $dataSetName,
+                );
+            } else {
+                $name .= sprintf(
+                    ' with data set "%s"',
+                    $dataSetName,
+                );
+            }
         }
 
-        $dataSetName = $this->testData->dataFromDataProvider()->dataSetName();
-
-        if (is_int($dataSetName)) {
-            $dataSetName = sprintf(
-                ' with data set #%d',
-                $dataSetName,
-            );
-        } else {
-            $dataSetName = sprintf(
-                ' with data set "%s"',
-                $dataSetName,
+        if ($this->totalRepetitions > 1) {
+            $name .= sprintf(
+                ' (repetition %d of %d)',
+                $this->repetition,
+                $this->totalRepetitions,
             );
         }
 
-        return $this->methodName . $dataSetName;
+        return $name;
     }
 }

src/Event/Value/Test/TestMethodBuilder.php

@@ -45,6 +45,8 @@ public static function fromTestCase(TestCase $testCase, bool $useTestCaseForTest
             $testDox,
             MetadataRegistry::parser()->forClassAndMethod($testCase::class, $methodName),
             self::dataFor($testCase),
+            $testCase->repetition(),
+            $testCase->totalRepetitions(),
         );
     }
 

src/Event/Value/TestSuite/TestSuite.php

@@ -78,4 +78,12 @@ public function isForTestMethodWithDataProvider(): bool
     {
         return false;
     }
+
+    /**
+     * @phpstan-assert-if-true TestSuiteForRepeatedTestMethod $this
+     */
+    public function isForRepeatedTestMethod(): bool
+    {
+        return false;
+    }
 }

src/Event/Value/TestSuite/TestSuiteBuilder.php

@@ -14,10 +14,13 @@
 use function count;
 use function explode;
 use function method_exists;
+use function strpos;
+use function substr;
 use PHPUnit\Event\Code\Test;
 use PHPUnit\Event\Code\TestCollection;
 use PHPUnit\Event\RuntimeException;
 use PHPUnit\Framework\DataProviderTestSuite;
+use PHPUnit\Framework\RepeatTestSuite;
 use PHPUnit\Framework\TestCase;
 use PHPUnit\Framework\TestSuite as FrameworkTestSuite;
 use PHPUnit\Runner\Phpt\TestCase as PhptTestCase;
@@ -66,6 +69,44 @@ public static function from(FrameworkTestSuite $testSuite): TestSuite
             );
         }
 
+        if ($testSuite instanceof RepeatTestSuite) {
+            $name = $testSuite->name();
+
+            $separatorPosition = strpos($name, '::');
+
+            assert($separatorPosition !== false);
+
+            $className  = substr($name, 0, $separatorPosition);
+            $methodName = substr($name, $separatorPosition + 2);
+
+            $hashPosition = strpos($methodName, '#');
+
+            if ($hashPosition !== false) {
+                $methodName = substr($methodName, 0, $hashPosition);
+            }
+
+            assert($className !== '' && class_exists($className));
+            assert($methodName !== '' && method_exists($className, $methodName));
+
+            $reflector = new ReflectionMethod($className, $methodName);
+
+            $file = $reflector->getFileName();
+            $line = $reflector->getStartLine();
+
+            assert($file !== false);
+            assert($line !== false);
+
+            return new TestSuiteForRepeatedTestMethod(
+                $name,
+                $testSuite->count(),
+                TestCollection::fromArray($tests),
+                $className,
+                $methodName,
+                $file,
+                $line,
+            );
+        }
+
         if ($testSuite->isForTestClass()) {
             $testClassName = $testSuite->name();
 

src/Event/Value/TestSuite/TestSuiteForRepeatedTestMethod.php

@@ -0,0 +1,78 @@
+<?php declare(strict_types=1);
+/*
+ * This file is part of PHPUnit.
+ *
+ * (c) Sebastian Bergmann <sebastian@phpunit.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace PHPUnit\Event\TestSuite;
+
+use PHPUnit\Event\Code\TestCollection;
+
+/**
+ * @immutable
+ *
+ * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit
+ */
+final readonly class TestSuiteForRepeatedTestMethod extends TestSuite
+{
+    /**
+     * @var class-string
+     */
+    private string $className;
+
+    /**
+     * @var non-empty-string
+     */
+    private string $methodName;
+    private string $file;
+    private int $line;
+
+    /**
+     * @param non-empty-string $name
+     * @param class-string     $className
+     * @param non-empty-string $methodName
+     */
+    public function __construct(string $name, int $size, TestCollection $tests, string $className, string $methodName, string $file, int $line)
+    {
+        parent::__construct($name, $size, $tests);
+
+        $this->className  = $className;
+        $this->methodName = $methodName;
+        $this->file       = $file;
+        $this->line       = $line;
+    }
+
+    /**
+     * @return class-string
+     */
+    public function className(): string
+    {
+        return $this->className;
+    }
+
+    /**
+     * @return non-empty-string
+     */
+    public function methodName(): string
+    {
+        return $this->methodName;
+    }
+
+    public function file(): string
+    {
+        return $this->file;
+    }
+
+    public function line(): int
+    {
+        return $this->line;
+    }
+
+    public function isForRepeatedTestMethod(): true
+    {
+        return true;
+    }
+}