Emit test runner warning when #[Repeat] attribute is used on non-eligible test method

Author: sebastianbergmann

.php-cs-fixer.dist.php

@@ -24,8 +24,9 @@
     ->notName('TestAttributeOnHookMethodsTest.php')
     // InvokableConstraintAndPipeOperatorTest.php uses PHP 8.5 syntax
     ->notName('InvokableConstraintAndPipeOperatorTest.php')
-    // NoReturnTypeTest.php must have a method without a return type declaration
+    // NoReturnTypeTest.php and RepeatAttributeWithoutReturnTypeTest.php must have a method without a return type declaration
     ->notPath('repeat/_files/NoReturnTypeTest.php')
+    ->notPath('repeat/_files/RepeatAttributeWithoutReturnTypeTest.php')
     // FirstPartyClass.php and PhpDeprecationTest.php must not use declare(strict_types=1);
     ->notPath('error-handler/_files/php-deprecation/src/FirstPartyClass.php')
     ->notPath('error-handler/_files/php-deprecation/tests/PhpDeprecationTest.php')

src/Framework/TestBuilder.php

@@ -13,6 +13,8 @@
 use function assert;
 use function get_parent_class;
 use function range;
+use function sprintf;
+use PHPUnit\Event\Facade as EventFacade;
 use PHPUnit\Metadata\Api\DataProvider;
 use PHPUnit\Metadata\Api\Groups;
 use PHPUnit\Metadata\Api\ProvidedData;
@@ -64,6 +66,26 @@ public function build(ReflectionClass $theClass, string $methodName, array $grou
 
             $numberOfRuns     = $metadata->times();
             $failureThreshold = $metadata->failureThreshold();
+
+            if (!$this->hasVoidReturnType($theClass->getMethod($methodName))) {
+                EventFacade::emitter()->testRunnerTriggeredPhpunitWarning(
+                    sprintf(
+                        'Method %s::%s is annotated with #[Repeat] but does not have a void return type declaration and will not be repeated',
+                        $className,
+                        $methodName,
+                    ),
+                );
+            }
+
+            if (!$this->doesNotDependOnAnotherTest($className, $methodName)) {
+                EventFacade::emitter()->testRunnerTriggeredPhpunitWarning(
+                    sprintf(
+                        'Method %s::%s is annotated with #[Repeat] but depends on another test and will not be repeated',
+                        $className,
+                        $methodName,
+                    ),
+                );
+            }
         }
 
         $repeat = $numberOfRuns > 1 &&

tests/end-to-end/repeat/_files/RepeatAttributeOnDependentTest.php

@@ -0,0 +1,29 @@
+<?php declare(strict_types=1);
+/*
+ * This file is part of PHPUnit.
+ *
+ * (c) Sebastian Bergmann <sebastian@phpunit.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace PHPUnit\TestFixture\Repeat;
+
+use PHPUnit\Framework\Attributes\Depends;
+use PHPUnit\Framework\Attributes\Repeat;
+use PHPUnit\Framework\TestCase;
+
+final class RepeatAttributeOnDependentTest extends TestCase
+{
+    public function testOne(): void
+    {
+        $this->assertTrue(true);
+    }
+
+    #[Depends('testOne')]
+    #[Repeat(3)]
+    public function testTwo(): void
+    {
+        $this->assertTrue(true);
+    }
+}

tests/end-to-end/repeat/_files/RepeatAttributeOnNonVoidReturnTest.php

@@ -0,0 +1,24 @@
+<?php declare(strict_types=1);
+/*
+ * This file is part of PHPUnit.
+ *
+ * (c) Sebastian Bergmann <sebastian@phpunit.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace PHPUnit\TestFixture\Repeat;
+
+use PHPUnit\Framework\Attributes\Repeat;
+use PHPUnit\Framework\TestCase;
+
+final class RepeatAttributeOnNonVoidReturnTest extends TestCase
+{
+    #[Repeat(3)]
+    public function testWithReturnValue(): int
+    {
+        $this->assertTrue(true);
+
+        return 1;
+    }
+}

tests/end-to-end/repeat/_files/RepeatAttributeWithoutReturnTypeTest.php

@@ -0,0 +1,22 @@
+<?php declare(strict_types=1);
+/*
+ * This file is part of PHPUnit.
+ *
+ * (c) Sebastian Bergmann <sebastian@phpunit.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace PHPUnit\TestFixture\Repeat;
+
+use PHPUnit\Framework\Attributes\Repeat;
+use PHPUnit\Framework\TestCase;
+
+final class RepeatAttributeWithoutReturnTypeTest extends TestCase
+{
+    #[Repeat(3)]
+    public function testWithoutReturnType()
+    {
+        $this->assertTrue(true);
+    }
+}

tests/end-to-end/repeat/repeat-attribute-warning-depends.phpt

@@ -0,0 +1,34 @@
+--TEST--
+#[Repeat] on method that depends on another test triggers a test runner warning
+--FILE--
+<?php declare(strict_types=1);
+$_SERVER['argv'][] = '--no-configuration';
+$_SERVER['argv'][] = '--do-not-cache-result';
+$_SERVER['argv'][] = '--debug';
+$_SERVER['argv'][] = __DIR__ . '/_files/RepeatAttributeOnDependentTest.php';
+
+require __DIR__ . '/../../bootstrap.php';
+
+(new PHPUnit\TextUI\Application)->run($_SERVER['argv']);
+--EXPECTF--
+PHPUnit Started (PHPUnit %s using %s)
+Test Runner Configured
+Event Facade Sealed
+Test Runner Triggered PHPUnit Warning (Method PHPUnit\TestFixture\Repeat\RepeatAttributeOnDependentTest::testTwo is annotated with #[Repeat] but depends on another test and will not be repeated)
+Test Suite Loaded (2 tests)
+Test Runner Started
+Test Suite Sorted
+Test Runner Execution Started (2 tests)
+Test Suite Started (PHPUnit\TestFixture\Repeat\RepeatAttributeOnDependentTest, 2 tests)
+Test Preparation Started (PHPUnit\TestFixture\Repeat\RepeatAttributeOnDependentTest::testOne)
+Test Prepared (PHPUnit\TestFixture\Repeat\RepeatAttributeOnDependentTest::testOne)
+Test Passed (PHPUnit\TestFixture\Repeat\RepeatAttributeOnDependentTest::testOne)
+Test Finished (PHPUnit\TestFixture\Repeat\RepeatAttributeOnDependentTest::testOne)
+Test Preparation Started (PHPUnit\TestFixture\Repeat\RepeatAttributeOnDependentTest::testTwo)
+Test Prepared (PHPUnit\TestFixture\Repeat\RepeatAttributeOnDependentTest::testTwo)
+Test Passed (PHPUnit\TestFixture\Repeat\RepeatAttributeOnDependentTest::testTwo)
+Test Finished (PHPUnit\TestFixture\Repeat\RepeatAttributeOnDependentTest::testTwo)
+Test Suite Finished (PHPUnit\TestFixture\Repeat\RepeatAttributeOnDependentTest, 2 tests)
+Test Runner Execution Finished
+Test Runner Finished
+PHPUnit Finished (Shell Exit Code: 1)

tests/end-to-end/repeat/repeat-attribute-warning-no-return-type.phpt

@@ -0,0 +1,30 @@
+--TEST--
+#[Repeat] on method without return type declaration triggers a test runner warning
+--FILE--
+<?php declare(strict_types=1);
+$_SERVER['argv'][] = '--no-configuration';
+$_SERVER['argv'][] = '--do-not-cache-result';
+$_SERVER['argv'][] = '--debug';
+$_SERVER['argv'][] = __DIR__ . '/_files/RepeatAttributeWithoutReturnTypeTest.php';
+
+require __DIR__ . '/../../bootstrap.php';
+
+(new PHPUnit\TextUI\Application)->run($_SERVER['argv']);
+--EXPECTF--
+PHPUnit Started (PHPUnit %s using %s)
+Test Runner Configured
+Event Facade Sealed
+Test Runner Triggered PHPUnit Warning (Method PHPUnit\TestFixture\Repeat\RepeatAttributeWithoutReturnTypeTest::testWithoutReturnType is annotated with #[Repeat] but does not have a void return type declaration and will not be repeated)
+Test Suite Loaded (1 test)
+Test Runner Started
+Test Suite Sorted
+Test Runner Execution Started (1 test)
+Test Suite Started (PHPUnit\TestFixture\Repeat\RepeatAttributeWithoutReturnTypeTest, 1 test)
+Test Preparation Started (PHPUnit\TestFixture\Repeat\RepeatAttributeWithoutReturnTypeTest::testWithoutReturnType)
+Test Prepared (PHPUnit\TestFixture\Repeat\RepeatAttributeWithoutReturnTypeTest::testWithoutReturnType)
+Test Passed (PHPUnit\TestFixture\Repeat\RepeatAttributeWithoutReturnTypeTest::testWithoutReturnType)
+Test Finished (PHPUnit\TestFixture\Repeat\RepeatAttributeWithoutReturnTypeTest::testWithoutReturnType)
+Test Suite Finished (PHPUnit\TestFixture\Repeat\RepeatAttributeWithoutReturnTypeTest, 1 test)
+Test Runner Execution Finished
+Test Runner Finished
+PHPUnit Finished (Shell Exit Code: 1)

tests/end-to-end/repeat/repeat-attribute-warning-non-void-return.phpt

@@ -0,0 +1,30 @@
+--TEST--
+#[Repeat] on method with non-void return type triggers a test runner warning
+--FILE--
+<?php declare(strict_types=1);
+$_SERVER['argv'][] = '--no-configuration';
+$_SERVER['argv'][] = '--do-not-cache-result';
+$_SERVER['argv'][] = '--debug';
+$_SERVER['argv'][] = __DIR__ . '/_files/RepeatAttributeOnNonVoidReturnTest.php';
+
+require __DIR__ . '/../../bootstrap.php';
+
+(new PHPUnit\TextUI\Application)->run($_SERVER['argv']);
+--EXPECTF--
+PHPUnit Started (PHPUnit %s using %s)
+Test Runner Configured
+Event Facade Sealed
+Test Runner Triggered PHPUnit Warning (Method PHPUnit\TestFixture\Repeat\RepeatAttributeOnNonVoidReturnTest::testWithReturnValue is annotated with #[Repeat] but does not have a void return type declaration and will not be repeated)
+Test Suite Loaded (1 test)
+Test Runner Started
+Test Suite Sorted
+Test Runner Execution Started (1 test)
+Test Suite Started (PHPUnit\TestFixture\Repeat\RepeatAttributeOnNonVoidReturnTest, 1 test)
+Test Preparation Started (PHPUnit\TestFixture\Repeat\RepeatAttributeOnNonVoidReturnTest::testWithReturnValue)
+Test Prepared (PHPUnit\TestFixture\Repeat\RepeatAttributeOnNonVoidReturnTest::testWithReturnValue)
+Test Passed (PHPUnit\TestFixture\Repeat\RepeatAttributeOnNonVoidReturnTest::testWithReturnValue)
+Test Finished (PHPUnit\TestFixture\Repeat\RepeatAttributeOnNonVoidReturnTest::testWithReturnValue)
+Test Suite Finished (PHPUnit\TestFixture\Repeat\RepeatAttributeOnNonVoidReturnTest, 1 test)
+Test Runner Execution Finished
+Test Runner Finished
+PHPUnit Finished (Shell Exit Code: 1)