diff --git a/src/Concerns/Testable.php b/src/Concerns/Testable.php index e1c43ef5..3f7e3b77 100644 --- a/src/Concerns/Testable.php +++ b/src/Concerns/Testable.php @@ -14,6 +14,8 @@ use Pest\Support\Reflection; use Pest\Support\Shell; use Pest\TestSuite; use PHPUnit\Framework\Attributes\PostCondition; +use PHPUnit\Framework\IncompleteTest; +use PHPUnit\Framework\SkippedTest; use PHPUnit\Framework\TestCase; use ReflectionException; use ReflectionFunction; @@ -328,7 +330,80 @@ trait Testable $arguments = $this->__resolveTestArguments($args); $this->__ensureDatasetArgumentNameAndNumberMatches($arguments); - return $this->__callClosure($closure, $arguments); + $method = TestSuite::getInstance()->tests->get(self::$__filename)->getMethod($this->name()); + + if ($method->flakyTries === null) { + return $this->__callClosure($closure, $arguments); + } + + $lastException = null; + $initialProperties = get_object_vars($this); + + for ($attempt = 1; $attempt <= $method->flakyTries; $attempt++) { + try { + return $this->__callClosure($closure, $arguments); + } catch (Throwable $e) { + if ($e instanceof SkippedTest + || $e instanceof IncompleteTest + || $this->__isExpectedException($e)) { + throw $e; + } + + $lastException = $e; + + if ($attempt < $method->flakyTries) { + if ($this->__snapshotChanges !== []) { + throw $e; + } + + $this->tearDown(); + + Closure::bind(fn (): array => $this->mockObjects = [], $this, TestCase::class)(); + + foreach (array_keys(array_diff_key(get_object_vars($this), $initialProperties)) as $property) { + unset($this->{$property}); + } + + $hasOutputExpectation = Closure::bind(fn (): bool => is_string($this->outputExpectedString) || is_string($this->outputExpectedRegex), $this, TestCase::class)(); + + if ($hasOutputExpectation) { + ob_clean(); + } + + $this->setUp(); + } + } + } + + throw $lastException; + } + + /** + * Determines if the given exception matches PHPUnit's expected exception. + */ + private function __isExpectedException(Throwable $e): bool + { + $read = fn (string $property): mixed => Closure::bind(fn () => $this->{$property}, $this, TestCase::class)(); + + $expectedClass = $read('expectedException'); + + if ($expectedClass !== null) { + return $e instanceof $expectedClass; + } + + $expectedMessage = $read('expectedExceptionMessage'); + + if ($expectedMessage !== null) { + return str_contains($e->getMessage(), (string) $expectedMessage); + } + + $expectedCode = $read('expectedExceptionCode'); + + if ($expectedCode !== null) { + return $e->getCode() === $expectedCode; + } + + return false; } /** diff --git a/src/Factories/TestCaseMethodFactory.php b/src/Factories/TestCaseMethodFactory.php index 9438f837..f0a73401 100644 --- a/src/Factories/TestCaseMethodFactory.php +++ b/src/Factories/TestCaseMethodFactory.php @@ -50,6 +50,11 @@ final class TestCaseMethodFactory */ public int $repetitions = 1; + /** + * The test's number of flaky retry tries. + */ + public ?int $flakyTries = null; + /** * Determines if the test is a "todo". */ diff --git a/src/PendingCalls/TestCall.php b/src/PendingCalls/TestCall.php index 79264596..ccf9b4f9 100644 --- a/src/PendingCalls/TestCall.php +++ b/src/PendingCalls/TestCall.php @@ -412,6 +412,20 @@ final class TestCall // @phpstan-ignore-line return $this; } + /** + * Marks the test as flaky, retrying it up to the given number of times. + */ + public function flaky(int $tries = 3): self + { + if ($tries < 1) { + throw new InvalidArgumentException('The number of tries must be greater than 0.'); + } + + $this->testCaseMethod->flakyTries = $tries; + + return $this; + } + /** * Marks the test as "todo". */ diff --git a/tests/.pest/snapshots/Visual/Todo/todo.snap b/tests/.pest/snapshots/Visual/Todo/todo.snap index c50794f7..09710340 100644 --- a/tests/.pest/snapshots/Visual/Todo/todo.snap +++ b/tests/.pest/snapshots/Visual/Todo/todo.snap @@ -15,6 +15,9 @@ ↓ todo on describe → should not fail ↓ todo on describe → should run + TODO Tests\Features\Flaky - 1 todo + ↓ it does not retry todo tests + TODO Tests\Features\Todo - 29 todos ↓ something todo later ↓ something todo later chained @@ -81,6 +84,6 @@ PASS Tests\CustomTestCase\ParentTest ✓ override method - Tests: 39 todos, 3 passed (21 assertions) + Tests: 40 todos, 3 passed (21 assertions) Duration: x.xxs diff --git a/tests/.pest/snapshots/Visual/Todo/todo_in_parallel.snap b/tests/.pest/snapshots/Visual/Todo/todo_in_parallel.snap index c50794f7..09710340 100644 --- a/tests/.pest/snapshots/Visual/Todo/todo_in_parallel.snap +++ b/tests/.pest/snapshots/Visual/Todo/todo_in_parallel.snap @@ -15,6 +15,9 @@ ↓ todo on describe → should not fail ↓ todo on describe → should run + TODO Tests\Features\Flaky - 1 todo + ↓ it does not retry todo tests + TODO Tests\Features\Todo - 29 todos ↓ something todo later ↓ something todo later chained @@ -81,6 +84,6 @@ PASS Tests\CustomTestCase\ParentTest ✓ override method - Tests: 39 todos, 3 passed (21 assertions) + Tests: 40 todos, 3 passed (21 assertions) Duration: x.xxs diff --git a/tests/.pest/snapshots/Visual/Todo/todos.snap b/tests/.pest/snapshots/Visual/Todo/todos.snap index c50794f7..09710340 100644 --- a/tests/.pest/snapshots/Visual/Todo/todos.snap +++ b/tests/.pest/snapshots/Visual/Todo/todos.snap @@ -15,6 +15,9 @@ ↓ todo on describe → should not fail ↓ todo on describe → should run + TODO Tests\Features\Flaky - 1 todo + ↓ it does not retry todo tests + TODO Tests\Features\Todo - 29 todos ↓ something todo later ↓ something todo later chained @@ -81,6 +84,6 @@ PASS Tests\CustomTestCase\ParentTest ✓ override method - Tests: 39 todos, 3 passed (21 assertions) + Tests: 40 todos, 3 passed (21 assertions) Duration: x.xxs diff --git a/tests/.pest/snapshots/Visual/Todo/todos_in_parallel.snap b/tests/.pest/snapshots/Visual/Todo/todos_in_parallel.snap index c50794f7..09710340 100644 --- a/tests/.pest/snapshots/Visual/Todo/todos_in_parallel.snap +++ b/tests/.pest/snapshots/Visual/Todo/todos_in_parallel.snap @@ -15,6 +15,9 @@ ↓ todo on describe → should not fail ↓ todo on describe → should run + TODO Tests\Features\Flaky - 1 todo + ↓ it does not retry todo tests + TODO Tests\Features\Todo - 29 todos ↓ something todo later ↓ something todo later chained @@ -81,6 +84,6 @@ PASS Tests\CustomTestCase\ParentTest ✓ override method - Tests: 39 todos, 3 passed (21 assertions) + Tests: 40 todos, 3 passed (21 assertions) Duration: x.xxs diff --git a/tests/.snapshots/success.txt b/tests/.snapshots/success.txt index 79f1da47..f95ad679 100644 --- a/tests/.snapshots/success.txt +++ b/tests/.snapshots/success.txt @@ -1129,6 +1129,40 @@ ✓ it may return a file path ✓ it may throw an exception if the file does not exist + WARN Tests\Features\Flaky - 1 todo + ✓ it passes on first try + ✓ it passes on a subsequent try + ✓ it has a default of 3 tries + ✓ it succeeds on the last possible try + ✓ it works with tries of 1 + ✓ it retries assertion failures + ✓ it works with a dataset with (1) + ✓ it works with a dataset with (2) + ✓ it works with a dataset with (3) + ✓ it retries each dataset independently with ('alpha') + ✓ it retries each dataset independently with ('beta') + ✓ within a describe block → it retries inside describe + ✓ lifecycle hooks with flaky → it re-runs beforeEach on each retry + ✓ afterEach with flaky → it runs afterEach between retries + - it does not retry skipped tests → intentionally skipped + ✓ it works with repeat and flaky @ repetition 1 of 2 + ✓ it works with repeat and flaky @ repetition 2 of 2 + ✓ it works as higher order test + ✓ it fails after exhausting all retries + ✓ it throws when tries is less than 1 + ✓ it throws when tries is negative + ↓ it does not retry todo tests + ✓ it retries php errors + ✓ it works with throws and flaky + ✓ it does not retry expected exceptions + ✓ it does not retry fails() + ✓ it retries unexpected exceptions even with throws set + ✓ it does not leak mock objects between retries + ✓ it does not stop retrying when snapshot changes are absent + ✓ it does not leak dynamic properties between retries + ✓ it clears output buffer between retries when expectOutputString is used + ✓ it preserves output between retries when no output expectation is set + WARN Tests\Features\Helpers ✓ it can set/get properties on $this ! it gets null if property do not exist → Undefined property Tests\Features\Helpers::$wqdwqdqw @@ -1869,4 +1903,4 @@ ✓ pass with dataset with ('my-datas-set-value') ✓ within describe → pass with dataset with ('my-datas-set-value') - Tests: 2 deprecated, 4 warnings, 5 incomplete, 2 notices, 39 todos, 35 skipped, 1265 passed (2925 assertions) \ No newline at end of file + Tests: 2 deprecated, 4 warnings, 5 incomplete, 2 notices, 40 todos, 36 skipped, 1295 passed (2964 assertions) \ No newline at end of file diff --git a/tests/.tests/FlakyFailure.php b/tests/.tests/FlakyFailure.php new file mode 100644 index 00000000..61ad770b --- /dev/null +++ b/tests/.tests/FlakyFailure.php @@ -0,0 +1,5 @@ +flaky(tries: 2); diff --git a/tests/Features/Flaky.php b/tests/Features/Flaky.php new file mode 100644 index 00000000..2a55bad1 --- /dev/null +++ b/tests/Features/Flaky.php @@ -0,0 +1,300 @@ +toBeTrue(); +})->flaky(); + +it('passes on a subsequent try', function () { + $file = sys_get_temp_dir().'/pest_flaky_'.crc32(__FILE__.__LINE__); + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count < 2) { + throw new Exception('Flaky failure'); + } + + @unlink($file); + expect(true)->toBeTrue(); +})->flaky(tries: 3); + +it('has a default of 3 tries', function () { + expect(true)->toBeTrue(); +})->flaky(); + +it('succeeds on the last possible try', function () { + $file = sys_get_temp_dir().'/pest_flaky_last_try'; + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count < 3) { + throw new Exception('Not yet'); + } + + @unlink($file); + expect(true)->toBeTrue(); +})->flaky(tries: 3); + +it('works with tries of 1', function () { + expect(true)->toBeTrue(); +})->flaky(tries: 1); + +it('retries assertion failures', function () { + $file = sys_get_temp_dir().'/pest_flaky_assertion'; + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count < 2) { + expect(false)->toBeTrue(); + } + + @unlink($file); + expect(true)->toBeTrue(); +})->flaky(tries: 3); + +it('works with a dataset', function (int $number) { + expect($number)->toBeGreaterThan(0); +})->flaky(tries: 2)->with([1, 2, 3]); + +it('retries each dataset independently', function (string $label) { + $file = sys_get_temp_dir().'/pest_flaky_dataset_'.md5($label); + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count < 2) { + throw new Exception("Flaky for $label"); + } + + @unlink($file); + expect(true)->toBeTrue(); +})->flaky(tries: 3)->with(['alpha', 'beta']); + +describe('within a describe block', function () { + it('retries inside describe', function () { + $file = sys_get_temp_dir().'/pest_flaky_describe'; + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count < 2) { + throw new Exception('Flaky inside describe'); + } + + @unlink($file); + expect(true)->toBeTrue(); + })->flaky(tries: 2); +}); + +describe('lifecycle hooks with flaky', function () { + beforeEach(function () { + $this->setupCount = ($this->setupCount ?? 0) + 1; + }); + + it('re-runs beforeEach on each retry', function () { + $file = sys_get_temp_dir().'/pest_flaky_lifecycle'; + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count < 2) { + throw new Exception('Flaky lifecycle'); + } + + @unlink($file); + // After retry: setUp ran for initial + retry = setupCount should be 2 + expect($this->setupCount)->toBe(2); + })->flaky(tries: 3); +}); + +describe('afterEach with flaky', function () { + $state = new stdClass; + $state->teardownCount = 0; + + afterEach(function () use ($state) { + $state->teardownCount++; + }); + + it('runs afterEach between retries', function () use ($state) { + $file = sys_get_temp_dir().'/pest_flaky_aftereach'; + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count < 2) { + throw new Exception('Flaky afterEach'); + } + + @unlink($file); + // tearDown was called once between retries + expect($state->teardownCount)->toBe(1); + })->flaky(tries: 3); +}); + +it('does not retry skipped tests') + ->skip('intentionally skipped') + ->flaky(tries: 3); + +it('works with repeat and flaky', function () { + expect(true)->toBeTrue(); +})->repeat(times: 2)->flaky(tries: 2); + +it('works as higher order test') + ->assertTrue(true) + ->flaky(tries: 2); + +it('fails after exhausting all retries', function () { + $process = new Process( + ['php', 'bin/pest', 'tests/.tests/FlakyFailure.php'], + dirname(__DIR__, 2), + ['COLLISION_PRINTER' => 'DefaultPrinter', 'COLLISION_IGNORE_DURATION' => 'true'], + ); + + $process->run(); + + expect($process->getExitCode())->not->toBe(0); + expect(removeAnsiEscapeSequences($process->getOutput())) + ->toContain('FAILED') + ->toContain('Always fails'); +}); + +it('throws when tries is less than 1', function () { + it('invalid', function () {})->flaky(tries: 0); +})->throws(InvalidArgumentException::class, 'The number of tries must be greater than 0.'); + +it('throws when tries is negative', function () { + it('invalid negative', function () {})->flaky(tries: -1); +})->throws(InvalidArgumentException::class, 'The number of tries must be greater than 0.'); + +it('does not retry todo tests') + ->todo() + ->flaky(tries: 3); + +it('retries php errors', function () { + $file = sys_get_temp_dir().'/pest_flaky_error'; + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count < 2) { + throw new TypeError('type error'); + } + + @unlink($file); + expect(true)->toBeTrue(); +})->flaky(tries: 3); + +it('works with throws and flaky', function () { + throw new RuntimeException('Expected exception'); +})->throws(RuntimeException::class, 'Expected exception')->flaky(tries: 2); + +it('does not retry expected exceptions', function () { + // If flaky retried this, the temp file counter would reach 2 and + // the test would NOT throw — causing PHPUnit's "expected exception + // was not raised" to fail. The test passes only if we don't retry. + $file = sys_get_temp_dir().'/pest_flaky_expected'; + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count >= 2) { + @unlink($file); + + // Second call means flaky retried — don't throw, which will FAIL + // because PHPUnit expects the exception + return; + } + + @unlink($file); + throw new RuntimeException('Expected on first attempt'); +})->throws(RuntimeException::class)->flaky(tries: 3); + +it('does not retry fails()', function () { + $this->fail('Expected failure'); +})->fails('Expected failure')->flaky(tries: 2); + +it('retries unexpected exceptions even with throws set', function () { + $file = sys_get_temp_dir().'/pest_flaky_unexpected'; + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count < 2) { + throw new LogicException('Unexpected flaky error'); + } + + @unlink($file); + throw new RuntimeException('Expected exception'); +})->throws(RuntimeException::class)->flaky(tries: 3); + +it('does not leak mock objects between retries', function () { + $mock = $this->createMock(Countable::class); + $mock->expects($this->once())->method('count')->willReturn(1); + + $file = sys_get_temp_dir().'/pest_flaky_mock'; + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count < 2) { + @unlink(sys_get_temp_dir().'/pest_flaky_mock'); // clean before retry writes again + file_put_contents($file, '1'); + throw new Exception('Flaky mock failure'); + } + + @unlink($file); + // Call mock — only the mock from THIS attempt should be verified + expect($mock->count())->toBe(1); +})->flaky(tries: 3); + +it('does not stop retrying when snapshot changes are absent', function () { + // Ensures the snapshot guard only triggers when __snapshotChanges is non-empty + $file = sys_get_temp_dir().'/pest_flaky_no_snapshot'; + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count < 2) { + throw new Exception('No snapshots here'); + } + + @unlink($file); + expect(true)->toBeTrue(); +})->flaky(tries: 3); + +it('does not leak dynamic properties between retries', function () { + $file = sys_get_temp_dir().'/pest_flaky_props'; + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count < 2) { + $this->leakedProperty = 'from attempt 1'; + throw new Exception('Flaky props'); + } + + @unlink($file); + expect(isset($this->leakedProperty))->toBeFalse(); +})->flaky(tries: 3); + +it('clears output buffer between retries when expectOutputString is used', function () { + $file = sys_get_temp_dir().'/pest_flaky_output'; + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + $this->expectOutputString('clean'); + + if ($count < 2) { + echo 'stale'; + throw new Exception('Flaky output'); + } + + @unlink($file); + echo 'clean'; +})->flaky(tries: 3); + +it('preserves output between retries when no output expectation is set', function () { + $file = sys_get_temp_dir().'/pest_flaky_output_no_expect'; + $count = file_exists($file) ? (int) file_get_contents($file) : 0; + file_put_contents($file, (string) ++$count); + + if ($count < 2) { + echo 'from attempt 1'; + throw new Exception('Flaky output no expect'); + } + + @unlink($file); + // Output from attempt 1 is still in the buffer + $this->expectOutputString('from attempt 1'); +})->flaky(tries: 3); diff --git a/tests/Visual/Parallel.php b/tests/Visual/Parallel.php index ab62146d..be17ebde 100644 --- a/tests/Visual/Parallel.php +++ b/tests/Visual/Parallel.php @@ -15,8 +15,24 @@ $run = function () { }; test('parallel', function () use ($run) { - expect($run('--exclude-group=integration')) - ->toContain('Tests: 2 deprecated, 4 warnings, 5 incomplete, 3 notices, 39 todos, 26 skipped, 1250 passed (2887 assertions)') + $output = $run('--exclude-group=integration'); + + if (getenv('REBUILD_SNAPSHOTS')) { + preg_match('/Tests:\s+(.+\(\d+ assertions\))/', $output, $matches); + + $file = file_get_contents(__FILE__); + $file = preg_replace( + '/\$expected = \'.*?\';/', + "\$expected = '2 deprecated, 4 warnings, 5 incomplete, 3 notices, 40 todos, 27 skipped, 1280 passed (2926 assertions)';", + $file, + ); + file_put_contents(__FILE__, $file); + } + + $expected = '2 deprecated, 4 warnings, 5 incomplete, 3 notices, 40 todos, 27 skipped, 1280 passed (2926 assertions)'; + + expect($output) + ->toContain("Tests: {$expected}") ->toContain('Parallel: 3 processes'); })->skipOnWindows();