feat: adds flaky

src/Concerns/Testable.php

@@ -14,6 +14,8 @@ use Pest\Support\Reflection;
 use Pest\Support\Shell;
 use Pest\TestSuite;
 use PHPUnit\Framework\Attributes\PostCondition;
+use PHPUnit\Framework\IncompleteTest;
+use PHPUnit\Framework\SkippedTest;
 use PHPUnit\Framework\TestCase;
 use ReflectionException;
 use ReflectionFunction;
@@ -328,7 +330,80 @@ trait Testable
         $arguments = $this->__resolveTestArguments($args);
         $this->__ensureDatasetArgumentNameAndNumberMatches($arguments);
 
-        return $this->__callClosure($closure, $arguments);
+        $method = TestSuite::getInstance()->tests->get(self::$__filename)->getMethod($this->name());
+
+        if ($method->flakyTries === null) {
+            return $this->__callClosure($closure, $arguments);
+        }
+
+        $lastException = null;
+        $initialProperties = get_object_vars($this);
+
+        for ($attempt = 1; $attempt <= $method->flakyTries; $attempt++) {
+            try {
+                return $this->__callClosure($closure, $arguments);
+            } catch (Throwable $e) {
+                if ($e instanceof SkippedTest
+                    || $e instanceof IncompleteTest
+                    || $this->__isExpectedException($e)) {
+                    throw $e;
+                }
+
+                $lastException = $e;
+
+                if ($attempt < $method->flakyTries) {
+                    if ($this->__snapshotChanges !== []) {
+                        throw $e;
+                    }
+
+                    $this->tearDown();
+
+                    Closure::bind(fn (): array => $this->mockObjects = [], $this, TestCase::class)();
+
+                    foreach (array_keys(array_diff_key(get_object_vars($this), $initialProperties)) as $property) {
+                        unset($this->{$property});
+                    }
+
+                    $hasOutputExpectation = Closure::bind(fn (): bool => is_string($this->outputExpectedString) || is_string($this->outputExpectedRegex), $this, TestCase::class)();
+
+                    if ($hasOutputExpectation) {
+                        ob_clean();
+                    }
+
+                    $this->setUp();
+                }
+            }
+        }
+
+        throw $lastException;
+    }
+
+    /**
+     * Determines if the given exception matches PHPUnit's expected exception.
+     */
+    private function __isExpectedException(Throwable $e): bool
+    {
+        $read = fn (string $property): mixed => Closure::bind(fn () => $this->{$property}, $this, TestCase::class)();
+
+        $expectedClass = $read('expectedException');
+
+        if ($expectedClass !== null) {
+            return $e instanceof $expectedClass;
+        }
+
+        $expectedMessage = $read('expectedExceptionMessage');
+
+        if ($expectedMessage !== null) {
+            return str_contains($e->getMessage(), (string) $expectedMessage);
+        }
+
+        $expectedCode = $read('expectedExceptionCode');
+
+        if ($expectedCode !== null) {
+            return $e->getCode() === $expectedCode;
+        }
+
+        return false;
     }
 
     /**

src/Factories/TestCaseMethodFactory.php

@@ -50,6 +50,11 @@ final class TestCaseMethodFactory
      */
     public int $repetitions = 1;
 
+    /**
+     * The test's number of flaky retry tries.
+     */
+    public ?int $flakyTries = null;
+
     /**
      * Determines if the test is a "todo".
      */

src/PendingCalls/TestCall.php

@@ -412,6 +412,20 @@ final class TestCall // @phpstan-ignore-line
         return $this;
     }
 
+    /**
+     * Marks the test as flaky, retrying it up to the given number of times.
+     */
+    public function flaky(int $tries = 3): self
+    {
+        if ($tries < 1) {
+            throw new InvalidArgumentException('The number of tries must be greater than 0.');
+        }
+
+        $this->testCaseMethod->flakyTries = $tries;
+
+        return $this;
+    }
+
     /**
      * Marks the test as "todo".
      */

tests/.pest/snapshots/Visual/Todo/todo.snap

@@ -15,6 +15,9 @@
   ↓ todo on describe → should not fail
   ↓ todo on describe → should run
 
+   TODO  Tests\Features\Flaky - 1 todo
+  ↓ it does not retry todo tests
+
    TODO  Tests\Features\Todo - 29 todos
   ↓ something todo later
   ↓ something todo later chained
@@ -81,6 +84,6 @@
    PASS  Tests\CustomTestCase\ParentTest
   ✓ override method
 
-  Tests:    39 todos, 3 passed (21 assertions)
+  Tests:    40 todos, 3 passed (21 assertions)
   Duration: x.xxs
 

tests/.pest/snapshots/Visual/Todo/todo_in_parallel.snap

@@ -15,6 +15,9 @@
   ↓ todo on describe → should not fail
   ↓ todo on describe → should run
 
+   TODO  Tests\Features\Flaky - 1 todo
+  ↓ it does not retry todo tests
+
    TODO  Tests\Features\Todo - 29 todos
   ↓ something todo later
   ↓ something todo later chained
@@ -81,6 +84,6 @@
    PASS  Tests\CustomTestCase\ParentTest
   ✓ override method
 
-  Tests:    39 todos, 3 passed (21 assertions)
+  Tests:    40 todos, 3 passed (21 assertions)
   Duration: x.xxs
 

tests/.pest/snapshots/Visual/Todo/todos.snap

@@ -15,6 +15,9 @@
   ↓ todo on describe → should not fail
   ↓ todo on describe → should run
 
+   TODO  Tests\Features\Flaky - 1 todo
+  ↓ it does not retry todo tests
+
    TODO  Tests\Features\Todo - 29 todos
   ↓ something todo later
   ↓ something todo later chained
@@ -81,6 +84,6 @@
    PASS  Tests\CustomTestCase\ParentTest
   ✓ override method
 
-  Tests:    39 todos, 3 passed (21 assertions)
+  Tests:    40 todos, 3 passed (21 assertions)
   Duration: x.xxs
 

tests/.pest/snapshots/Visual/Todo/todos_in_parallel.snap

@@ -15,6 +15,9 @@
   ↓ todo on describe → should not fail
   ↓ todo on describe → should run
 
+   TODO  Tests\Features\Flaky - 1 todo
+  ↓ it does not retry todo tests
+
    TODO  Tests\Features\Todo - 29 todos
   ↓ something todo later
   ↓ something todo later chained
@@ -81,6 +84,6 @@
    PASS  Tests\CustomTestCase\ParentTest
   ✓ override method
 
-  Tests:    39 todos, 3 passed (21 assertions)
+  Tests:    40 todos, 3 passed (21 assertions)
   Duration: x.xxs
 

tests/.snapshots/success.txt

@@ -1129,6 +1129,40 @@
   ✓ it may return a file path
   ✓ it may throw an exception if the file does not exist
 
+   WARN  Tests\Features\Flaky - 1 todo
+  ✓ it passes on first try
+  ✓ it passes on a subsequent try
+  ✓ it has a default of 3 tries
+  ✓ it succeeds on the last possible try
+  ✓ it works with tries of 1
+  ✓ it retries assertion failures
+  ✓ it works with a dataset with (1)
+  ✓ it works with a dataset with (2)
+  ✓ it works with a dataset with (3)
+  ✓ it retries each dataset independently with ('alpha')
+  ✓ it retries each dataset independently with ('beta')
+  ✓ within a describe block → it retries inside describe
+  ✓ lifecycle hooks with flaky → it re-runs beforeEach on each retry
+  ✓ afterEach with flaky → it runs afterEach between retries
+  - it does not retry skipped tests → intentionally skipped
+  ✓ it works with repeat and flaky @ repetition 1 of 2
+  ✓ it works with repeat and flaky @ repetition 2 of 2
+  ✓ it works as higher order test
+  ✓ it fails after exhausting all retries
+  ✓ it throws when tries is less than 1
+  ✓ it throws when tries is negative
+  ↓ it does not retry todo tests
+  ✓ it retries php errors
+  ✓ it works with throws and flaky
+  ✓ it does not retry expected exceptions
+  ✓ it does not retry fails()
+  ✓ it retries unexpected exceptions even with throws set
+  ✓ it does not leak mock objects between retries
+  ✓ it does not stop retrying when snapshot changes are absent
+  ✓ it does not leak dynamic properties between retries
+  ✓ it clears output buffer between retries when expectOutputString is used
+  ✓ it preserves output between retries when no output expectation is set
+
    WARN  Tests\Features\Helpers
   ✓ it can set/get properties on $this
   ! it gets null if property do not exist → Undefined property Tests\Features\Helpers::$wqdwqdqw
@@ -1869,4 +1903,4 @@
   ✓ pass with dataset with ('my-datas-set-value')
   ✓ within describe → pass with dataset with ('my-datas-set-value')
 
-  Tests:    2 deprecated, 4 warnings, 5 incomplete, 2 notices, 39 todos, 35 skipped, 1265 passed (2925 assertions)
+  Tests:    2 deprecated, 4 warnings, 5 incomplete, 2 notices, 40 todos, 36 skipped, 1295 passed (2964 assertions)

tests/.tests/FlakyFailure.php

@@ -0,0 +1,5 @@
+<?php
+
+it('fails after exhausting all retries', function () {
+    throw new Exception('Always fails');
+})->flaky(tries: 2);

tests/Features/Flaky.php

@@ -0,0 +1,300 @@
+<?php
+
+use Symfony\Component\Process\Process;
+
+it('passes on first try', function () {
+    expect(true)->toBeTrue();
+})->flaky();
+
+it('passes on a subsequent try', function () {
+    $file = sys_get_temp_dir().'/pest_flaky_'.crc32(__FILE__.__LINE__);
+    $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+    file_put_contents($file, (string) ++$count);
+
+    if ($count < 2) {
+        throw new Exception('Flaky failure');
+    }
+
+    @unlink($file);
+    expect(true)->toBeTrue();
+})->flaky(tries: 3);
+
+it('has a default of 3 tries', function () {
+    expect(true)->toBeTrue();
+})->flaky();
+
+it('succeeds on the last possible try', function () {
+    $file = sys_get_temp_dir().'/pest_flaky_last_try';
+    $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+    file_put_contents($file, (string) ++$count);
+
+    if ($count < 3) {
+        throw new Exception('Not yet');
+    }
+
+    @unlink($file);
+    expect(true)->toBeTrue();
+})->flaky(tries: 3);
+
+it('works with tries of 1', function () {
+    expect(true)->toBeTrue();
+})->flaky(tries: 1);
+
+it('retries assertion failures', function () {
+    $file = sys_get_temp_dir().'/pest_flaky_assertion';
+    $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+    file_put_contents($file, (string) ++$count);
+
+    if ($count < 2) {
+        expect(false)->toBeTrue();
+    }
+
+    @unlink($file);
+    expect(true)->toBeTrue();
+})->flaky(tries: 3);
+
+it('works with a dataset', function (int $number) {
+    expect($number)->toBeGreaterThan(0);
+})->flaky(tries: 2)->with([1, 2, 3]);
+
+it('retries each dataset independently', function (string $label) {
+    $file = sys_get_temp_dir().'/pest_flaky_dataset_'.md5($label);
+    $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+    file_put_contents($file, (string) ++$count);
+
+    if ($count < 2) {
+        throw new Exception("Flaky for $label");
+    }
+
+    @unlink($file);
+    expect(true)->toBeTrue();
+})->flaky(tries: 3)->with(['alpha', 'beta']);
+
+describe('within a describe block', function () {
+    it('retries inside describe', function () {
+        $file = sys_get_temp_dir().'/pest_flaky_describe';
+        $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+        file_put_contents($file, (string) ++$count);
+
+        if ($count < 2) {
+            throw new Exception('Flaky inside describe');
+        }
+
+        @unlink($file);
+        expect(true)->toBeTrue();
+    })->flaky(tries: 2);
+});
+
+describe('lifecycle hooks with flaky', function () {
+    beforeEach(function () {
+        $this->setupCount = ($this->setupCount ?? 0) + 1;
+    });
+
+    it('re-runs beforeEach on each retry', function () {
+        $file = sys_get_temp_dir().'/pest_flaky_lifecycle';
+        $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+        file_put_contents($file, (string) ++$count);
+
+        if ($count < 2) {
+            throw new Exception('Flaky lifecycle');
+        }
+
+        @unlink($file);
+        // After retry: setUp ran for initial + retry = setupCount should be 2
+        expect($this->setupCount)->toBe(2);
+    })->flaky(tries: 3);
+});
+
+describe('afterEach with flaky', function () {
+    $state = new stdClass;
+    $state->teardownCount = 0;
+
+    afterEach(function () use ($state) {
+        $state->teardownCount++;
+    });
+
+    it('runs afterEach between retries', function () use ($state) {
+        $file = sys_get_temp_dir().'/pest_flaky_aftereach';
+        $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+        file_put_contents($file, (string) ++$count);
+
+        if ($count < 2) {
+            throw new Exception('Flaky afterEach');
+        }
+
+        @unlink($file);
+        // tearDown was called once between retries
+        expect($state->teardownCount)->toBe(1);
+    })->flaky(tries: 3);
+});
+
+it('does not retry skipped tests')
+    ->skip('intentionally skipped')
+    ->flaky(tries: 3);
+
+it('works with repeat and flaky', function () {
+    expect(true)->toBeTrue();
+})->repeat(times: 2)->flaky(tries: 2);
+
+it('works as higher order test')
+    ->assertTrue(true)
+    ->flaky(tries: 2);
+
+it('fails after exhausting all retries', function () {
+    $process = new Process(
+        ['php', 'bin/pest', 'tests/.tests/FlakyFailure.php'],
+        dirname(__DIR__, 2),
+        ['COLLISION_PRINTER' => 'DefaultPrinter', 'COLLISION_IGNORE_DURATION' => 'true'],
+    );
+
+    $process->run();
+
+    expect($process->getExitCode())->not->toBe(0);
+    expect(removeAnsiEscapeSequences($process->getOutput()))
+        ->toContain('FAILED')
+        ->toContain('Always fails');
+});
+
+it('throws when tries is less than 1', function () {
+    it('invalid', function () {})->flaky(tries: 0);
+})->throws(InvalidArgumentException::class, 'The number of tries must be greater than 0.');
+
+it('throws when tries is negative', function () {
+    it('invalid negative', function () {})->flaky(tries: -1);
+})->throws(InvalidArgumentException::class, 'The number of tries must be greater than 0.');
+
+it('does not retry todo tests')
+    ->todo()
+    ->flaky(tries: 3);
+
+it('retries php errors', function () {
+    $file = sys_get_temp_dir().'/pest_flaky_error';
+    $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+    file_put_contents($file, (string) ++$count);
+
+    if ($count < 2) {
+        throw new TypeError('type error');
+    }
+
+    @unlink($file);
+    expect(true)->toBeTrue();
+})->flaky(tries: 3);
+
+it('works with throws and flaky', function () {
+    throw new RuntimeException('Expected exception');
+})->throws(RuntimeException::class, 'Expected exception')->flaky(tries: 2);
+
+it('does not retry expected exceptions', function () {
+    // If flaky retried this, the temp file counter would reach 2 and
+    // the test would NOT throw — causing PHPUnit's "expected exception
+    // was not raised" to fail. The test passes only if we don't retry.
+    $file = sys_get_temp_dir().'/pest_flaky_expected';
+    $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+    file_put_contents($file, (string) ++$count);
+
+    if ($count >= 2) {
+        @unlink($file);
+
+        // Second call means flaky retried — don't throw, which will FAIL
+        // because PHPUnit expects the exception
+        return;
+    }
+
+    @unlink($file);
+    throw new RuntimeException('Expected on first attempt');
+})->throws(RuntimeException::class)->flaky(tries: 3);
+
+it('does not retry fails()', function () {
+    $this->fail('Expected failure');
+})->fails('Expected failure')->flaky(tries: 2);
+
+it('retries unexpected exceptions even with throws set', function () {
+    $file = sys_get_temp_dir().'/pest_flaky_unexpected';
+    $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+    file_put_contents($file, (string) ++$count);
+
+    if ($count < 2) {
+        throw new LogicException('Unexpected flaky error');
+    }
+
+    @unlink($file);
+    throw new RuntimeException('Expected exception');
+})->throws(RuntimeException::class)->flaky(tries: 3);
+
+it('does not leak mock objects between retries', function () {
+    $mock = $this->createMock(Countable::class);
+    $mock->expects($this->once())->method('count')->willReturn(1);
+
+    $file = sys_get_temp_dir().'/pest_flaky_mock';
+    $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+    file_put_contents($file, (string) ++$count);
+
+    if ($count < 2) {
+        @unlink(sys_get_temp_dir().'/pest_flaky_mock'); // clean before retry writes again
+        file_put_contents($file, '1');
+        throw new Exception('Flaky mock failure');
+    }
+
+    @unlink($file);
+    // Call mock — only the mock from THIS attempt should be verified
+    expect($mock->count())->toBe(1);
+})->flaky(tries: 3);
+
+it('does not stop retrying when snapshot changes are absent', function () {
+    // Ensures the snapshot guard only triggers when __snapshotChanges is non-empty
+    $file = sys_get_temp_dir().'/pest_flaky_no_snapshot';
+    $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+    file_put_contents($file, (string) ++$count);
+
+    if ($count < 2) {
+        throw new Exception('No snapshots here');
+    }
+
+    @unlink($file);
+    expect(true)->toBeTrue();
+})->flaky(tries: 3);
+
+it('does not leak dynamic properties between retries', function () {
+    $file = sys_get_temp_dir().'/pest_flaky_props';
+    $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+    file_put_contents($file, (string) ++$count);
+
+    if ($count < 2) {
+        $this->leakedProperty = 'from attempt 1';
+        throw new Exception('Flaky props');
+    }
+
+    @unlink($file);
+    expect(isset($this->leakedProperty))->toBeFalse();
+})->flaky(tries: 3);
+
+it('clears output buffer between retries when expectOutputString is used', function () {
+    $file = sys_get_temp_dir().'/pest_flaky_output';
+    $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+    file_put_contents($file, (string) ++$count);
+
+    $this->expectOutputString('clean');
+
+    if ($count < 2) {
+        echo 'stale';
+        throw new Exception('Flaky output');
+    }
+
+    @unlink($file);
+    echo 'clean';
+})->flaky(tries: 3);
+
+it('preserves output between retries when no output expectation is set', function () {
+    $file = sys_get_temp_dir().'/pest_flaky_output_no_expect';
+    $count = file_exists($file) ? (int) file_get_contents($file) : 0;
+    file_put_contents($file, (string) ++$count);
+
+    if ($count < 2) {
+        echo 'from attempt 1';
+        throw new Exception('Flaky output no expect');
+    }
+
+    @unlink($file);
+    // Output from attempt 1 is still in the buffer
+    $this->expectOutputString('from attempt 1');
+})->flaky(tries: 3);

tests/Visual/Parallel.php

@@ -15,8 +15,24 @@ $run = function () {
 };
 
 test('parallel', function () use ($run) {
-    expect($run('--exclude-group=integration'))
-        ->toContain('Tests:    2 deprecated, 4 warnings, 5 incomplete, 3 notices, 39 todos, 26 skipped, 1250 passed (2887 assertions)')
+    $output = $run('--exclude-group=integration');
+
+    if (getenv('REBUILD_SNAPSHOTS')) {
+        preg_match('/Tests:\s+(.+\(\d+ assertions\))/', $output, $matches);
+
+        $file = file_get_contents(__FILE__);
+        $file = preg_replace(
+            '/\$expected = \'.*?\';/',
+            "\$expected = '2 deprecated, 4 warnings, 5 incomplete, 3 notices, 40 todos, 27 skipped, 1280 passed (2926 assertions)';",
+            $file,
+        );
+        file_put_contents(__FILE__, $file);
+    }
+
+    $expected = '2 deprecated, 4 warnings, 5 incomplete, 3 notices, 40 todos, 27 skipped, 1280 passed (2926 assertions)';
+
+    expect($output)
+        ->toContain("Tests:    {$expected}")
         ->toContain('Parallel: 3 processes');
 })->skipOnWindows();