fix(tia): replayable scenarios running afterEach

This commit is contained in:
nuno maduro
2026-08-04 20:19:12 +01:00
parent 668809bc75
commit 872f0a50c2
29 changed files with 168 additions and 57 deletions
+13 -13
View File
@@ -1,27 +1,27 @@
##teamcity[testSuiteStarted name='Tests/tests/Failure' locationHint='pest_qn://tests/.tests/Failure.php' flowId='1234']
##teamcity[testSuiteStarted name='Tests/Fixtures/Suites/Failure' locationHint='pest_qn://tests/Fixtures/Suites/Failure.php' flowId='1234']
##teamcity[testCount count='8' flowId='1234']
##teamcity[testStarted name='it can fail with comparison' locationHint='pest_qn://tests/.tests/Failure.php::it can fail with comparison' flowId='1234']
##teamcity[testFailed name='it can fail with comparison' message='Failed asserting that true matches expected false.' details='at tests/.tests/Failure.php:6' type='comparisonFailure' actual='true' expected='false' flowId='1234']
##teamcity[testStarted name='it can fail with comparison' locationHint='pest_qn://tests/Fixtures/Suites/Failure.php::it can fail with comparison' flowId='1234']
##teamcity[testFailed name='it can fail with comparison' message='Failed asserting that true matches expected false.' details='at tests/Fixtures/Suites/Failure.php:6' type='comparisonFailure' actual='true' expected='false' flowId='1234']
##teamcity[testFinished name='it can fail with comparison' duration='100000' flowId='1234']
##teamcity[testStarted name='it can be ignored because of no assertions' locationHint='pest_qn://tests/.tests/Failure.php::it can be ignored because of no assertions' flowId='1234']
##teamcity[testStarted name='it can be ignored because of no assertions' locationHint='pest_qn://tests/Fixtures/Suites/Failure.php::it can be ignored because of no assertions' flowId='1234']
##teamcity[testIgnored name='it can be ignored because of no assertions' message='This test did not perform any assertions' details='' flowId='1234']
##teamcity[testFinished name='it can be ignored because of no assertions' duration='100000' flowId='1234']
##teamcity[testStarted name='it can be ignored because it is skipped' locationHint='pest_qn://tests/.tests/Failure.php::it can be ignored because it is skipped' flowId='1234']
##teamcity[testStarted name='it can be ignored because it is skipped' locationHint='pest_qn://tests/Fixtures/Suites/Failure.php::it can be ignored because it is skipped' flowId='1234']
##teamcity[testIgnored name='it can be ignored because it is skipped' message='This test was ignored.' details='' flowId='1234']
##teamcity[testFinished name='it can be ignored because it is skipped' duration='100000' flowId='1234']
##teamcity[testStarted name='it can fail' locationHint='pest_qn://tests/.tests/Failure.php::it can fail' flowId='1234']
##teamcity[testFailed name='it can fail' message='oh noo' details='at tests/.tests/Failure.php:18' flowId='1234']
##teamcity[testStarted name='it can fail' locationHint='pest_qn://tests/Fixtures/Suites/Failure.php::it can fail' flowId='1234']
##teamcity[testFailed name='it can fail' message='oh noo' details='at tests/Fixtures/Suites/Failure.php:18' flowId='1234']
##teamcity[testFinished name='it can fail' duration='100000' flowId='1234']
##teamcity[testStarted name='it throws exception' locationHint='pest_qn://tests/.tests/Failure.php::it throws exception' flowId='1234']
##teamcity[testFailed name='it throws exception' message='Exception: test error' details='at tests/.tests/Failure.php:22' flowId='1234']
##teamcity[testStarted name='it throws exception' locationHint='pest_qn://tests/Fixtures/Suites/Failure.php::it throws exception' flowId='1234']
##teamcity[testFailed name='it throws exception' message='Exception: test error' details='at tests/Fixtures/Suites/Failure.php:22' flowId='1234']
##teamcity[testFinished name='it throws exception' duration='100000' flowId='1234']
##teamcity[testStarted name='it is not done yet' locationHint='pest_qn://tests/.tests/Failure.php::it is not done yet' flowId='1234']
##teamcity[testStarted name='it is not done yet' locationHint='pest_qn://tests/Fixtures/Suites/Failure.php::it is not done yet' flowId='1234']
##teamcity[testFinished name='it is not done yet' duration='100000' flowId='1234']
##teamcity[testStarted name='build this one.' locationHint='pest_qn://tests/.tests/Failure.php::build this one.' flowId='1234']
##teamcity[testStarted name='build this one.' locationHint='pest_qn://tests/Fixtures/Suites/Failure.php::build this one.' flowId='1234']
##teamcity[testFinished name='build this one.' duration='100000' flowId='1234']
##teamcity[testStarted name='it is passing' locationHint='pest_qn://tests/.tests/Failure.php::it is passing' flowId='1234']
##teamcity[testStarted name='it is passing' locationHint='pest_qn://tests/Fixtures/Suites/Failure.php::it is passing' flowId='1234']
##teamcity[testFinished name='it is passing' duration='100000' flowId='1234']
##teamcity[testSuiteFinished name='Tests/tests/Failure' flowId='1234']
##teamcity[testSuiteFinished name='Tests/Fixtures/Suites/Failure' flowId='1234']
Tests: 3 failed, 1 risky, 2 todos, 1 skipped, 1 passed (3 assertions)
Duration: 1.00s
+8 -8
View File
@@ -1,18 +1,18 @@
##teamcity[testSuiteStarted name='Tests/tests/SuccessOnly' locationHint='pest_qn://tests/.tests/SuccessOnly.php' flowId='1234']
##teamcity[testSuiteStarted name='Tests/Fixtures/Suites/SuccessOnly' locationHint='pest_qn://tests/Fixtures/Suites/SuccessOnly.php' flowId='1234']
##teamcity[testCount count='4' flowId='1234']
##teamcity[testStarted name='it can pass with comparison' locationHint='pest_qn://tests/.tests/SuccessOnly.php::it can pass with comparison' flowId='1234']
##teamcity[testStarted name='it can pass with comparison' locationHint='pest_qn://tests/Fixtures/Suites/SuccessOnly.php::it can pass with comparison' flowId='1234']
##teamcity[testFinished name='it can pass with comparison' duration='100000' flowId='1234']
##teamcity[testStarted name='can also pass' locationHint='pest_qn://tests/.tests/SuccessOnly.php::can also pass' flowId='1234']
##teamcity[testStarted name='can also pass' locationHint='pest_qn://tests/Fixtures/Suites/SuccessOnly.php::can also pass' flowId='1234']
##teamcity[testFinished name='can also pass' duration='100000' flowId='1234']
##teamcity[testSuiteStarted name='can pass with dataset' locationHint='pest_qn://tests/.tests/SuccessOnly.php::can pass with dataset' flowId='1234']
##teamcity[testStarted name='can pass with dataset with data set "(true)"' locationHint='pest_qn://tests/.tests/SuccessOnly.php::can pass with dataset with data set "(true)"' flowId='1234']
##teamcity[testSuiteStarted name='can pass with dataset' locationHint='pest_qn://tests/Fixtures/Suites/SuccessOnly.php::can pass with dataset' flowId='1234']
##teamcity[testStarted name='can pass with dataset with data set "(true)"' locationHint='pest_qn://tests/Fixtures/Suites/SuccessOnly.php::can pass with dataset with data set "(true)"' flowId='1234']
##teamcity[testFinished name='can pass with dataset with data set "(true)"' duration='100000' flowId='1234']
##teamcity[testSuiteFinished name='can pass with dataset' flowId='1234']
##teamcity[testSuiteStarted name='`block` → can pass with dataset in describe block' locationHint='pest_qn://tests/.tests/SuccessOnly.php::`block` → can pass with dataset in describe block' flowId='1234']
##teamcity[testStarted name='`block` → can pass with dataset in describe block with data set "(1)"' locationHint='pest_qn://tests/.tests/SuccessOnly.php::`block` → can pass with dataset in describe block with data set "(1)"' flowId='1234']
##teamcity[testSuiteStarted name='`block` → can pass with dataset in describe block' locationHint='pest_qn://tests/Fixtures/Suites/SuccessOnly.php::`block` → can pass with dataset in describe block' flowId='1234']
##teamcity[testStarted name='`block` → can pass with dataset in describe block with data set "(1)"' locationHint='pest_qn://tests/Fixtures/Suites/SuccessOnly.php::`block` → can pass with dataset in describe block with data set "(1)"' flowId='1234']
##teamcity[testFinished name='`block` → can pass with dataset in describe block with data set "(1)"' duration='100000' flowId='1234']
##teamcity[testSuiteFinished name='`block` → can pass with dataset in describe block' flowId='1234']
##teamcity[testSuiteFinished name='Tests/tests/SuccessOnly' flowId='1234']
##teamcity[testSuiteFinished name='Tests/Fixtures/Suites/SuccessOnly' flowId='1234']
Tests: 4 passed (4 assertions)
Duration: 1.00s
+1 -14
View File
@@ -1,17 +1,4 @@
WARN Tests\Fixtures\CollisionTest
- error
- success
PASS Tests\Fixtures\DirectoryWithTests\ExampleTest
✓ it example 1
PASS Tests\Fixtures\ExampleTest
✓ it example 2
WARN Tests\Fixtures\Inheritance\Base\ExampleTest
- example
PASS Tests\Fixtures\Inheritance\ExampleTest
✓ example
Tests: 3 skipped, 3 passed (3 assertions)
Tests: 1 passed (1 assertions)
+4 -1
View File
@@ -1565,6 +1565,9 @@
✓ it allows performing no expectations without being risky
✓ a "describe" group of tests → it allows performing no expectations without being risky
PASS Tests\Features\Tia
✓ it does not run user hooks when replaying cached skipped and incomplete results
PASS Tests\Features\Ticket
✓ it may be associated with an ticket #1, #2
✓ nested → it may be associated with an ticket #1, #4, #5, #6, #3
@@ -2194,4 +2197,4 @@
✓ pass with dataset with ('my-datas-set-value')
✓ within describe → pass with dataset with ('my-datas-set-value')
Tests: 1 deprecated, 4 warnings, 5 incomplete, 2 notices, 40 todos, 35 skipped, 1548 passed (3383 assertions)
Tests: 1 deprecated, 4 warnings, 5 incomplete, 2 notices, 40 todos, 35 skipped, 1549 passed (3389 assertions)
+3 -3
View File
@@ -18,7 +18,7 @@ $run = function (string $target): array {
};
test('reports missing datasets as errors for a single file run', function () use ($run): void {
$result = $run('tests/.tests/IssueOnly.php');
$result = $run('tests/Fixtures/Suites/IssueOnly.php');
expect($result['output'])
->toContain("A dataset with the name `missing` does not exist. You can create it using `dataset('missing', ['a', 'b']);`.")
@@ -28,7 +28,7 @@ test('reports missing datasets as errors for a single file run', function () use
})->skipOnWindows();
test('reports missing datasets as errors alongside passing tests', function () use ($run): void {
$result = $run('tests/.tests/IssueWithPassing.php');
$result = $run('tests/Fixtures/Suites/IssueWithPassing.php');
expect($result['output'])
->toContain("A dataset with the name `missing` does not exist. You can create it using `dataset('missing', ['a', 'b']);`.")
@@ -38,7 +38,7 @@ test('reports missing datasets as errors alongside passing tests', function () u
})->skipOnWindows();
test('reports dataset closure exceptions as errors', function () use ($run): void {
$result = $run('tests/.tests/DatasetClosureThrows.php');
$result = $run('tests/Fixtures/Suites/DatasetClosureThrows.php');
expect($result['output'])
->toContain('boom from dataset')
+1 -1
View File
@@ -142,7 +142,7 @@ it('works as higher order test')
it('fails after exhausting all retries', function (): void {
$process = new Process(
['php', 'bin/pest', 'tests/.tests/FlakyFailure.php'],
['php', 'bin/pest', 'tests/Fixtures/Suites/FlakyFailure.php'],
dirname(__DIR__, 2),
['COLLISION_PRINTER' => 'DefaultPrinter', 'COLLISION_IGNORE_DURATION' => 'true', 'PAO_DISABLE' => '1'],
);
+84
View File
@@ -0,0 +1,84 @@
<?php
use Pest\Plugins\Tia;
use Pest\Plugins\Tia\ChangedFiles;
use Pest\Plugins\Tia\FileState;
use Pest\Plugins\Tia\Fingerprint;
use Pest\Plugins\Tia\Graph;
use Pest\Plugins\Tia\Storage;
use Pest\Support\Str;
use Symfony\Component\Process\Process;
it('does not run user hooks when replaying cached skipped and incomplete results', function (): void {
$projectRoot = dirname(__DIR__, 2);
$home = sys_get_temp_dir().'/pest-tia-'.bin2hex(random_bytes(8));
$fixture = 'tests/Fixtures/Suites/TiaReplayHooks.php';
mkdir($home, 0755, true);
try {
$changedFiles = new ChangedFiles($projectRoot);
$branch = $changedFiles->currentBranch() ?? 'main';
$sha = $changedFiles->currentSha();
$id = fn (string $description): string => 'P\Tests\Fixtures\Suites\TiaReplayHooks::'.Str::evaluable($description);
$graph = new Graph($projectRoot);
$graph->setFingerprint(Fingerprint::compute($projectRoot));
$graph->setRecordedAtSha($branch, $sha);
// Hashes the working tree as it stands, so the replay sees nothing as changed.
$graph->setLastRunTree($branch, $changedFiles->snapshotTree($changedFiles->since($sha) ?? []));
$graph->markKnownTestFiles([$fixture]);
$graph->setResult($branch, $id('replayed pass'), 0, '', 0.01, 1, $fixture);
$graph->setResult($branch, $id('replayed skip'), 1, 'cached skip', 0.01, 0, $fixture);
$graph->setResult($branch, $id('replayed incomplete'), 2, 'cached incomplete', 0.01, 0, $fixture);
$json = $graph->encode();
expect($json)->not->toBeNull();
$originalHome = getenv('HOME');
putenv('HOME='.$home);
try {
$storage = new FileState(Storage::tempDir($projectRoot));
} finally {
putenv($originalHome === false ? 'HOME' : 'HOME='.$originalHome);
}
expect($storage->write(Tia::KEY_GRAPH, (string) $json))->toBeTrue();
$process = new Process(
['php', 'bin/pest', $fixture, '--tia'],
$projectRoot,
[
'COLLISION_PRINTER' => 'DefaultPrinter',
'COLLISION_IGNORE_DURATION' => 'true',
'PARATEST' => 0,
'PAO_DISABLE' => '1',
'HOME' => $home,
],
);
$process->run();
$output = removeAnsiEscapeSequences($process->getOutput().$process->getErrorOutput());
// Both hooks throw, so the run stays green only if neither one ran.
expect($output)->toContain('3 replayed')
->and($output)->not->toContain('must not run for replayed tests')
->and($output)->toContain('1 incomplete, 1 skipped, 1 passed')
->and($process->getExitCode())->toBe(0);
} finally {
$paths = new RecursiveIteratorIterator(
new RecursiveDirectoryIterator($home, FilesystemIterator::SKIP_DOTS),
RecursiveIteratorIterator::CHILD_FIRST,
);
foreach ($paths as $path) {
$path->isDir() ? @rmdir($path->getPathname()) : @unlink($path->getPathname());
}
@rmdir($home);
}
})->skipOnWindows();
+24
View File
@@ -0,0 +1,24 @@
<?php
// Only ever run through `tests/Features/Tia.php`, against a seeded TIA graph.
// Both hooks throw, so a replayed test that wrongly runs one fails the run.
beforeEach(function (): void {
throw new RuntimeException('The beforeEach hook must not run for replayed tests.');
});
afterEach(function (): void {
throw new RuntimeException('The afterEach hook must not run for replayed tests.');
});
test('replayed pass', function (): void {
expect(true)->toBeTrue();
});
test('replayed skip', function (): void {
expect(true)->toBeTrue();
});
test('replayed incomplete', function (): void {
expect(true)->toBeTrue();
});
+8 -8
View File
@@ -29,30 +29,30 @@ $run = function () {
$normalizedPath = (fn (string $path): string => str_replace('/', DIRECTORY_SEPARATOR, $path));
test('junit output', function () use ($normalizedPath, $run): void {
$result = $run('tests/.tests/SuccessOnly.php');
$result = $run('tests/Fixtures/Suites/SuccessOnly.php');
expect($result['testsuite']['@attributes'])
->name->toBe('Tests\tests\SuccessOnly')
->file->toBe($normalizedPath('tests/.tests/SuccessOnly.php'))
->name->toBe('Tests\Fixtures\Suites\SuccessOnly')
->file->toBe($normalizedPath('tests/Fixtures/Suites/SuccessOnly.php'))
->tests->toBe('4')
->assertions->toBe('4')
->errors->toBe('0')
->failures->toBe('0')
->skipped->toBe('0')
->and($result['testsuite']['testcase'])->toHaveCount(2)
->and($result['testsuite']['testcase'][0]['@attributes'])->name->toBe('it can pass with comparison')->file->toBe($normalizedPath('tests/.tests/SuccessOnly.php::it can pass with comparison'))->class->toBe('Tests\tests\SuccessOnly')->classname->toBe('Tests.tests.SuccessOnly')->assertions->toBe('1')->time->toStartWith('0.0');
->and($result['testsuite']['testcase'][0]['@attributes'])->name->toBe('it can pass with comparison')->file->toBe($normalizedPath('tests/Fixtures/Suites/SuccessOnly.php::it can pass with comparison'))->class->toBe('Tests\Fixtures\Suites\SuccessOnly')->classname->toBe('Tests.Fixtures.Suites.SuccessOnly')->assertions->toBe('1')->time->toStartWith('0.0');
});
test('junit with parallel', function () use ($normalizedPath, $run): void {
$result = $run('tests/.tests/SuccessOnly.php', '--parallel', '--processes=1', '--filter', 'can pass with comparison');
$result = $run('tests/Fixtures/Suites/SuccessOnly.php', '--parallel', '--processes=1', '--filter', 'can pass with comparison');
expect($result['testsuite']['@attributes'])
->name->toBe('Tests\tests\SuccessOnly')
->file->toBe($normalizedPath('tests/.tests/SuccessOnly.php'))
->name->toBe('Tests\Fixtures\Suites\SuccessOnly')
->file->toBe($normalizedPath('tests/Fixtures/Suites/SuccessOnly.php'))
->tests->toBe('1')
->assertions->toBe('1')
->errors->toBe('0')
->failures->toBe('0')
->skipped->toBe('0')
->and($result['testsuite']['testcase']['@attributes'])->name->toBe('it can pass with comparison')->file->toBe($normalizedPath('tests/.tests/SuccessOnly.php::it can pass with comparison'))->class->toBe('Tests\tests\SuccessOnly')->classname->toBe('Tests.tests.SuccessOnly')->assertions->toBe('1')->time->toStartWith('0.0');
->and($result['testsuite']['testcase']['@attributes'])->name->toBe('it can pass with comparison')->file->toBe($normalizedPath('tests/Fixtures/Suites/SuccessOnly.php::it can pass with comparison'))->class->toBe('Tests\Fixtures\Suites\SuccessOnly')->classname->toBe('Tests.Fixtures.Suites.SuccessOnly')->assertions->toBe('1')->time->toStartWith('0.0');
});
+4 -4
View File
@@ -24,13 +24,13 @@ test('parallel', function () use ($run): void {
$file = file_get_contents(__FILE__);
$file = preg_replace(
'/\$expected = \'.*?\';/',
"\$expected = '1 deprecated, 4 warnings, 5 incomplete, 3 notices, 40 todos, 27 skipped, 1530 passed (3328 assertions)';",
"\$expected = '1 deprecated, 4 warnings, 5 incomplete, 3 notices, 40 todos, 27 skipped, 1531 passed (3334 assertions)';",
$file,
);
file_put_contents(__FILE__, $file);
}
$expected = '1 deprecated, 4 warnings, 5 incomplete, 3 notices, 40 todos, 27 skipped, 1530 passed (3328 assertions)';
$expected = '1 deprecated, 4 warnings, 5 incomplete, 3 notices, 40 todos, 27 skipped, 1531 passed (3334 assertions)';
expect($output)
->toContain("Tests: {$expected}")
@@ -42,7 +42,7 @@ test('a parallel test can extend another test with same name', function () use (
})->skipOnWindows();
test('parallel reports invalid datasets as failures', function () use ($run): void {
expect($run('tests/.tests/ParallelInvalidDataset'))
expect($run('tests/Fixtures/Suites/ParallelInvalidDataset'))
->toContain("A dataset with the name `missing.dataset` does not exist. You can create it using `dataset('missing.dataset', ['a', 'b']);`.")
->toContain('Tests: 1 failed, 1 passed (1 assertions)')
->toContain('Parallel: 3 processes');
@@ -60,7 +60,7 @@ test('parallel can have multiple exclude-groups', function () use ($run): void {
})->skipOnWindows();
test('parallel can have multiple groups', function () use ($run): void {
$output = $run('tests/.tests/MultipleGroups', '--group=one', '--group=two');
$output = $run('tests/Fixtures/Suites/MultipleGroups', '--group=one', '--group=two');
expect($output)
->toContain('Tests: 2 passed (2 assertions)')
+1 -1
View File
@@ -27,7 +27,7 @@ test('allows to run a single test', function () use ($run, $snapshot): void {
})->skipOnWindows();
test('allows to run a directory', function () use ($run, $snapshot): void {
expect($run('tests/Fixtures'))->toContain($snapshot('allows-to-run-a-directory'));
expect($run('tests/Fixtures/DirectoryWithTests'))->toContain($snapshot('allows-to-run-a-directory'));
})->skipOnWindows();
it('disable decorating printer when colors is set to never', function () use ($snapshot): void {
+1 -1
View File
@@ -10,7 +10,7 @@ function normalize_windows_os_output(string $text): string
}
test('visual snapshot of team city', function (string $testFile): void {
$testsPath = dirname(__DIR__)."/.tests/$testFile";
$testsPath = dirname(__DIR__)."/Fixtures/Suites/$testFile";
$snapshot = implode(DIRECTORY_SEPARATOR, [
dirname(__DIR__),
+2 -2
View File
@@ -8,7 +8,7 @@ test('filter works with unicode characters in filename', function (): void {
$process = new Process([
'php',
'bin/pest',
'tests/.tests/StraßenTest.php',
'tests/Fixtures/Suites/StraßenTest.php',
'--colors=never',
], dirname(__DIR__, 2), ['COLLISION_PRINTER' => 'DefaultPrinter', 'COLLISION_IGNORE_DURATION' => 'true', 'PAO_DISABLE' => '1']);
@@ -26,7 +26,7 @@ test('filter with unicode regex matches unicode filename', function (): void {
'php',
'bin/pest',
'--filter=.*Straß.*',
'tests/.tests/',
'tests/Fixtures/Suites/',
'--colors=never',
], dirname(__DIR__, 2), ['COLLISION_PRINTER' => 'DefaultPrinter', 'COLLISION_IGNORE_DURATION' => 'true', 'PAO_DISABLE' => '1']);