From b795af3b1b0a20b69d73814dd9c77f7217dc055c Mon Sep 17 00:00:00 2001 From: nuno maduro Date: Thu, 6 Aug 2026 17:52:04 +0100 Subject: [PATCH] wip --- CLAUDE.md | 16 ++ tests/Features/Tia/CompleteRunWriteTier.php | 209 ++++++++++++++++++ .../Features/Tia/DefaultBranchResolution.php | 4 +- tests/Features/Tia/DefaultBranchWriteTier.php | 37 ++-- tests/Features/Tia/FilteredMode.php | 83 +++++++ tests/Features/Tia/PartialRunWriteTier.php | 169 ++++++++++++++ tests/Fixtures/Tia/Project.php | 13 ++ 7 files changed, 509 insertions(+), 22 deletions(-) create mode 100644 tests/Features/Tia/CompleteRunWriteTier.php create mode 100644 tests/Features/Tia/FilteredMode.php create mode 100644 tests/Features/Tia/PartialRunWriteTier.php diff --git a/CLAUDE.md b/CLAUDE.md index afc98c89..ee13752e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,3 +16,19 @@ composer test:integration # visual and snapshot tests composer test # everything CI runs, in CI's order composer update:snapshots # only when a test was added or removed ``` + +## TIA scenario tests + +`tests/Features/Tia/*` scaffold a throwaway git project, run a real `pest` subprocess against it, and diff the TIA graph it wrote. They exist because TIA's contract is about what a run *writes* — replay, branch keys, and the COMPLETE / RESULTS-ONLY / HARD-SUPPRESSED tiers are invisible to ordinary assertions, and every case used to be measured by hand against a playground app. + +Add one whenever a change touches branch resolution, replay, filtered mode, or the write tiers. How: + +- `Project::make('master')` scaffolds; `seed('master')` writes a graph and sentinels every cached result (`time=9.999`, `assertions=42`) so any rewrite shows up. +- `$project->pest('--tia', …)` runs it; `$project->delta()` compares against that snapshot. `writtenCount()` is the discriminator — `0` means "replayed", not "wrote the same values". `mutateGraph()` bends one entry; overlays in `tests/Fixtures/Tia/overlays//` supply a different `tests/Pest.php`. +- Keep expectations driver-independent: a cold recording run needs pcov/xdebug and behaves differently without one. Seed a graph instead of recording one. + +Run them by file (a directory argument finds nothing) or by `--filter`: + +```bash +php bin/pest tests/Features/Tia/PartialRunWriteTier.php +``` diff --git a/tests/Features/Tia/CompleteRunWriteTier.php b/tests/Features/Tia/CompleteRunWriteTier.php new file mode 100644 index 00000000..71d53f28 --- /dev/null +++ b/tests/Features/Tia/CompleteRunWriteTier.php @@ -0,0 +1,209 @@ +seed('master'); + + $project->write('tests/Unit/GreeterTest.php', <<<'PHP' + greet('Nuno'))->toBe('Hello, Nuno!'); + }); + PHP); + + $result = $project->pest(); + $delta = $project->delta(); + + expect($result->tally())->toContain('5 passed') + ->and($delta->removed())->toBe(1, $delta->summary()) + ->and($delta->added())->toBe(0, $delta->summary()) + ->and($delta->structureMoved())->toBeFalse($delta->summary()); +})->skipOnWindows(); + +test('a complete run records nothing for a test file the graph does not know', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/BrandNewTest.php', <<<'PHP' + toBeTrue(); + }); + PHP); + + $result = $project->pest(); + $delta = $project->delta(); + + expect($result->tally())->toContain('7 passed') + ->and($delta->added())->toBe(0, $delta->summary()) + ->and($delta->removed())->toBe(0, $delta->summary()) + ->and($delta->writtenCount())->toBe(Project::TOTAL_TESTS, $delta->summary()); +})->skipOnWindows(); + +test('a partial run records nothing for a test file the graph does not know', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/BrandNewTest.php', <<<'PHP' + toBeTrue(); + }); + PHP); + + $result = $project->pest('--filter=brand new thing'); + $delta = $project->delta(); + + expect($result->tally())->toContain('1 passed') + ->and($delta->added())->toBe(0, $delta->summary()) + ->and($delta->writtenCount())->toBe(0, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('a truncated run does not prune', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/GreeterTest.php', <<<'PHP' + greet('Nuno'))->toBe('Goodbye, Nuno!'); + }); + + test('greets the world', function (): void { + expect((new Greeter)->greet('world'))->toBe('Hello, world!'); + }); + PHP); + + $result = $project->pest('--bail'); + $delta = $project->delta(); + + expect($result->exitCode)->toBe(1, $result->describe()) + ->and($result->tally())->toContain('1 failed') + ->and($delta->removed())->toBe(0, $delta->summary()) + ->and($delta->structureMoved())->toBeFalse($delta->summary()); +})->skipOnWindows(); + +test('a green bail run is complete', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--bail'); + $delta = $project->delta(); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($delta->writtenCount())->toBe(Project::TOTAL_TESTS, $delta->summary()) + ->and($delta->removed())->toBe(0, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('--no-tia refreshes results without enabling tia', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--tia', '--no-tia'); + $delta = $project->delta(); + + expect($result->output)->not->toContain('Experimental TIA mode enabled') + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed') + ->and($delta->writtenCount())->toBe(Project::TOTAL_TESTS, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('a run that never enables tia creates no graph', function (array $arguments): void { + $project = Project::make('master'); + + $result = $project->pest(...$arguments); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($project->graphExists())->toBeFalse(); +})->with([ + 'plain' => [[]], + 'filtered' => [['--filter=adds two numbers']], +])->skipOnWindows(); + +test('a test edit narrows to the affected file and replays the rest', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/GreeterTest.php', <<<'PHP' + greet('Nuno'))->toBe('Hello, Nuno!'); + expect((new Greeter)->greet('Nuno'))->toBeString(); + }); + + test('greets the world', function (): void { + expect((new Greeter)->greet('world'))->toBe('Hello, world!'); + }); + PHP); + + $result = $project->pest('--tia'); + $delta = $project->delta(); + + expect($result->affected())->toBe(2, $result->describe()) + ->and($result->replayed())->toBe(4, $result->describe()) + ->and($delta->writtenCount())->toBe(2, $delta->summary()) + ->and($delta->edgesMoved())->toBeFalse($delta->summary()) + ->and($delta->removed())->toBe(0, $delta->summary()); +})->skipOnWindows(); + +test('a parallel run merges worker results into the parent baseline', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/GreeterTest.php', <<<'PHP' + greet('Nuno'))->toBe('Hello, Nuno!'); + expect((new Greeter)->greet('Nuno'))->toBeString(); + }); + + test('greets the world', function (): void { + expect((new Greeter)->greet('world'))->toBe('Hello, world!'); + }); + PHP); + + $result = $project->pest('--tia', '--parallel', '--processes=2'); + $delta = $project->delta(); + + expect($result->affected())->toBe(2, $result->describe()) + ->and($result->replayed())->toBe(4, $result->describe()) + ->and($delta->writtenCount())->toBe(2, $delta->summary()) + ->and($delta->edgesMoved())->toBeFalse($delta->summary()) + ->and($delta->removed())->toBe(0, $delta->summary()); +})->skipOnWindows(); diff --git a/tests/Features/Tia/DefaultBranchResolution.php b/tests/Features/Tia/DefaultBranchResolution.php index b30485b2..640dd8fd 100644 --- a/tests/Features/Tia/DefaultBranchResolution.php +++ b/tests/Features/Tia/DefaultBranchResolution.php @@ -54,6 +54,7 @@ test('the CI provider names the default branch where the checkout cannot', funct $project = Project::make('master'); $project->git()->unsetOriginHead(); $project->seed('master'); + $project->addBaseline('legacy'); $project->git()->switchTo('feature-x', new: true); @@ -73,6 +74,7 @@ test('GitLab names the default branch through its own variable', function (): vo $project = Project::make('master'); $project->git()->unsetOriginHead(); $project->seed('master'); + $project->addBaseline('legacy'); $project->git()->switchTo('feature-x', new: true); @@ -124,7 +126,7 @@ test('an init.defaultBranch naming a branch that exists is still trusted', funct expect($result->exitCode)->toBe(0, $result->describe()) ->and($result->output)->not->toContain('could not determine the default branch') - ->and($project->branchKeys())->toBe(['feature-x']); + ->and($project->branchKeys())->not->toContain('master'); })->skipOnWindows(); test('a repository with no remote is refused rather than silently re-run', function (): void { diff --git a/tests/Features/Tia/DefaultBranchWriteTier.php b/tests/Features/Tia/DefaultBranchWriteTier.php index dee41d6b..75e9d985 100644 --- a/tests/Features/Tia/DefaultBranchWriteTier.php +++ b/tests/Features/Tia/DefaultBranchWriteTier.php @@ -14,37 +14,32 @@ test('narrows to the affected tests on a new branch', function (): void { $project->git()->switchTo('feature-x', new: true); - $project->write('app/Calculator.php', <<<'PHP' + $project->write('tests/Unit/GreeterTest.php', <<<'PHP' greet('Nuno'))->toBe('Hello, Nuno!'); + expect((new Greeter)->greet('Nuno'))->toBeString(); + }); - public function subtract(int $a, int $b): int - { - return $a - $b; - } - - public function multiply(int $a, int $b): int - { - return $a * $b; - } - } + test('greets the world', function (): void { + expect((new Greeter)->greet('world'))->toBe('Hello, world!'); + }); PHP); $result = $project->pest('--tia'); + $delta = $project->delta(); - expect($result->affected())->toBe(4, $result->describe()) - ->and($result->replayed())->toBe(2, $result->describe()) - ->and($result->exitCode)->toBe(0, $result->describe()); + expect($result->affected())->toBe(2, $result->describe()) + ->and($result->replayed())->toBe(4, $result->describe()) + ->and($result->exitCode)->toBe(0, $result->describe()) + ->and($project->branchKeys())->toBe(['master', 'feature-x']) + ->and($delta->baselineUntouched('master'))->toBeTrue($delta->summary()) + ->and($delta->edgesMoved())->toBeFalse($delta->summary()); })->skipOnWindows(); test('filtered mode reads the fallback too', function (): void { diff --git a/tests/Features/Tia/FilteredMode.php b/tests/Features/Tia/FilteredMode.php new file mode 100644 index 00000000..3844f855 --- /dev/null +++ b/tests/Features/Tia/FilteredMode.php @@ -0,0 +1,83 @@ +seed('master', failing: ['adds two numbers']); + + $result = $project->pest('--tia', '--filtered'); + $delta = $project->delta(); + + expect($result->output)->toContain('from 1 previously unsuccessful test') + ->and($result->affected())->toBe(2, $result->describe()) + ->and($result->tally())->toContain('2 passed') + ->and($delta->writtenCount())->toBe(2, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('an explicit path turns filtered mode off', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--tia', '--filtered', 'tests/Unit'); + $delta = $project->delta(); + + expect($result->output)->toContain('TIA does not apply to partial runs') + ->and($result->output)->not->toContain('No affected tests found') + ->and($result->tally())->toContain('4 passed') + ->and($delta->writtenCount())->toBe(4, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('filtered mode records a baseline when there is none', function (): void { + $project = Project::make('master'); + + $result = $project->pest('--tia', '--filtered'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed') + ->and($result->output)->not->toContain('No affected tests found'); +})->skipOnWindows(); + +test('filtered mode finds nothing to do in parallel either', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--tia', '--filtered', '--parallel', '--processes=2'); + $delta = $project->delta(); + + expect($result->output)->toContain('No affected tests found') + ->and($result->exitCode)->toBe(0, $result->describe()) + ->and($delta->isHardSuppressed())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('a corrupt graph is rebuilt rather than crashing the run', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + file_put_contents($project->graphDir().'/graph.json', '{not json'); + + $result = $project->pest('--tia'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed'); +})->skipOnWindows(); + +test('--parallel --retry is refused and leaves the graph alone', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--parallel', '--retry'); + $delta = $project->delta(); + + expect($result->exitCode)->not->toBe(0) + ->and($result->output)->toContain('--retry') + ->and($delta->isHardSuppressed())->toBeTrue($delta->summary()); +})->skipOnWindows(); diff --git a/tests/Features/Tia/PartialRunWriteTier.php b/tests/Features/Tia/PartialRunWriteTier.php new file mode 100644 index 00000000..f0490555 --- /dev/null +++ b/tests/Features/Tia/PartialRunWriteTier.php @@ -0,0 +1,169 @@ +seed('master'); + + $result = $project->pest('--filter=adds two numbers'); + $delta = $project->delta(); + + expect($result->tally())->toContain('1 passed') + ->and($result->output)->not->toContain('TIA does not apply to partial runs') + ->and($delta->writtenCount())->toBe(1, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('a filtered run under --tia announces that tia does not apply', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--tia', '--filter=adds two numbers'); + $delta = $project->delta(); + + expect($result->output)->toContain('TIA does not apply to partial runs') + ->and($delta->writtenCount())->toBe(1, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('a test suffix narrows the tier even though every test runs', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--tia', '--test-suffix=Test.php'); + $delta = $project->delta(); + + expect($result->output)->toContain('TIA does not apply to partial runs') + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed') + ->and($delta->writtenCount())->toBe(Project::TOTAL_TESTS, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('a dirty run narrows to the uncommitted test edit', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/GreeterTest.php', <<<'PHP' + greet('Nuno'))->toBe('Hello, Nuno!'); + }); + + test('greets the world', function (): void { + expect((new Greeter)->greet('world'))->toBe('Hello, world!'); + }); + + test('greets again', function (): void { + expect((new Greeter)->greet('again'))->toBe('Hello, again!'); + }); + PHP); + + $result = $project->pest('--tia', '--dirty'); + $delta = $project->delta(); + + expect($result->output)->toContain('TIA does not apply to partial runs') + ->and($result->tally())->toContain('3 passed') + ->and($delta->removed())->toBe(0, $delta->summary()) + ->and($delta->structureMoved())->toBeFalse($delta->summary()); +})->skipOnWindows(); + +test('filtered mode yields to an explicit filter', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--tia', '--filtered', '--filter=adds two numbers'); + $delta = $project->delta(); + + expect($result->output)->toContain('TIA does not apply to partial runs') + ->and($result->tally())->toContain('1 passed') + ->and($delta->writtenCount())->toBe(1, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('an env flag narrows exactly like the option it mirrors', function (string $variable): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pestWithEnvironment($project->path(), [ + $variable => '1', + ], '--filter=adds two numbers'); + + $delta = $project->delta(); + + expect($result->output)->toContain('TIA does not apply to partial runs') + ->and($delta->writtenCount())->toBe(1, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->with(['PEST_TIA', 'PEST_TIA_FILTERED'])->skipOnWindows(); + +test('a partial run does not purge the graph even with --fresh', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--tia', '--fresh', '--filter=adds two numbers'); + $delta = $project->delta(); + + expect($result->output)->toContain('TIA does not apply to partial runs') + ->and($delta->writtenCount())->toBe(1, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('--no-tia does not stop a partial run from refreshing its own entry', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--no-tia', '--filter=adds two numbers'); + $delta = $project->delta(); + + expect($result->output)->not->toContain('TIA does not apply to partial runs') + ->and($delta->writtenCount())->toBe(1, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('two partial runs each keep the other entry', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->pest('--filter=adds two numbers'); + $project->pest('--filter=greets a person'); + + $delta = $project->delta(); + + expect($delta->writtenCount())->toBe(2, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('a shard is a partial run', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--tia', '--shard=1/2'); + $delta = $project->delta(); + + expect($result->output)->toContain('TIA does not apply to partial runs') + ->and($delta->removed())->toBe(0, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('a parallel partial run writes nothing at all', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--tia', '--parallel', '--processes=2', '--filter=adds two numbers'); + $delta = $project->delta(); + + expect($result->output)->toContain('TIA does not apply to partial runs') + ->and($delta->writtenCount())->toBe(0, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); diff --git a/tests/Fixtures/Tia/Project.php b/tests/Fixtures/Tia/Project.php index e5d80bc2..2b1a4c0b 100644 --- a/tests/Fixtures/Tia/Project.php +++ b/tests/Fixtures/Tia/Project.php @@ -259,6 +259,19 @@ final class Project }); } + /** + * Adds a second, empty baseline key, so a lone recorded baseline can no + * longer stand in for the default branch. + */ + public function addBaseline(string $branch): void + { + $this->mutateGraph(function (array $graph) use ($branch): array { + $graph['baselines'][$branch] = ['sha' => null, 'tree' => [], 'results' => []]; + + return $graph; + }); + } + /** * @param callable(array): array $callback */