diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ce6734d1..511a653d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -75,3 +75,70 @@ jobs: - name: Integration Tests run: composer test:integration + + tia: + if: github.event_name != 'schedule' || github.repository == 'pestphp/pest' + + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: true + matrix: + php: ['8.4'] + coverage: [xdebug] + + name: TIA - PHP ${{ matrix.php }} - ${{ matrix.coverage }} - ubuntu-latest + + # The `tests` matrix runs with `coverage: none`, which leaves half of TIA + # untested: recording a dependency graph needs a coverage driver, and the + # scenarios fall back to a seeded graph without one. This job is the other + # half — and the only place the driver-specific recording code runs at all. + # + # Off for *this* suite, on for the subprocesses each scenario spawns (see + # `Project::pestWithEnvironment()`): they are the ones that record, and + # collecting Pest's own ~2000 tests under xdebug costs more than every + # scenario put together. + env: + XDEBUG_MODE: 'off' + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup PHP + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2 + with: + php-version: ${{ matrix.php }} + tools: composer:v2 + coverage: ${{ matrix.coverage }} + extensions: sockets + + - name: Get Composer cache directory + id: composer-cache + shell: bash + run: echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT + + - name: Cache Composer dependencies + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 + with: + path: ${{ steps.composer-cache.outputs.dir }} + key: ubuntu-latest-php-${{ matrix.php }}-composer-${{ hashFiles('**/composer.json', '**/composer.lock') }} + restore-keys: | + ubuntu-latest-php-${{ matrix.php }}-composer- + + - name: Install PHP dependencies + shell: bash + env: + COMPOSER_ROOT_VERSION: 5.x-dev + run: composer update --prefer-stable --no-interaction --no-progress --ansi + + # A missing driver would not fail these rows — they fall back to a seeded + # graph and stay green — so the job has to check for itself that the thing + # it exists to exercise is present, and that a scenario subprocess gets it. + - name: Assert xdebug is present, and reaches a scenario subprocess + run: | + php -r 'exit(extension_loaded("xdebug") ? 0 : 1);' + XDEBUG_MODE=coverage php -r 'exit(in_array("coverage", (array) xdebug_info("mode"), true) ? 0 : 1);' + + - name: TIA Scenario Tests + run: composer test:tia diff --git a/composer.json b/composer.json index 1da65caa..cf9f602d 100644 --- a/composer.json +++ b/composer.json @@ -94,6 +94,7 @@ "test:inline": "php bin/pest --configuration=phpunit.inline.xml", "test:parallel": "php bin/pest --exclude-group=integration --parallel --processes=3", "test:integration": "php bin/pest --group=integration -v", + "test:tia": "php bin/pest --group=tia -v", "update:snapshots": "REBUILD_SNAPSHOTS=true php bin/pest --update-snapshots", "test": [ "@test:lint", diff --git a/src/Exceptions/TiaRequiresCommit.php b/src/Exceptions/TiaRequiresCommit.php new file mode 100644 index 00000000..0ebaf576 --- /dev/null +++ b/src/Exceptions/TiaRequiresCommit.php @@ -0,0 +1,49 @@ +writeln([ + '', + ' ERROR Tia mode requires at least one commit.', + '', + ' A baseline is anchored to the revision it was recorded at, and this repository', + ' has none yet, so there is nothing to record against and nothing to compare a', + ' later run to.', + '', + ' Commit once, then run again:', + '', + ' git add . && git commit -m "Initial commit"', + '', + ' Runs without --tia are unaffected.', + '', + ]); + } + + public function exitCode(): int + { + return 1; + } +} diff --git a/src/Plugins/Tia.php b/src/Plugins/Tia.php index 95625cb2..b43a76d9 100644 --- a/src/Plugins/Tia.php +++ b/src/Plugins/Tia.php @@ -12,6 +12,7 @@ use Pest\Contracts\Plugins\Terminable; use Pest\Exceptions\InvalidOption; use Pest\Exceptions\MissingDependency; use Pest\Exceptions\NoAffectedTestsFound; +use Pest\Exceptions\TiaRequiresCommit; use Pest\Exceptions\TiaRequiresDefaultBranch; use Pest\Exceptions\TiaRequiresRemote; use Pest\Exceptions\TiaRequiresRepositoryRoot; @@ -210,6 +211,8 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument private bool $detachedHead = false; + private bool $graphUnreachable = false; + /** @var array */ private array $originalArguments = []; @@ -692,7 +695,17 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument return $exitCode; } - if ($this->replayRan) { + // Re-anchor the baseline. Reaching here means the run was complete — + // nothing suppressed, narrowed or truncated it — so its results are the + // truth at HEAD and the recorded revision may say so. + // + // That matters most when the recorded commit had become unreachable (a + // rebase, a force-push) and no coverage driver was available to rebuild: + // without this the stale revision survives, and every later run warns + // and re-runs the whole suite, for good. Stale edges are no objection — + // a complete run just re-recorded every result, and later changes are + // compared against the revision written here. + if ($this->replayRan || $this->graphUnreachable) { $this->bumpRecordedSha(); } @@ -836,7 +849,20 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument Panic::with(new TiaRequiresRepositoryRoot($subdirectoryPrefix)); } - $this->resolveBranch($projectRoot); + try { + $this->resolveBranch($projectRoot); + } catch (MissingDependency $missingGit) { + // Every git call TIA makes fails on `HEAD` in a repository that has + // no commits yet, which reads as "git is missing" when git is right + // there. Say what is actually wrong instead. + $repository = new ChangedFiles($projectRoot); + + if ($repository->isRepository() && ! $repository->hasCommits()) { + Panic::with(new TiaRequiresCommit); + } + + throw $missingGit; + } if (! $this->fallbackBranchResolved) { Panic::with(new ChangedFiles($projectRoot)->hasRemote() @@ -865,6 +891,7 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument && $changedFiles->since($branchSha) === null) { $this->renderBadge('WARN', 'Recorded commit is no longer reachable — graph will be rebuilt.'); $graph = null; + $this->graphUnreachable = true; } } @@ -886,8 +913,23 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument $this->state->write(self::KEY_COVERAGE_MARKER, ''); } + // An active coverage report owns the driver, so edges have to be + // piggybacked off its session — and that session is scoped to + // phpunit.xml's , not to the whole project. Refreshing an + // existing graph that way is safe (`replaceEdges()` keeps what it + // already has), but *founding* one on it is not: every source file + // outside the coverage scope would be missing from the graph for good, + // and a change to one of them would select nothing and replay a pass. + if (! $graph instanceof Graph && $this->piggybackCoverage) { + $this->emitCoverageScopedRecordSkipped(); + + return $arguments; + } + + // Past the guard above, a coverage-owned run always has a graph to + // refresh — a run without one never gets here. if ($coverageCacheOwned && ! $this->state->exists(self::KEY_COVERAGE_CACHE)) { - if ($graph instanceof Graph && $this->driftLabel === null) { + if ($this->driftLabel === null) { $this->freshGraphReason = 'recording a coverage baseline'; } @@ -1042,7 +1084,7 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument return $arguments; } - $affectedFromChanges = $changed === [] ? [] : $graph->affected($changed); + $affectedFromChanges = $changed === [] ? [] : $graph->testFilesOnDisk($graph->affected($changed)); $rerunFromCache = []; if ($this->filteredMode && $graph->hasUnlocatedTestsToRerun($this->branch)) { @@ -1295,6 +1337,14 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument $this->renderChild('Running in TIA mode, however TIA is skipped as it needs ext-pcov or Xdebug.'); } + private function emitCoverageScopedRecordSkipped(): void + { + $this->output->writeln(''); + + $this->renderChild('Running in TIA mode, however TIA is skipped as an active coverage report narrows the edges it could record.'); + $this->renderChild('Record the baseline with a plain --tia run first; coverage runs then reuse it.'); + } + /** * @param array> $perTestFiles * @param array> $perTestTables diff --git a/src/Plugins/Tia/ChangedFiles.php b/src/Plugins/Tia/ChangedFiles.php index ea442f89..35c1809a 100644 --- a/src/Plugins/Tia/ChangedFiles.php +++ b/src/Plugins/Tia/ChangedFiles.php @@ -298,6 +298,31 @@ final readonly class ChangedFiles return $this->gitOutput(['git', 'remote']) !== null; } + public function isRepository(): bool + { + $process = new Process(['git', 'rev-parse', '--git-dir'], $this->projectRoot); + $process->setTimeout(5.0); + $process->run(); + + return $process->getExitCode() === 0; + } + + /** + * Whether this repository has a revision to anchor a baseline to. + * + * A freshly initialised repository has none, and every other git call TIA + * makes — {@see self::currentBranch()}, {@see self::currentSha()} — fails on + * `HEAD` there and reports git as missing, which it is not. + */ + public function hasCommits(): bool + { + $process = new Process(['git', 'rev-parse', '--verify', '--quiet', 'HEAD'], $this->projectRoot); + $process->setTimeout(5.0); + $process->run(); + + return $process->getExitCode() === 0; + } + /** * @param array $command */ @@ -332,8 +357,12 @@ final readonly class ChangedFiles */ private function diffSinceSha(string $sha): array { + // `--no-renames` matters: with rename detection on, git reports only the + // destination of a moved file, so the path the graph has edges for — the + // one that is gone — never reaches selection, and every test that + // depended on it replays its recorded pass. $process = new Process( - ['git', 'diff', '--name-only', $sha.'..HEAD'], + ['git', 'diff', '--name-only', '--no-renames', $sha.'..HEAD'], $this->projectRoot, ); $process->run(); diff --git a/src/Plugins/Tia/Graph.php b/src/Plugins/Tia/Graph.php index ad25d6ab..bf2c2da5 100644 --- a/src/Plugins/Tia/Graph.php +++ b/src/Plugins/Tia/Graph.php @@ -119,6 +119,32 @@ final class Graph return array_keys($affectedSet); } + /** + * Keep only the test files this checkout actually has. + * + * A stale edge key — a test file a fetched baseline knew, or one another + * branch deleted — cannot be run by anyone, and selecting it strands + * `--filtered` on a run that matches nothing and reports success on a + * change no test looked at. {@see self::testFilesToRerun()} has always + * dropped these; the change-driven half of selection must agree. + * + * @param array $testFiles Project-relative paths. + * @return list + */ + public function testFilesOnDisk(array $testFiles): array + { + $root = rtrim($this->projectRoot, DIRECTORY_SEPARATOR).DIRECTORY_SEPARATOR; + $onDisk = []; + + foreach ($testFiles as $testFile) { + if (is_file($root.$testFile)) { + $onDisk[] = $testFile; + } + } + + return $onDisk; + } + /** * @param array $changedFiles * @return array{0: list, 1: list} diff --git a/src/Plugins/Tia/Recorder.php b/src/Plugins/Tia/Recorder.php index c9e71870..e1156900 100644 --- a/src/Plugins/Tia/Recorder.php +++ b/src/Plugins/Tia/Recorder.php @@ -359,8 +359,19 @@ final class Recorder continue; } + // A file whose *only* executed line is its last one was included, + // not used — the trailing line of an include is all that ran. + // + // That reading only holds for a driver that reports unexecuted + // lines too: pcov returns every executable line (`-1` for the ones + // that did not run), so "the single covered line is the highest + // line reported" means something. Xdebug reports executed lines + // only, where it is true of *any* file that ran a single line — + // which is most of them, and dropping those loses the edge. $lineKeys = array_keys($lines); - if ($lineKeys !== [] && count($covered) === 1 && $covered[0] === max($lineKeys)) { + $reportsUnexecutedLines = count($covered) < count($lines); + + if ($reportsUnexecutedLines && $lineKeys !== [] && count($covered) === 1 && $covered[0] === max($lineKeys)) { continue; } diff --git a/tests/Features/Tia/BranchShapes.php b/tests/Features/Tia/BranchShapes.php index 86f1b1b8..d608cc8b 100644 --- a/tests/Features/Tia/BranchShapes.php +++ b/tests/Features/Tia/BranchShapes.php @@ -159,3 +159,48 @@ test('the default branch baseline survives every branch that comes and goes', fu ->and($delta->removed())->toBe(0, $delta->summary()) ->and($project->graph()['baselines']['master']['results'])->toHaveCount(Project::TOTAL_TESTS); })->skipOnWindows(); + +test('a project below the git repository root refuses to run and writes nothing', function (array $arguments): void { + $project = Project::make('master'); + $nested = $project->nested(); + + // git addresses paths from the repository root while the graph is + // project-relative, so the two have to coincide. TIA says so and stops. + $result = $project->pestIn($nested, '--tia', ...$arguments); + + expect($result->exitCode)->toBe(1, $result->describe()) + ->and($result->output)->toContain('Tia mode requires the git repository root') + ->and(is_dir($project->path('.home/.pest')))->toBeFalse() + ->and(is_dir($nested.DIRECTORY_SEPARATOR.'.pest'))->toBeFalse(); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('a repository with no commits says so, and leaves plain runs alone', function (): void { + $project = Project::withoutGit(); + $project->git()->run(['init', '--quiet']); + $project->git()->run(['checkout', '--quiet', '-b', 'master']); + $project->git()->addOrigin(); + + // Every git call TIA makes asks about HEAD, which does not exist yet. That + // used to surface as `requires "git"`, with git installed and working. + $tia = $project->pest('--tia'); + + expect($tia->exitCode)->toBe(1, $tia->describe()) + ->and($tia->output)->toContain('Tia mode requires at least one commit') + ->and($tia->output)->not->toContain('requires "git"') + ->and($project->graphExists())->toBeFalse(); + + $plain = $project->pest(); + + expect($plain->exitCode)->toBe(0, $plain->describe()) + ->and($plain->tally())->toContain(Project::TOTAL_TESTS.' passed'); +})->skipOnWindows(); + +test('a directory with no repository at all still asks for git', function (): void { + $project = Project::withoutGit(); + + $result = $project->pest('--tia'); + + expect($result->exitCode)->toBe(1, $result->describe()) + ->and($result->output)->toContain('requires "git"') + ->and($project->graphExists())->toBeFalse(); +})->skipOnWindows(); diff --git a/tests/Features/Tia/CoveragePiggyback.php b/tests/Features/Tia/CoveragePiggyback.php new file mode 100644 index 00000000..59711ac7 --- /dev/null +++ b/tests/Features/Tia/CoveragePiggyback.php @@ -0,0 +1,70 @@ +, not to the whole project. Edges recorded that way + * are missing every source file outside that scope, and a change to one of them + * would select nothing and replay a pass. Invariant 3 at its most dangerous. + * + * The rows below assert what a coverage run may and may not leave behind. They + * are deliberately silent about exit codes and result counts: `--coverage` + * itself fails on an interpreter with no driver, so only the graph's fate is + * the same everywhere. + */ + +test('a coverage report does not found a dependency graph', function (array $arguments): void { + $project = Project::make('master'); + + $project->pest('--tia', ...$arguments); + + expect($project->graphExists())->toBeFalse(); +})->with([ + 'pest coverage' => [['--coverage']], + 'phpunit coverage report' => [['--coverage-text']], + 'parallel' => [['--coverage', '--parallel', '--processes=2']], +])->skipOnWindows(); + +test('a plain run after a coverage run records the whole project scope', function (): void { + $project = Project::make('master'); + + $project->pest('--tia', '--coverage'); + $project->pest('--tia'); + + $graph = $project->graph(); + + // Nothing to assert without a driver: there is no graph either way, and the + // point of the row is that the *plain* run is the one that founds it. + if ($graph === null) { + expect($project->graphExists())->toBeFalse(); + + return; + } + + // Self-edges included — they are the first thing a coverage-scoped + // recording drops, since test files are not in . + expect(array_keys($graph['edges']))->toEqualCanonicalizing(array_keys(Project::EDGES)) + ->and($graph['files'])->toContain('tests/Unit/CalculatorTest.php') + ->and($graph['files'])->toContain('app/Calculator.php'); +})->skipOnWindows(); + +test('a coverage report leaves the edges of an existing graph alone', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->pest('--tia', '--coverage'); + $delta = $project->delta(); + + expect($delta->edgesMoved())->toBeFalse($delta->summary()) + ->and($delta->filesMoved())->toBeFalse($delta->summary()) + ->and($delta->removed())->toBe(0, $delta->summary()) + ->and($delta->added())->toBe(0, $delta->summary()); +})->skipOnWindows(); diff --git a/tests/Features/Tia/RemoteBaseline.php b/tests/Features/Tia/RemoteBaseline.php new file mode 100644 index 00000000..49a4a06c --- /dev/null +++ b/tests/Features/Tia/RemoteBaseline.php @@ -0,0 +1,154 @@ +): array|null $mutator + * @return array{0: Project, 1: array} + */ +function tiaPublishedBaseline(string $mode = 'ok', ?callable $mutator = null): array +{ + $project = Project::make('master'); + $project->seed('master'); + + $payload = $project->detachGraph(); + + if ($mutator !== null) { + /** @var array $decoded */ + $decoded = json_decode($payload, true); + $payload = (string) json_encode($mutator($decoded), JSON_UNESCAPED_SLASHES); + } + + return [$project, $project->gh($mode, $payload)]; +} + +test('a published baseline is fetched instead of recorded locally', function (): void { + [$project, $environment] = tiaPublishedBaseline(); + + $result = $project->pestWithEnvironment($project->path(), $environment, '--tia', '--baselined'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->output)->toContain('Downloading TIA baseline') + ->and($result->replayed())->toBe(Project::TOTAL_TESTS, $result->describe()) + ->and($project->graphExists())->toBeTrue(); +})->skipOnWindows(); + +test('a fetched baseline that will not decode is discarded rather than trusted', function (): void { + [$project, $environment] = tiaPublishedBaseline('corrupt'); + + $result = $project->pestWithEnvironment($project->path(), $environment, '--tia', '--baselined'); + + // Nothing may be replayed out of it. Whether the run then records a graph + // of its own depends on the coverage driver, so that is not asserted here. + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->output)->toContain('The dependency graph could not be read') + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed') + ->and($result->replayed())->toBe(0, $result->describe()); +})->skipOnWindows(); + +test('a fetched baseline recorded against another tree is not used', function (): void { + [$project, $environment] = tiaPublishedBaseline('ok', function (array $graph): array { + $graph['fingerprint']['structural']['composer_lock'] = 'a-lockfile-this-project-never-had'; + + return $graph; + }); + + $result = $project->pestWithEnvironment($project->path(), $environment, '--tia', '--baselined'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed') + ->and($result->replayed())->toBe(0, $result->describe()); +})->skipOnWindows(); + +test('an artifact without a graph in it fails loudly', function (): void { + [$project, $environment] = tiaPublishedBaseline('missing-asset'); + + $result = $project->pestWithEnvironment($project->path(), $environment, '--tia', '--baselined'); + + expect($result->exitCode)->toBe(1, $result->describe()) + ->and($result->output)->toContain('the artifact is missing expected files') + ->and($project->graphExists())->toBeFalse(); +})->skipOnWindows(); + +test('a baseline that cannot be authenticated for fails loudly', function (): void { + [$project, $environment] = tiaPublishedBaseline('unauthenticated'); + + $result = $project->pestWithEnvironment($project->path(), $environment, '--tia', '--baselined'); + + expect($result->exitCode)->toBe(1, $result->describe()) + ->and($result->output)->toContain('is not authenticated') + ->and($project->graphExists())->toBeFalse(); +})->skipOnWindows(); + +test('a workflow or artifact that is not there fails loudly', function (): void { + [$project, $environment] = tiaPublishedBaseline('list-404'); + + $result = $project->pestWithEnvironment($project->path(), $environment, '--tia', '--baselined'); + + expect($result->exitCode)->toBe(1, $result->describe()) + ->and($result->output)->toContain('not found in repo') + ->and($project->graphExists())->toBeFalse(); +})->skipOnWindows(); + +test('a network failure warns and lets the suite run', function (string $mode): void { + [$project, $environment] = tiaPublishedBaseline($mode); + + $result = $project->pestWithEnvironment($project->path(), $environment, '--tia', '--baselined'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->output)->toContain('network error') + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed'); +})->with([ + 'querying the runs' => ['list-network'], + 'downloading the artifact' => ['download-network'], +])->skipOnWindows(); + +test('no published baseline yet starts a cooldown, and a corrupt cooldown does not break the run', function (): void { + [$project, $environment] = tiaPublishedBaseline('no-runs'); + + // On a machine with a coverage driver each run below records a graph of its + // own, and a run that has a graph never reaches the fetch at all. Take it + // away between runs, so what is under test is the cooldown and nothing else. + $discardGraph = function () use ($project): void { + if ($project->graphExists()) { + $project->detachGraph(); + } + }; + + $first = $project->pestWithEnvironment($project->path(), $environment, '--tia', '--baselined'); + + expect($first->exitCode)->toBe(0, $first->describe()) + ->and($first->output)->toContain('No baseline published yet') + ->and(is_file($project->graphDir().DIRECTORY_SEPARATOR.'fetch-cooldown.json'))->toBeTrue(); + + $discardGraph(); + + $second = $project->pestWithEnvironment($project->path(), $environment, '--tia', '--baselined'); + + expect($second->exitCode)->toBe(0, $second->describe()) + ->and($second->output)->toContain('next auto-retry in'); + + file_put_contents($project->graphDir().DIRECTORY_SEPARATOR.'fetch-cooldown.json', 'not json{'); + + $discardGraph(); + + $third = $project->pestWithEnvironment($project->path(), $environment, '--tia', '--baselined'); + + expect($third->exitCode)->toBe(0, $third->describe()) + ->and($third->output)->toContain('No baseline published yet') + ->and($third->tally())->toContain(Project::TOTAL_TESTS.' passed'); +})->skipOnWindows(); diff --git a/tests/Features/Tia/SelectionPaths.php b/tests/Features/Tia/SelectionPaths.php new file mode 100644 index 00000000..8106e448 --- /dev/null +++ b/tests/Features/Tia/SelectionPaths.php @@ -0,0 +1,233 @@ +seed('master'); + + $project->mutateGraph(function (array $graph) use ($view): array { + $id = count($graph['files']); + $graph['files'][$id] = $view; + $graph['edges']['tests/Unit/GreeterTest.php'][] = $id; + + return $graph; + }); +} + +/** + * @param array $components + * @param array> $jsFileToComponents + */ +function tiaSeedWithInertia(Project $project, array $components, array $jsFileToComponents = []): void +{ + $project->seed('master'); + + $project->mutateGraph(function (array $graph) use ($components, $jsFileToComponents): array { + $graph['test_inertia_components'] = ['tests/Unit/GreeterTest.php' => $components]; + $graph['js_file_to_components'] = $jsFileToComponents; + + return $graph; + }); +} + +test('a committed rename selects the tests that depended on the old path', function (array $arguments): void { + $project = Project::make('master'); + $project->write('resources/views/greeting.blade.php', "

Hello

\n"); + $project->git()->commit('add view'); + + tiaSeedWithView($project, 'resources/views/greeting.blade.php'); + + // git reports only the destination of a rename unless asked not to, so the + // path the graph holds an edge for is the one that must still show up. + $project->git()->run(['mv', 'resources/views/greeting.blade.php', 'resources/views/hello.blade.php']); + $project->git()->commit('move the view'); + $project->snapshot(); + + $result = $project->pest('--tia', ...$arguments); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->affected())->toBe(2, $result->describe()) + ->and($result->replayed())->toBe(4, $result->describe()); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('an affected test file that is gone does not strand a filtered run', function (array $arguments): void { + $project = Project::make('master'); + $project->write('resources/views/page.blade.php', "

one

\n"); + $project->git()->commit('add view'); + $project->seed('master'); + + // A graph written before this checkout existed — a fetched baseline, or a + // branch that deleted the file — can hold an edge for a test file nothing + // can run. Selecting it would filter the suite down to nothing and report + // success on a change no test looked at. + $project->mutateGraph(function (array $graph): array { + $id = count($graph['files']); + $graph['files'][$id] = 'resources/views/page.blade.php'; + $graph['edges']['tests/Unit/GhostTest.php'] = [$id]; + + return $graph; + }); + + $project->write('resources/views/page.blade.php', "

two

\n"); + $project->snapshot(); + + $result = $project->pest('--tia', '--filtered', ...$arguments); + $delta = $project->delta(); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->output)->toContain('No affected tests found') + ->and($result->output)->not->toContain('tests/Unit/GhostTest.php') + ->and($delta->isHardSuppressed())->toBeTrue($delta->summary()); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('a plain run reclaims the edge of a test file that is gone', function (): void { + $project = Project::make('master'); + $project->write('resources/views/page.blade.php', "

one

\n"); + $project->git()->commit('add view'); + $project->seed('master'); + + $project->mutateGraph(function (array $graph): array { + $id = count($graph['files']); + $graph['files'][$id] = 'resources/views/page.blade.php'; + $graph['edges']['tests/Unit/GhostTest.php'] = [$id]; + $graph['baselines']['master']['results']['P\\Tests\\Unit\\GhostTest::ghostly'] = [ + 'status' => 0, + 'message' => '', + 'time' => 9.999, + 'assertions' => 42, + 'file' => 'tests/Unit/GhostTest.php', + ]; + + return $graph; + }); + + $project->write('resources/views/page.blade.php', "

two

\n"); + $project->snapshot(); + + $result = $project->pest('--tia'); + $delta = $project->delta(); + + expect($result->replayed())->toBe(Project::TOTAL_TESTS, $result->describe()) + ->and($delta->removed())->toBe(1, $delta->summary()) + ->and($project->graph()['edges'])->not->toHaveKey('tests/Unit/GhostTest.php'); +})->skipOnWindows(); + +test('a changed view selects the test that rendered it', function (): void { + $project = Project::make('master'); + $project->write('resources/views/page.blade.php', "

one

\n"); + $project->git()->commit('add view'); + + tiaSeedWithView($project, 'resources/views/page.blade.php'); + + $project->write('resources/views/page.blade.php', "

two

\n"); + $project->snapshot(); + + $result = $project->pest('--tia'); + + expect($result->affected())->toBe(2, $result->describe()) + ->and($result->replayed())->toBe(4, $result->describe()); +})->skipOnWindows(); + +test('a changed partial selects the test that rendered its ancestor', function (array $views, string $changed): void { + $project = Project::make('master'); + + foreach ($views as $path => $contents) { + $project->write($path, $contents); + } + + $project->git()->commit('add views'); + + tiaSeedWithView($project, 'resources/views/page.blade.php'); + + $project->write($changed, $views[$changed]."edited\n"); + $project->snapshot(); + + $result = $project->pest('--tia'); + + expect($result->affected())->toBe(2, $result->describe()) + ->and($result->replayed())->toBe(4, $result->describe()); +})->with([ + 'direct @include' => [[ + 'resources/views/page.blade.php' => "@include('partials.nav')\n", + 'resources/views/partials/nav.blade.php' => "\n", + ], 'resources/views/partials/nav.blade.php'], + 'transitive @include' => [[ + 'resources/views/page.blade.php' => "@include('partials.wrapper')\n", + 'resources/views/partials/wrapper.blade.php' => "@include('partials.nav')\n", + 'resources/views/partials/nav.blade.php' => "\n", + ], 'resources/views/partials/nav.blade.php'], + 'x- component' => [[ + 'resources/views/page.blade.php' => "hi\n", + 'resources/views/components/card.blade.php' => "
one
\n", + ], 'resources/views/components/card.blade.php'], + // Two partials that include each other: the ancestor walk has to notice it + // has seen them and stop, rather than chase the cycle forever. + 'include cycle' => [[ + 'resources/views/page.blade.php' => "@include('partials.a')\n", + 'resources/views/partials/a.blade.php' => "@include('partials.b')\n", + 'resources/views/partials/b.blade.php' => "@include('partials.a')\n", + ], 'resources/views/partials/b.blade.php'], +])->skipOnWindows(); + +test('a changed Inertia page selects the test that rendered its component', function (): void { + $project = Project::make('master'); + $project->write('resources/js/Pages/Foo.vue', "\n"); + $project->git()->commit('add page'); + + tiaSeedWithInertia($project, ['Foo']); + + $project->write('resources/js/Pages/Foo.vue', "\n"); + $project->snapshot(); + + $result = $project->pest('--tia'); + + expect($result->affected())->toBe(2, $result->describe()) + ->and($result->replayed())->toBe(4, $result->describe()); +})->skipOnWindows(); + +test('a changed shared JS module selects the tests of the pages that import it', function (): void { + $project = Project::make('master'); + $project->write('resources/js/Pages/Foo.vue', "\n"); + $project->write('resources/js/Shared/Nav.vue', "\n"); + $project->git()->commit('add pages'); + + tiaSeedWithInertia($project, ['Foo'], ['resources/js/Shared/Nav.vue' => ['Foo']]); + + $project->write('resources/js/Shared/Nav.vue', "\n"); + $project->snapshot(); + + $result = $project->pest('--tia'); + + expect($result->affected())->toBe(2, $result->describe()) + ->and($result->replayed())->toBe(4, $result->describe()); +})->skipOnWindows(); + +test('a changed frontend runtime file selects every Inertia test', function (): void { + $project = Project::make('master'); + $project->write('resources/js/app.js', "console.log(1)\n"); + $project->git()->commit('add runtime'); + + tiaSeedWithInertia($project, ['Foo']); + + $project->write('resources/js/app.js', "console.log(2)\n"); + $project->snapshot(); + + $result = $project->pest('--tia'); + + expect($result->affected())->toBe(2, $result->describe()) + ->and($result->replayed())->toBe(4, $result->describe()); +})->skipOnWindows(); diff --git a/tests/Features/Tia/StateReclamation.php b/tests/Features/Tia/StateReclamation.php index 41114f92..54a53fa6 100644 --- a/tests/Features/Tia/StateReclamation.php +++ b/tests/Features/Tia/StateReclamation.php @@ -343,3 +343,35 @@ test('a second green run on a feature branch writes nothing at all', function (a expect($result->replayed())->toBe(Project::TOTAL_TESTS, $result->describe()) ->and($delta->isHardSuppressed())->toBeTrue($delta->summary()); })->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('a graph whose recorded commit is gone is re-anchored, not warned about forever', function (array $arguments): void { + $project = Project::make('master'); + $project->git()->commit('second'); + $project->seed('master'); + + $recordedSha = $project->graph()['baselines']['master']['sha']; + + // A rebase, a force-push, a reset: the commit the baseline was recorded at + // is no longer an ancestor of HEAD, so nothing can be diffed against it. + $project->git()->run(['reset', '--quiet', '--hard', 'HEAD~1']); + $project->snapshot(); + + $first = $project->pest('--tia', ...$arguments); + + // The whole suite runs, and its results are the truth at HEAD — so the + // recorded revision has to move, whether or not a coverage driver was + // around to refresh the edges. Without that, the run below repeats forever. + expect($first->exitCode)->toBe(0, $first->describe()) + ->and($first->output)->toContain('no longer reachable') + ->and($first->tally())->toContain(Project::TOTAL_TESTS.' passed') + ->and($project->graph()['baselines']['master']['sha'])->not->toBe($recordedSha) + ->and($project->graph()['baselines']['master']['sha'])->toBe($project->git()->sha()); + + $project->snapshot(); + $second = $project->pest('--tia', ...$arguments); + $delta = $project->delta(); + + expect($second->output)->not->toContain('no longer reachable') + ->and($second->replayed())->toBe(Project::TOTAL_TESTS, $second->describe()) + ->and($delta->writtenCount())->toBe(0, $delta->summary()); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); diff --git a/tests/Fixtures/Tia/GitRepo.php b/tests/Fixtures/Tia/GitRepo.php index 0969ac3a..e962acd5 100644 --- a/tests/Fixtures/Tia/GitRepo.php +++ b/tests/Fixtures/Tia/GitRepo.php @@ -130,7 +130,11 @@ final readonly class GitRepo private function process(array $arguments, bool $mustSucceed): Process { $process = new Process(['git', ...$arguments], $this->path, self::ENV); - $process->setTimeout(30.0); + // Generous on purpose: these rows each spawn a real pest subprocess, so + // a loaded machine — a shared CI runner, or two of these suites at once + // — can starve a git call for tens of seconds. A timeout here fails the + // row for reasons that have nothing to do with what it asserts. + $process->setTimeout(120.0); $process->run(); if ($mustSucceed && ! $process->isSuccessful()) { diff --git a/tests/Fixtures/Tia/Project.php b/tests/Fixtures/Tia/Project.php index d019ec71..73cbabc8 100644 --- a/tests/Fixtures/Tia/Project.php +++ b/tests/Fixtures/Tia/Project.php @@ -124,6 +124,26 @@ final class Project } } + /** + * A second copy of the fixture app in a subdirectory of this project, so a + * run can be started from a root that sits *below* the git repository root. + * + * @return string The nested project's absolute path. + */ + public function nested(string $directory = 'nested'): string + { + $path = $this->path($directory); + + if (! is_dir($path) && ! @mkdir($path, 0755, true) && ! is_dir($path)) { + throw new RuntimeException(sprintf('Unable to create [%s].', $path)); + } + + self::copy(__DIR__.'/app', $path); + $this->scaffoldVendor($path); + + return $path; + } + public function worktree(string $branch): string { $path = $this->path.'-worktree-'.preg_replace('/[^a-z0-9]+/i', '-', $branch); @@ -159,6 +179,12 @@ final class Project 'COLLISION_IGNORE_DURATION' => 'true', 'PARATEST' => '0', 'PAO_DISABLE' => '1', + // Recording is what needs a driver, and recording happens here, + // in the subprocess — never in the process running these rows. + // Asking for coverage mode only here lets a CI job leave xdebug + // off for the suite it is running (whose collection under xdebug + // costs more than every scenario put together) and still record. + 'XDEBUG_MODE' => 'coverage', 'HOME' => $this->home(), 'GITHUB_EVENT_PATH' => '', 'CI_DEFAULT_BRANCH' => '', @@ -239,6 +265,48 @@ final class Project $sentinel ? $this->sentinel() : $this->snapshot(); } + /** + * Take the graph out of the state dir and hand back its JSON, so it can be + * served as the artifact a remote baseline fetch downloads. + */ + public function detachGraph(): string + { + $json = $this->state()->read(Tia::KEY_GRAPH); + + if ($json === null) { + throw new RuntimeException('There is no graph to detach.'); + } + + if (! $this->state()->delete(Tia::KEY_GRAPH)) { + throw new RuntimeException('Unable to remove the detached graph.'); + } + + $this->snapshot(); + + return $json; + } + + /** + * Install a stand-in for the GitHub CLI and return the environment that + * points a run at it. `$mode` names the failure it should serve (see + * `stubs/gh`); `$payload` is the graph.json its artifact carries. + * + * @return array + */ + public function gh(string $mode = 'ok', string $payload = '{}'): array + { + self::mirror(__DIR__.'/stubs/gh', $this->path('stub/gh')); + chmod($this->path('stub/gh'), 0755); + + $this->write('payload/graph.json', $payload); + + return [ + 'PATH' => $this->path('stub').PATH_SEPARATOR.(string) getenv('PATH'), + 'GH_STUB_MODE' => $mode, + 'GH_STUB_PAYLOAD' => $this->path('payload/graph.json'), + ]; + } + public static function testId(string $testFile, string $description): string { $basename = basename($testFile, '.php'); diff --git a/tests/Fixtures/Tia/stubs/gh b/tests/Fixtures/Tia/stubs/gh new file mode 100755 index 00000000..7f4ae9eb --- /dev/null +++ b/tests/Fixtures/Tia/stubs/gh @@ -0,0 +1,50 @@ +#!/bin/sh +# A stand-in for the GitHub CLI, so the remote-baseline path can be exercised +# without a network. `GH_STUB_MODE` picks the failure to serve; `GH_STUB_PAYLOAD` +# names the graph.json the fake artifact carries. + +if [ "$1" = "auth" ]; then + [ "$GH_STUB_MODE" = "unauthenticated" ] && exit 1 + exit 0 +fi + +if [ "$1" = "run" ] && [ "$2" = "list" ]; then + case "$GH_STUB_MODE" in + no-runs) exit 0 ;; + list-404) echo "HTTP 404: Not Found" >&2; exit 1 ;; + list-network) echo "could not resolve host: api.github.com" >&2; exit 1 ;; + esac + echo 987654321 + exit 0 +fi + +if [ "$1" = "api" ]; then + echo 2048 + exit 0 +fi + +if [ "$1" = "run" ] && [ "$2" = "download" ]; then + case "$GH_STUB_MODE" in + download-403) echo "HTTP 403: Forbidden" >&2; exit 1 ;; + download-network) echo "connection refused" >&2; exit 1 ;; + esac + + dir="" + previous="" + for argument in "$@"; do + [ "$previous" = "-D" ] && dir="$argument" + previous="$argument" + done + + [ -z "$dir" ] && exit 1 + + case "$GH_STUB_MODE" in + missing-asset) echo "{}" > "$dir/other.json" ;; + corrupt) printf 'not json at all' > "$dir/graph.json" ;; + *) cp "$GH_STUB_PAYLOAD" "$dir/graph.json" ;; + esac + + exit 0 +fi + +exit 1 diff --git a/tests/Pest.php b/tests/Pest.php index 6247c703..41086a64 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -19,7 +19,10 @@ pest()->in('PHPUnit/GlobPatternTests/SubFolder2/*AsPattern.php')->use(CustomTest pest()->in('Visual')->group('integration'); -pest()->in('Features/Tia')->group('integration'); +// Also their own group, so a CI job with a coverage driver can run the TIA +// scenarios — the half of TIA that only exists when a driver is present — +// without dragging the visual snapshots along. +pest()->in('Features/Tia')->group('integration', 'tia'); // NOTE: global test value container to be mutated and checked across files, as needed $_SERVER['globalHook'] = (object) ['calls' => (object) ['beforeAll' => 0, 'afterAll' => 0]];