diff --git a/src/Bootstrappers/BootSubscribers.php b/src/Bootstrappers/BootSubscribers.php index 065a5e0f..c7d33053 100644 --- a/src/Bootstrappers/BootSubscribers.php +++ b/src/Bootstrappers/BootSubscribers.php @@ -35,6 +35,12 @@ final readonly class BootSubscribers implements Bootstrapper Subscribers\EnsureTiaResultIsRecordedOnSkipped::class, Subscribers\EnsureTiaResultIsRecordedOnIncomplete::class, Subscribers\EnsureTiaResultIsRecordedOnRisky::class, + Subscribers\EnsureTiaResultIsRecordedOnNoticeTriggered::class, + Subscribers\EnsureTiaResultIsRecordedOnPhpNoticeTriggered::class, + Subscribers\EnsureTiaResultIsRecordedOnDeprecationTriggered::class, + Subscribers\EnsureTiaResultIsRecordedOnPhpDeprecationTriggered::class, + Subscribers\EnsureTiaResultIsRecordedOnWarningTriggered::class, + Subscribers\EnsureTiaResultIsRecordedOnPhpWarningTriggered::class, Subscribers\EnsureTiaAssertionsAreRecordedOnFinished::class, ]; diff --git a/src/Plugins/Tia.php b/src/Plugins/Tia.php index e0f6d2df..95625cb2 100644 --- a/src/Plugins/Tia.php +++ b/src/Plugins/Tia.php @@ -159,6 +159,16 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument /** @var array */ private array $cachedAssertionsByTestId = []; + /** + * The status a replayed test was replayed *as*, so the write-back records + * what was cached rather than what the replay looked like from the + * outside. A cached deprecation replays as a pass — recording that pass + * would erase the deprecation from the baseline on the very next run. + * + * @var array + */ + private array $cachedStatusByTestId = []; + /** * @var array */ @@ -188,8 +198,6 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument private bool $baselineFetchAttemptedForDrift = false; - private bool $freshRebuild = false; - private bool $filteredMode = false; private bool $writesSuppressed = false; @@ -272,7 +280,9 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument return; } - $this->state->delete(self::KEY_GRAPH); + if (! $this->deleteState(self::KEY_GRAPH)) { + return; + } if ($this->unreadableGraphReported) { return; @@ -284,6 +294,25 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument $this->renderBadge('WARN', 'The dependency graph could not be read — it will be rebuilt.'); } + /** + * Delete a state file, unless this checkout may not write. + * + * A detached HEAD names no branch, so {@see self::saveGraph()} refuses to + * write — which means anything deleted here could never be rebuilt from + * this checkout. Read-only has to mean deletes too, or a drifted + * `composer.lock` on a detached CI checkout wipes the whole team's baseline. + * + * @return bool Whether the delete happened. + */ + private function deleteState(string $key): bool + { + if ($this->detachedHead) { + return false; + } + + return $this->state->delete($key); + } + private function saveGraph(Graph $graph): bool { // A detached HEAD names no branch of its own, so `$this->branch` is the @@ -412,6 +441,10 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument } $this->replayedCount++; + $this->cachedStatusByTestId[$testId] = [ + 'status' => $result->asInt(), + 'message' => $result->message(), + ]; $assertions = $this->replayGraph->getAssertions($this->branch, $testId); $this->cachedAssertionsByTestId[$testId] = $assertions ?? 0; @@ -502,7 +535,6 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument $this->forceRefetch = false; $this->filteredMode = false; - $this->freshRebuild = false; return $arguments; } @@ -515,7 +547,6 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument } $forceRebuild = $freshRequested && ($enabled || $recordingGlobal || $replayingGlobal); - $this->freshRebuild = $forceRebuild; if (! $enabled && ! $this->forceRefetch && ! $recordingGlobal && ! $replayingGlobal) { $this->requestWorkerResults(); @@ -546,7 +577,10 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument $this->flushWorkerReplay(); } - if ($this->writesSuppressed || $this->resultsOnlyWrites) { + // `terminate()` also runs from the shutdown handler, which is how a run + // that `exit()`s inside a test gets here — with a test prepared and + // never finished, and so with no right to a complete write. + if ($this->writesSuppressed || $this->resultsOnlyWrites || $this->hasUnfinishedTest()) { $this->recorder->reset(); $this->coverageCollector->reset(); @@ -619,10 +653,6 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument $graph->replaceTestInertiaComponents($perTestInertia); $graph->replaceJsFileToComponents(JsModuleGraph::build($projectRoot)); - if ($this->freshRebuild) { - $graph->pruneMissingTests(); - } - $this->seedResultsInto($graph); if (! $this->saveGraph($graph)) { @@ -642,7 +672,7 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument return $exitCode; } - if (Only::isEnabled() || $this->stoppedEarly()) { + if (Only::isEnabled() || $this->stoppedEarly() || $this->hasUnfinishedTest()) { $this->resultsOnlyWrites = true; } @@ -727,10 +757,6 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument $graph->replaceTestInertiaComponents($finalisedInertia); $graph->replaceJsFileToComponents(JsModuleGraph::build($projectRoot)); - if ($this->freshRebuild) { - $graph->pruneMissingTests(); - } - if (! $this->saveGraph($graph)) { $this->renderBadge('ERROR', 'Could not write the dependency graph.'); @@ -773,8 +799,8 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument return $this->reconcileFingerprint($rebuilt, $current); } - $this->state->delete(self::KEY_GRAPH); - $this->state->delete(self::KEY_COVERAGE_CACHE); + $this->deleteState(self::KEY_GRAPH); + $this->deleteState(self::KEY_COVERAGE_CACHE); return null; } @@ -790,7 +816,7 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument $graph->clearResults($this->branch); $graph->setFingerprint($current); $this->saveGraph($graph); - $this->state->delete(self::KEY_COVERAGE_CACHE); + $this->deleteState(self::KEY_COVERAGE_CACHE); } return $graph; @@ -821,7 +847,7 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument $fingerprint = Fingerprint::compute($projectRoot); $this->startFingerprint = $fingerprint; - if ($forceRebuild) { + if ($forceRebuild && ! $this->detachedHead) { Storage::purge($projectRoot); } @@ -1362,7 +1388,7 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument } foreach ($results as $testId => $result) { - $results[$testId]['time'] = $this->resultTime($testId, $result['time']); + $results[$testId] = $this->replayedAsRecorded($testId, $result); } $json = json_encode([ @@ -1370,7 +1396,7 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument 'replayed' => $this->replayedCount, 'affected' => $this->affectedCount, 'executed' => $this->executedCount, - 'truncated' => $this->stoppedEarly(), + 'truncated' => $this->stoppedEarly() || $collector->hasUnfinishedTest(), ], JSON_UNESCAPED_SLASHES); if ($json === false) { @@ -1639,6 +1665,24 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument return $this->cachedTimeByTestId[$testId] ?? $time; } + /** + * @param array{status: int, message: string, time: float, assertions: int, file?: string} $result + * @return array{status: int, message: string, time: float, assertions: int, file?: string} + */ + private function replayedAsRecorded(string $testId, array $result): array + { + $result['time'] = $this->resultTime($testId, $result['time']); + + $cached = $this->cachedStatusByTestId[$testId] ?? null; + + if ($cached !== null) { + $result['status'] = $cached['status']; + $result['message'] = $cached['message']; + } + + return $result; + } + private function seedResultsInto(Graph $graph): void { /** @var ResultCollector $collector */ @@ -1658,12 +1702,14 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument $touchedFiles[$file] = true; } + $result = $this->replayedAsRecorded($testId, $result); + $graph->setResult( $this->branch, $testId, $result['status'], $result['message'], - $this->resultTime($testId, $result['time']), + $result['time'], $result['assertions'], $file, ); @@ -1671,10 +1717,45 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument $graph->markKnownTestFiles(array_keys($touchedFiles)); $graph->pruneStaleResults($this->branch, array_keys($touchedFiles), array_keys($results)); + $this->reclaim($graph); $collector->reset(); } + /** + * Give back what the graph no longer needs. Only ever called from a + * complete write — the RESULTS-ONLY and HARD-SUPPRESSED tiers may not + * remove an entry, and a narrowed run has not seen enough to judge. + */ + private function reclaim(Graph $graph): void + { + // The fallback branch never layers under itself, so marking it would + // write the graph for no reader's benefit — and cost a clean green run + // its "wrote nothing at all". + if ($this->branch !== $this->fallbackBranch) { + $graph->markBaselineComplete($this->branch); + } + + $graph->pruneMissingTests(); + $graph->pruneResultsForMissingFiles($this->branch); + + $branches = new ChangedFiles(TestSuite::getInstance()->rootPath)->branchNames(); + + if ($branches === null) { + return; + } + + // A shallow, single-branch CI checkout can see almost no refs, and + // "git has never heard of it" would then mean "this clone is narrow", + // not "that branch is gone". Only reclaim from a checkout that can at + // least see the branch everything else falls back to. + if (! in_array($this->fallbackBranch, $branches, true)) { + return; + } + + $graph->pruneMissingBranches([...$branches, $this->branch, $this->fallbackBranch]); + } + private function snapshotTestResults(bool $markKnownTestFiles = false, bool $complete = true): void { /** @var ResultCollector $collector */ @@ -1720,12 +1801,14 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument continue; } + $result = $this->replayedAsRecorded($testId, $result); + $graph->setResult( $this->branch, $testId, $result['status'], $result['message'], - $this->resultTime($testId, $result['time']), + $result['time'], $result['assertions'], $file, ); @@ -1737,6 +1820,7 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument if ($complete) { $graph->pruneStaleResults($this->branch, array_keys($touchedFiles), array_keys($results)); + $this->reclaim($graph); } $this->saveGraph($graph); @@ -1832,6 +1916,18 @@ final class Tia implements AddsOutput, HandlesArguments, HandlesOriginalArgument return TestResultFacade::shouldStop(); } + /** + * A test that was prepared and never finished means this process is being + * torn down mid-file, so it has not seen enough of that file to prune it. + */ + private function hasUnfinishedTest(): bool + { + $collector = Container::getInstance()->get(ResultCollector::class); + assert($collector instanceof ResultCollector); + + return $collector->hasUnfinishedTest(); + } + private function resolveBranch(string $projectRoot): void { if ($this->branchResolved) { diff --git a/src/Plugins/Tia/ChangedFiles.php b/src/Plugins/Tia/ChangedFiles.php index 01fd90c1..ea442f89 100644 --- a/src/Plugins/Tia/ChangedFiles.php +++ b/src/Plugins/Tia/ChangedFiles.php @@ -243,6 +243,56 @@ final readonly class ChangedFiles return $exists ? $configured : null; } + /** + * Every branch name this checkout knows, local and remote alike. Remotes + * count: a branch that only lives on the origin is still a branch someone + * will check out, and its baseline must survive. + * + * @return list|null `null` when git cannot answer. + */ + public function branchNames(): ?array + { + $process = new Process( + ['git', 'for-each-ref', '--format=%(refname)', 'refs/heads', 'refs/remotes'], + $this->projectRoot, + ); + $process->setTimeout(5.0); + $process->run(); + + if (! $process->isSuccessful()) { + return null; + } + + $names = []; + + foreach ($this->splitLines($process->getOutput()) as $ref) { + if (str_starts_with($ref, 'refs/heads/')) { + $names[substr($ref, strlen('refs/heads/'))] = true; + + continue; + } + + if (! str_starts_with($ref, 'refs/remotes/')) { + continue; + } + + $tail = substr($ref, strlen('refs/remotes/')); + $slash = strpos($tail, '/'); + + if ($slash === false) { + continue; + } + + $branch = substr($tail, $slash + 1); + + if ($branch !== '' && $branch !== 'HEAD') { + $names[$branch] = true; + } + } + + return array_keys($names); + } + public function hasRemote(): bool { return $this->gitOutput(['git', 'remote']) !== null; diff --git a/src/Plugins/Tia/Enums/ReplayType.php b/src/Plugins/Tia/Enums/ReplayType.php index 6b669cc6..bd93a299 100644 --- a/src/Plugins/Tia/Enums/ReplayType.php +++ b/src/Plugins/Tia/Enums/ReplayType.php @@ -29,7 +29,13 @@ enum ReplayType $status->isRisky() => self::Risky, $status->isSkipped() => self::Skipped, $status->isIncomplete() => self::Incomplete, - default => self::Failure, + // A recorded notice, deprecation or warning only reaches replay when + // the configured failOn* / displayDetailsOn* policies say it is not + // worth re-running — which means the test passed. Folding it into + // Failure below would turn a green run red on cache alone. + $status->isNotice(), $status->isDeprecation(), $status->isWarning() => self::Pass, + $status->isFailure(), $status->isError() => self::Failure, + default => self::None, }; } } diff --git a/src/Plugins/Tia/Graph.php b/src/Plugins/Tia/Graph.php index 0930dedc..ad25d6ab 100644 --- a/src/Plugins/Tia/Graph.php +++ b/src/Plugins/Tia/Graph.php @@ -43,6 +43,7 @@ final class Graph * @var array, + * complete?: bool, * results: array * }> */ @@ -650,6 +651,9 @@ final class Graph $r = $baseline['results'][$testId]; + // A status this build does not know — a graph written by a newer Pest, + // or a corrupt one — is not a result. Returning null re-executes the + // test rather than replaying an outcome nobody can interpret. return match ($r['status']) { 0 => TestStatus::success(), 1 => TestStatus::skipped($r['message']), @@ -660,7 +664,7 @@ final class Graph 6 => TestStatus::warning($r['message']), 7 => TestStatus::failure($r['message']), 8 => TestStatus::error($r['message']), - default => TestStatus::unknown(), + default => null, }; } @@ -687,7 +691,9 @@ final class Graph $rel = $this->relative($file); - if ($rel !== null) { + // A test file that is no longer on disk cannot be re-run by anyone, + // so selecting it would only widen the run for nothing. + if ($rel !== null && is_file($this->projectRoot.'/'.$rel)) { $files[$rel] = true; } } @@ -695,6 +701,15 @@ final class Graph return array_keys($files); } + /** + * Whether a cached result due a re-run names a test file this project + * cannot address — an empty path, or one that resolves outside the project + * root. Those are genuinely lost, so the caller widens to the full suite. + * + * A path that resolves fine but is simply absent is *deleted*, not lost: + * widening would not run it either, and treating it as unlocated used to + * strand `--filtered` on a full replay for good. + */ public function hasUnlocatedTestsToRerun(string $branch, ?string $fallbackBranch = null): bool { $baseline = $this->baselineFor($branch, $fallbackBranch); @@ -710,9 +725,7 @@ final class Graph return true; } - $rel = $this->relative($file); - - if ($rel === null || ! is_file($this->projectRoot.'/'.$rel)) { + if ($this->relative($file) === null) { return true; } } @@ -736,6 +749,10 @@ final class Graph return true; } + if ($testStatus->isUnknown()) { + return true; + } + $configuration = Registry::get(); if ($testStatus->isRisky()) { @@ -813,10 +830,21 @@ final class Graph * default branch's, so a key minted by a narrowed run — which only holds the * handful of tests that ran — does not shadow the fallback for everything else. * + * Once this branch has had a complete run, the layering becomes per *file* + * rather than per test id: the branch's entries for a file it executed are + * the whole truth, so the fallback's entries for that same file are dropped + * rather than merged. Without that, a test the branch renamed or removed — + * and {@see self::pruneStaleResults()} therefore unset — is resurrected by + * the default branch on the very next read, and never stops coming back. + * + * A branch whose key was minted by a *narrowed* run holds only the handful + * of tests that ran, and has no business speaking for the rest of their + * file, so it keeps the per-test-id merge. + * * Read-only: the layering never reaches `$this->baselines`, so writes stay on * the branch that ran. * - * @return array{sha: ?string, tree: array, results: array} + * @return array{sha: ?string, tree: array, complete?: bool, results: array} */ private function baselineFor(string $branch, ?string $fallbackBranch): array { @@ -833,13 +861,49 @@ final class Graph return $own; } + $under = ($own['complete'] ?? false) === true + ? $this->withoutFilesCoveredBy($fallback['results'], $own['results']) + : $fallback['results']; + return [ 'sha' => $own['sha'] ?? $fallback['sha'], 'tree' => $own['tree'] !== [] ? $own['tree'] : $fallback['tree'], - 'results' => array_replace($fallback['results'], $own['results']), + 'results' => array_replace($under, $own['results']), ]; } + /** + * @param array $results + * @param array $authoritative + * @return array + */ + private function withoutFilesCoveredBy(array $results, array $authoritative): array + { + $covered = []; + + foreach ($authoritative as $entry) { + $file = $entry['file'] ?? null; + + if (is_string($file) && $file !== '') { + $covered[$file] = true; + } + } + + if ($covered === []) { + return $results; + } + + foreach ($results as $testId => $entry) { + $file = $entry['file'] ?? null; + + if (is_string($file) && isset($covered[$file])) { + unset($results[$testId]); + } + } + + return $results; + } + private function ensureBaseline(string $branch): void { if (! isset($this->baselines[$branch])) { @@ -1422,6 +1486,67 @@ final class Graph } } + /** + * Record that this branch has run the whole suite at least once, which is + * what lets {@see self::baselineFor()} treat its entries as authoritative + * for the files they cover. Never mints a key: a run that recorded nothing + * has nothing to be authoritative about. + */ + public function markBaselineComplete(string $branch): void + { + if (isset($this->baselines[$branch])) { + $this->baselines[$branch]['complete'] = true; + } + } + + /** + * Drop this branch's result entries whose test file is no longer on disk. + * + * Without this nothing but `--fresh` ever reclaims them, and a *failing* + * one keeps `--filtered` widened to a full replay on every later run. + */ + public function pruneResultsForMissingFiles(string $branch): void + { + if (! isset($this->baselines[$branch]['results'])) { + return; + } + + $root = rtrim($this->projectRoot, DIRECTORY_SEPARATOR).DIRECTORY_SEPARATOR; + + foreach ($this->baselines[$branch]['results'] as $testId => $result) { + $file = $result['file'] ?? null; + + if (! is_string($file) || $file === '') { + continue; + } + + $rel = $this->relative($file); + + if ($rel === null || is_file($root.$rel)) { + continue; + } + + unset($this->baselines[$branch]['results'][$testId]); + } + } + + /** + * Drop baselines for branches git no longer knows, so the graph does not + * carry one full copy of the suite per branch ever created. + * + * @param array $keep Branch names that must survive. + */ + public function pruneMissingBranches(array $keep): void + { + $survivors = array_fill_keys($keep, true); + + foreach (array_keys($this->baselines) as $branch) { + if (! isset($survivors[$branch])) { + unset($this->baselines[$branch]); + } + } + } + /** * Prune baseline result entries whose test files were just executed but whose * test IDs are no longer present (e.g. the test method was removed or renamed). @@ -1499,10 +1624,10 @@ final class Graph $graph = new self($projectRoot); $graph->fingerprint = is_array($data['fingerprint'] ?? null) ? $data['fingerprint'] : []; - $graph->files = is_array($data['files'] ?? null) ? array_values($data['files']) : []; + $graph->files = self::decodeFiles($data['files'] ?? null); $graph->fileIds = array_flip($graph->files); - $graph->edges = is_array($data['edges'] ?? null) ? $data['edges'] : []; - $graph->baselines = is_array($data['baselines'] ?? null) ? $data['baselines'] : []; + $graph->edges = self::decodeEdges($data['edges'] ?? null); + $graph->baselines = self::decodeBaselines($data['baselines'] ?? null); $graph->testTables = self::decodeStringMap($data['test_tables'] ?? null); $graph->testInertiaComponents = self::decodeStringMap($data['test_inertia_components'] ?? null); @@ -1511,6 +1636,160 @@ final class Graph return $graph; } + /** + * @return array + */ + private static function decodeFiles(mixed $section): array + { + if (! is_array($section)) { + return []; + } + + $files = []; + + foreach ($section as $path) { + if (is_string($path) && $path !== '') { + $files[] = $path; + } + } + + return $files; + } + + /** + * @return array> + */ + private static function decodeEdges(mixed $section): array + { + if (! is_array($section)) { + return []; + } + + $edges = []; + + foreach ($section as $key => $ids) { + $testFile = (string) $key; + + if ($testFile === '') { + continue; + } + if (! is_array($ids)) { + continue; + } + + $clean = []; + + foreach ($ids as $id) { + if (is_int($id)) { + $clean[] = $id; + } + } + + $edges[$testFile] = $clean; + } + + return $edges; + } + + /** + * A graph is state on disk that any process may have written: a newer Pest, + * a half-finished write, a hand edit. Every branch, every entry and every + * field is checked here so that a malformed one is dropped rather than + * reaching a read path and taking the run down with it. + * + * @return array, complete?: bool, results: array}> + */ + private static function decodeBaselines(mixed $section): array + { + if (! is_array($section)) { + return []; + } + + $baselines = []; + + foreach ($section as $key => $baseline) { + // A branch named `12345` decodes as an integer key, and must not be + // mistaken for a malformed one. + $branch = (string) $key; + + if ($branch === '') { + continue; + } + if (! is_array($baseline)) { + continue; + } + + $sha = $baseline['sha'] ?? null; + $tree = []; + + if (is_array($baseline['tree'] ?? null)) { + foreach ($baseline['tree'] as $path => $hash) { + if (is_string($path) && is_string($hash)) { + $tree[$path] = $hash; + } + } + } + + $baselines[$branch] = [ + 'sha' => is_string($sha) ? $sha : null, + 'tree' => $tree, + 'results' => self::decodeResults($baseline['results'] ?? null), + ]; + + if (($baseline['complete'] ?? null) === true) { + $baselines[$branch]['complete'] = true; + } + } + + return $baselines; + } + + /** + * @return array + */ + private static function decodeResults(mixed $section): array + { + if (! is_array($section)) { + return []; + } + + $results = []; + + foreach ($section as $key => $entry) { + $testId = (string) $key; + + if ($testId === '') { + continue; + } + if (! is_array($entry)) { + continue; + } + if (! is_int($entry['status'] ?? null)) { + continue; + } + + $time = $entry['time'] ?? null; + + $result = [ + 'status' => $entry['status'], + 'message' => is_string($entry['message'] ?? null) ? $entry['message'] : '', + 'time' => is_int($time) || is_float($time) ? (float) $time : 0.0, + ]; + + if (is_int($entry['assertions'] ?? null)) { + $result['assertions'] = $entry['assertions']; + } + + if (is_string($entry['file'] ?? null) && $entry['file'] !== '') { + $result['file'] = $entry['file']; + } + + $results[$testId] = $result; + } + + return $results; + } + /** * @return array> */ diff --git a/src/Plugins/Tia/ResultCollector.php b/src/Plugins/Tia/ResultCollector.php index 1b84c8c8..22ccb916 100644 --- a/src/Plugins/Tia/ResultCollector.php +++ b/src/Plugins/Tia/ResultCollector.php @@ -16,6 +16,9 @@ final class ResultCollector */ private array $results = []; + /** @var array */ + private array $triggered = []; + private ?string $currentTestId = null; private ?string $currentTestFile = null; @@ -35,9 +38,35 @@ final class ResultCollector return; } + // PHPUnit reports a test that triggered a notice, deprecation or + // warning as passed, and emits Passed for it. Recording success here + // would erase the issue from the baseline, and a later replay under + // --fail-on-deprecation (and friends) would come back green where a + // fresh run fails. Keep the issue; only refresh what it cannot know. + if (isset($this->triggered[$this->currentTestId])) { + $this->refreshTime(); + + return; + } + $this->record(TestStatus::success()); } + public function testTriggeredNotice(string $message): void + { + $this->recordIssue(TestStatus::notice($message)); + } + + public function testTriggeredDeprecation(string $message): void + { + $this->recordIssue(TestStatus::deprecation($message)); + } + + public function testTriggeredWarning(string $message): void + { + $this->recordIssue(TestStatus::warning($message)); + } + public function testFailed(string $message): void { if ($this->currentTestId === null) { @@ -91,6 +120,17 @@ final class ResultCollector return $this->results; } + /** + * Whether a test was prepared but never finished — the process is being + * torn down in the middle of it (an `exit()` inside a test, a killed + * worker). What it collected is therefore a partial view of that test + * file, and must not license pruning the siblings it never reached. + */ + public function hasUnfinishedTest(): bool + { + return $this->currentTestId !== null; + } + public function recordAssertions(string $testId, int $assertions): void { if (isset($this->results[$testId])) { @@ -111,6 +151,7 @@ final class ResultCollector public function reset(): void { $this->results = []; + $this->triggered = []; $this->currentTestId = null; $this->currentTestFile = null; $this->startTime = null; @@ -123,6 +164,43 @@ final class ResultCollector $this->startTime = null; } + /** + * Record an issue raised while the test was running. The most important + * one wins, exactly as PHPUnit ranks them, so a deprecation does not + * shadow the warning that followed it — or the failure. + */ + private function recordIssue(TestStatus $status): void + { + if ($this->currentTestId === null) { + return; + } + + $existing = $this->results[$this->currentTestId]['status'] ?? null; + + if (is_int($existing) && $existing >= $status->asInt()) { + return; + } + + $this->triggered[$this->currentTestId] = true; + + $this->record($status); + } + + private function refreshTime(): void + { + if ($this->currentTestId === null) { + return; + } + if (! isset($this->results[$this->currentTestId])) { + return; + } + if ($this->startTime === null) { + return; + } + + $this->results[$this->currentTestId]['time'] = round(microtime(true) - $this->startTime, 3); + } + private function record(TestStatus $status): void { if ($this->currentTestId === null) { diff --git a/src/Subscribers/EnsureTiaResultIsRecordedOnDeprecationTriggered.php b/src/Subscribers/EnsureTiaResultIsRecordedOnDeprecationTriggered.php new file mode 100644 index 00000000..2ad83cdc --- /dev/null +++ b/src/Subscribers/EnsureTiaResultIsRecordedOnDeprecationTriggered.php @@ -0,0 +1,26 @@ +wasSuppressed()) { + return; + } + + $this->collector->testTriggeredDeprecation($event->message()); + } +} diff --git a/src/Subscribers/EnsureTiaResultIsRecordedOnNoticeTriggered.php b/src/Subscribers/EnsureTiaResultIsRecordedOnNoticeTriggered.php new file mode 100644 index 00000000..c14d9e4a --- /dev/null +++ b/src/Subscribers/EnsureTiaResultIsRecordedOnNoticeTriggered.php @@ -0,0 +1,26 @@ +wasSuppressed()) { + return; + } + + $this->collector->testTriggeredNotice($event->message()); + } +} diff --git a/src/Subscribers/EnsureTiaResultIsRecordedOnPhpDeprecationTriggered.php b/src/Subscribers/EnsureTiaResultIsRecordedOnPhpDeprecationTriggered.php new file mode 100644 index 00000000..a98ff355 --- /dev/null +++ b/src/Subscribers/EnsureTiaResultIsRecordedOnPhpDeprecationTriggered.php @@ -0,0 +1,26 @@ +wasSuppressed()) { + return; + } + + $this->collector->testTriggeredDeprecation($event->message()); + } +} diff --git a/src/Subscribers/EnsureTiaResultIsRecordedOnPhpNoticeTriggered.php b/src/Subscribers/EnsureTiaResultIsRecordedOnPhpNoticeTriggered.php new file mode 100644 index 00000000..869dd8b7 --- /dev/null +++ b/src/Subscribers/EnsureTiaResultIsRecordedOnPhpNoticeTriggered.php @@ -0,0 +1,26 @@ +wasSuppressed()) { + return; + } + + $this->collector->testTriggeredNotice($event->message()); + } +} diff --git a/src/Subscribers/EnsureTiaResultIsRecordedOnPhpWarningTriggered.php b/src/Subscribers/EnsureTiaResultIsRecordedOnPhpWarningTriggered.php new file mode 100644 index 00000000..bc9f9ed7 --- /dev/null +++ b/src/Subscribers/EnsureTiaResultIsRecordedOnPhpWarningTriggered.php @@ -0,0 +1,26 @@ +wasSuppressed()) { + return; + } + + $this->collector->testTriggeredWarning($event->message()); + } +} diff --git a/src/Subscribers/EnsureTiaResultIsRecordedOnWarningTriggered.php b/src/Subscribers/EnsureTiaResultIsRecordedOnWarningTriggered.php new file mode 100644 index 00000000..15c0f472 --- /dev/null +++ b/src/Subscribers/EnsureTiaResultIsRecordedOnWarningTriggered.php @@ -0,0 +1,26 @@ +wasSuppressed()) { + return; + } + + $this->collector->testTriggeredWarning($event->message()); + } +} diff --git a/tests/Features/Tia/BranchShapes.php b/tests/Features/Tia/BranchShapes.php new file mode 100644 index 00000000..86f1b1b8 --- /dev/null +++ b/tests/Features/Tia/BranchShapes.php @@ -0,0 +1,161 @@ +seed('master'); + + $project->git()->switchTo($branch, new: true); + + $result = $project->pest('--tia'); + $delta = $project->delta(); + + expect($result->replayed())->toBe(Project::TOTAL_TESTS, $result->describe()) + ->and($result->uncached())->toBe(0, $result->describe()) + ->and($project->branchKeys())->toBe(['master', $branch]) + ->and($delta->baselineUntouched('master'))->toBeTrue($delta->summary()); +})->with([ + 'slashes' => 'feature/deep/nesting', + 'dots' => 'release.1.2.x', + 'unicode' => 'feature-café-日本', + 'digits' => '12345', + 'underscores' => 'feature_x_y', + 'very long' => 'feature-'.str_repeat('x', 180), +])->skipOnWindows(); + +test('a branch differing from the default only in case gets its own key', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->git()->switchTo('MASTER-2', new: true); + + $result = $project->pest('--tia'); + + expect($result->replayed())->toBe(Project::TOTAL_TESTS, $result->describe()) + ->and($project->branchKeys())->toBe(['master', 'MASTER-2']); +})->skipOnWindows(); + +test('a branch that only lives on the remote keeps its baseline', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->git()->switchTo('remote-only', new: true); + $project->pest('--tia'); + + $project->git()->switchTo('master'); + $project->git()->run(['update-ref', 'refs/remotes/origin/remote-only', 'HEAD']); + $project->git()->run(['branch', '-D', 'remote-only']); + + $project->pest('--tia'); + + expect($project->branchKeys())->toBe(['master', 'remote-only']); +})->skipOnWindows(); + +test('a branch checked out in a worktree keeps its baseline', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->git()->switchTo('wt-branch', new: true); + $project->pest('--tia'); + + $project->git()->switchTo('master'); + $project->git()->run(['worktree', 'add', '--quiet', $project->path().'-wt', 'wt-branch']); + + $project->pest('--tia'); + + $project->git()->run(['worktree', 'remove', '--force', $project->path().'-wt']); + + expect($project->branchKeys())->toBe(['master', 'wt-branch']); +})->skipOnWindows(); + +test('deleting many branches reclaims every one of their baselines', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master'); + + foreach (range(1, 4) as $index) { + $project->git()->switchTo('feature-'.$index, new: true); + $project->pest('--tia', ...$arguments); + } + + expect($project->branchKeys())->toHaveCount(5); + + $project->git()->switchTo('master'); + + foreach (range(1, 4) as $index) { + $project->git()->run(['branch', '-D', 'feature-'.$index]); + } + + $project->pest('--tia', ...$arguments); + + expect($project->branchKeys())->toBe(['master']); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('a narrowed run does not reclaim anything', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->git()->switchTo('feature-x', new: true); + $project->pest('--tia'); + + $project->git()->switchTo('master'); + $project->git()->run(['branch', '-D', 'feature-x']); + + $project->snapshot(); + $project->pest('--tia', '--filter=adds two numbers'); + $delta = $project->delta(); + + expect($project->branchKeys())->toBe(['master', 'feature-x']) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('a detached HEAD does not reclaim anything either', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->git()->switchTo('feature-x', new: true); + $project->pest('--tia'); + + $project->git()->switchTo('master'); + $project->git()->run(['branch', '-D', 'feature-x']); + $project->git()->detach(); + + $project->snapshot(); + $project->pest('--tia'); + $delta = $project->delta(); + + expect($project->branchKeys())->toBe(['master', 'feature-x']) + ->and($delta->isHardSuppressed())->toBeTrue($delta->summary()); +})->skipOnWindows(); + +test('the default branch baseline survives every branch that comes and goes', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->git()->switchTo('feature-x', new: true); + $project->pest('--tia'); + + $project->git()->switchTo('master'); + $project->git()->run(['branch', '-D', 'feature-x']); + + $project->snapshot(); + $result = $project->pest('--tia'); + $delta = $project->delta(); + + expect($result->replayed())->toBe(Project::TOTAL_TESTS, $result->describe()) + ->and($delta->writtenCount())->toBe(0, $delta->summary()) + ->and($delta->added())->toBe(0, $delta->summary()) + ->and($delta->removed())->toBe(0, $delta->summary()) + ->and($project->graph()['baselines']['master']['results'])->toHaveCount(Project::TOTAL_TESTS); +})->skipOnWindows(); diff --git a/tests/Features/Tia/DefaultBranchWriteTier.php b/tests/Features/Tia/DefaultBranchWriteTier.php index bf819097..9a239632 100644 --- a/tests/Features/Tia/DefaultBranchWriteTier.php +++ b/tests/Features/Tia/DefaultBranchWriteTier.php @@ -75,10 +75,13 @@ test('filtered mode falls back to a full replay when a cached failure cannot be $project->seed('master', failing: ['adds two numbers']); + // A path this project cannot address at all — recorded on another machine. + // A path that merely no longer exists is a *deleted* test, not a lost one, + // and widening the run would not find it either; see StateReclamation. $project->mutateGraph(function (array $graph): array { $testId = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); - $graph['baselines']['master']['results'][$testId]['file'] = 'tests/Unit/DeletedTest.php'; + $graph['baselines']['master']['results'][$testId]['file'] = '/build/agent/tests/Unit/DeletedTest.php'; return $graph; }); diff --git a/tests/Features/Tia/HostileState.php b/tests/Features/Tia/HostileState.php new file mode 100644 index 00000000..6e664d3e --- /dev/null +++ b/tests/Features/Tia/HostileState.php @@ -0,0 +1,226 @@ +seed('master'); + + file_put_contents($project->graphDir().'/graph.json', $contents); + + $result = $project->pest('--tia'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed'); +})->with([ + 'empty' => '', + 'truncated' => '{"schema":1,"files":["app/Calculator.php"],"edg', + 'not json' => '{not json', + 'json scalar' => '"just a string"', + 'json list' => '[1,2,3]', + 'json null' => 'null', + 'empty object' => '{}', + 'nul bytes' => "\0\0\0\0", +])->skipOnWindows(); + +test('a graph whose shape is wrong everywhere is repaired rather than trusted', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->mutateGraph(function (array $graph): array { + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + + $graph['baselines']['master']['results'][$id] = 'nope'; + $graph['baselines']['master']['results'][7] = ['status' => 0, 'message' => '', 'time' => 0.1]; + $graph['baselines']['master']['tree'] = 'nope'; + $graph['baselines']['master']['sha'] = 42; + $graph['baselines'][''] = ['sha' => null, 'tree' => [], 'results' => []]; + $graph['baselines']['broken'] = 'nope'; + $graph['edges']['tests/Unit/GreeterTest.php'] = 'nope'; + $graph['edges'][''] = [0]; + $graph['files'][] = ['nested']; + + return $graph; + }); + + $result = $project->pest('--tia', ...$arguments); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->output)->not->toContain('TypeError') + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed') + ->and($project->branchKeys())->toBe(['master']); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('a cached status this build cannot interpret is re-run, not replayed', function (int $status): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->mutateGraph(function (array $graph) use ($status): array { + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + $graph['baselines']['master']['results'][$id]['status'] = $status; + + return $graph; + }); + + $result = $project->pest('--tia'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed') + ->and($result->uncached())->toBe(1, $result->describe()); +})->with([ + 'below the range' => -1, + 'one past the range' => 9, + 'far past the range' => 99, + 'huge' => PHP_INT_MAX, +])->skipOnWindows(); + +test('a cached status with no replay of its own does not fail the run', function (int $status): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->mutateGraph(function (array $graph) use ($status): array { + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + $graph['baselines']['master']['results'][$id]['status'] = $status; + $graph['baselines']['master']['results'][$id]['message'] = 'cached detail'; + + return $graph; + }); + + $result = $project->pest('--tia'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed'); +})->with(['notice' => 3, 'deprecation' => 4, 'warning' => 6])->skipOnWindows(); + +test('a cached skip or todo replays with its message intact', function (int $status, string $tally): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->mutateGraph(function (array $graph) use ($status): array { + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + $graph['baselines']['master']['results'][$id]['status'] = $status; + $graph['baselines']['master']['results'][$id]['message'] = 'a recorded reason'; + + return $graph; + }); + + $result = $project->pest('--tia'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->tally())->toContain($tally) + ->and($result->output)->toContain('a recorded reason') + ->and($result->replayed())->toBe(Project::TOTAL_TESTS, $result->describe()); +})->with([ + 'skipped' => [1, '1 skipped'], + 'incomplete' => [2, '1 incomplete'], +])->skipOnWindows(); + +test('a cached failure with a multi-line message re-runs rather than replaying the text', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->mutateGraph(function (array $graph): array { + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + $graph['baselines']['master']['results'][$id]['status'] = 7; + $graph['baselines']['master']['results'][$id]['message'] = "line one\nline two\nline three"; + + return $graph; + }); + + $result = $project->pest('--tia'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->uncached())->toBe(1, $result->describe()) + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed'); +})->skipOnWindows(); + +test('a result pointing outside the project is not addressable and widens the run', function (): void { + $project = Project::make('master'); + $project->seed('master', failing: ['adds two numbers']); + + $project->mutateGraph(function (array $graph): array { + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + $graph['baselines']['master']['results'][$id]['file'] = '/build/agent/tests/Unit/CalculatorTest.php'; + + return $graph; + }); + + $result = $project->pest('--tia', '--filtered'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->output)->toContain('could not be located on disk') + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed'); +})->skipOnWindows(); + +test('an edge pointing at a file id that does not exist is ignored', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->mutateGraph(function (array $graph): array { + $graph['edges']['tests/Unit/CalculatorTest.php'] = [0, 999, -5]; + + return $graph; + }); + + $result = $project->pest('--tia'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->replayed())->toBe(Project::TOTAL_TESTS, $result->describe()); +})->skipOnWindows(); + +test('a graph from a schema this build does not know is rebuilt, not read', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->mutateGraph(function (array $graph): array { + $graph['schema'] = 2; + + return $graph; + }); + + $result = $project->pest('--tia'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed') + ->and($result->replayed())->toBe(0, $result->describe()); +})->skipOnWindows(); + +test('graph.json being a directory does not stop the run', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + unlink($project->graphDir().'/graph.json'); + mkdir($project->graphDir().'/graph.json'); + + $result = $project->pest('--tia'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed'); +})->skipOnWindows(); + +test('a state dir it cannot write to still replays', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + chmod($project->graphDir(), 0500); + + try { + $result = $project->pest('--tia'); + } finally { + chmod($project->graphDir(), 0700); + } + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->replayed())->toBe(Project::TOTAL_TESTS, $result->describe()); +})->skipOnWindows(); diff --git a/tests/Features/Tia/IssueStatuses.php b/tests/Features/Tia/IssueStatuses.php new file mode 100644 index 00000000..343395f5 --- /dev/null +++ b/tests/Features/Tia/IssueStatuses.php @@ -0,0 +1,143 @@ +add(1, 2))->toBe(3); + }); + + test('subtracts two numbers', function (): void { + expect((new Calculator)->subtract(3, 1))->toBe(2); + }); + PHP; +} + +test('a triggered issue is recorded as itself, not as a pass', function (array $arguments, string $call, int $status): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/CalculatorTest.php', tiaTriggering($call)); + $project->git()->commit('trigger an issue'); + + $project->pest('--tia', ...$arguments); + + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + + expect($project->graph()['baselines']['master']['results'][$id]['status'])->toBe($status); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->with([ + 'deprecation' => ["trigger_error('legacy adder', E_USER_DEPRECATED);", 4], + 'notice' => ["trigger_error('a notice', E_USER_NOTICE);", 3], + 'warning' => ["trigger_error('a warning', E_USER_WARNING);", 6], +])->skipOnWindows(); + +test('a cached deprecation still fails the run that asked to fail on one', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/CalculatorTest.php', tiaTriggering("trigger_error('legacy adder', E_USER_DEPRECATED);")); + $project->git()->commit('trigger a deprecation'); + + $project->pest('--tia', ...$arguments); + + $result = $project->pest('--tia', '--fail-on-deprecation', ...$arguments); + + expect($result->exitCode)->toBe(1, $result->describe()) + ->and($result->tally())->toContain('1 deprecated') + ->and($result->uncached())->toBe(1, $result->describe()); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('replaying a cached issue does not downgrade it to a pass', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/CalculatorTest.php', tiaTriggering("trigger_error('legacy adder', E_USER_DEPRECATED);")); + $project->git()->commit('trigger a deprecation'); + + $project->pest('--tia', ...$arguments); + + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + + foreach (range(1, 3) as $ignored) { + $project->pest('--tia', ...$arguments); + + expect($project->graph()['baselines']['master']['results'][$id]['status'])->toBe(4); + } + + $result = $project->pest('--tia', '--fail-on-deprecation', ...$arguments); + + expect($result->exitCode)->toBe(1, $result->describe()); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('a failure outranks an issue triggered on the way to it', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/CalculatorTest.php', str_replace( + 'toBe(3)', + 'toBe(999)', + tiaTriggering("trigger_error('legacy adder', E_USER_DEPRECATED);"), + )); + $project->git()->commit('an issue then a failure'); + + $result = $project->pest('--tia'); + + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + + expect($result->exitCode)->not->toBe(0) + ->and($project->graph()['baselines']['master']['results'][$id]['status'])->toBe(7); +})->skipOnWindows(); + +test('a skip outranks an issue triggered on the way to it', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/CalculatorTest.php', str_replace( + 'expect((new Calculator)->add(1, 2))->toBe(3);', + "\$this->markTestSkipped('not today');", + tiaTriggering("trigger_error('legacy adder', E_USER_DEPRECATED);"), + )); + $project->git()->commit('an issue then a skip'); + + $project->pest('--tia'); + + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + + expect($project->graph()['baselines']['master']['results'][$id]['status'])->toBe(1); +})->skipOnWindows(); + +test('a suppressed issue is not recorded', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/CalculatorTest.php', tiaTriggering("@trigger_error('quiet', E_USER_DEPRECATED);")); + $project->git()->commit('a suppressed deprecation'); + + $project->pest('--tia'); + + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + + expect($project->graph()['baselines']['master']['results'][$id]['status'])->toBe(0); +})->skipOnWindows(); diff --git a/tests/Features/Tia/StateReclamation.php b/tests/Features/Tia/StateReclamation.php new file mode 100644 index 00000000..41114f92 --- /dev/null +++ b/tests/Features/Tia/StateReclamation.php @@ -0,0 +1,345 @@ +seed('master'); + + $project->git()->detach(); + + $project->write('composer.lock', (string) json_encode([ + 'content-hash' => 'drifted', + 'packages' => [], + 'packages-dev' => [], + ])); + + $result = $project->pest('--tia', ...$arguments); + $delta = $project->delta(); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($project->graphExists())->toBeTrue('the detached run deleted graph.json') + ->and($delta->isHardSuppressed())->toBeTrue($delta->summary()); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('a detached HEAD does not purge the graph with --fresh either', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->git()->detach(); + + $result = $project->pest('--tia', '--fresh', ...$arguments); + $delta = $project->delta(); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($project->graphExists())->toBeTrue('the detached --fresh run deleted graph.json') + ->and($delta->isHardSuppressed())->toBeTrue($delta->summary()); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('a detached HEAD leaves an unreadable graph for a checkout that can rebuild it', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->git()->detach(); + + file_put_contents($project->graphDir().'/graph.json', '{not json'); + + $result = $project->pest('--tia'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed') + ->and(file_get_contents($project->graphDir().'/graph.json'))->toBe('{not json'); +})->skipOnWindows(); + +test('a cached failure whose test file was deleted stops widening later runs', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master', failing: ['adds two numbers']); + + unlink($project->path('tests/Unit/CalculatorTest.php')); + $project->git()->commit('drop CalculatorTest'); + + $project->pest('--tia', '--filtered', ...$arguments); + + $project->snapshot(); + $second = $project->pest('--tia', '--filtered', ...$arguments); + $delta = $project->delta(); + + expect($second->output)->not->toContain('could not be located on disk') + ->and($second->output)->toContain('No affected tests found') + ->and($delta->isHardSuppressed())->toBeTrue($delta->summary()); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('a complete run reclaims the entry and the edge of a deleted test file', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master', failing: ['adds two numbers']); + + unlink($project->path('tests/Unit/CalculatorTest.php')); + $project->git()->commit('drop CalculatorTest'); + + $project->pest('--tia', ...$arguments); + + $graph = $project->graph(); + + expect($graph['edges'] ?? [])->not->toHaveKey('tests/Unit/CalculatorTest.php') + ->and(array_column($graph['baselines']['master']['results'] ?? [], 'file')) + ->not->toContain('tests/Unit/CalculatorTest.php'); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('a pruned result does not come back from the fallback', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master', failing: ['adds two numbers']); + + $project->git()->switchTo('feature-x', new: true); + + $project->write('tests/Unit/CalculatorTest.php', <<<'PHP' + add(1, 2))->toBe(3); + }); + + test('subtracts two numbers', function (): void { + expect((new Calculator)->subtract(3, 1))->toBe(2); + }); + PHP); + + $project->git()->commit('rename the test on the branch'); + + $project->pest('--tia', '--filtered', ...$arguments); + + $second = $project->pest('--tia', '--filtered', ...$arguments); + + expect($second->output)->not->toContain('previously unsuccessful') + ->and($second->output)->toContain('No affected tests found'); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('the fallback still reaches a branch that has never run a test file', function (): void { + $project = Project::make('master'); + $project->seed('master', failing: ['adds two numbers']); + + $project->git()->switchTo('feature-x', new: true); + + // A narrowed run mints the branch key holding only the Greeter entries, so + // the layering must still serve master's cached failure for the Calculator. + $project->pest('--filter=greets a person'); + + $result = $project->pest('--tia', '--filtered'); + + expect($result->output)->toContain('previously unsuccessful') + ->and($result->affected())->toBe(2, $result->describe()); +})->skipOnWindows(); + +test('a branch that git no longer knows loses its baseline', function (): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->git()->switchTo('feature-x', new: true); + $project->pest('--tia'); + + expect($project->branchKeys())->toBe(['master', 'feature-x']); + + $project->git()->switchTo('master'); + $project->git()->run(['branch', '-D', 'feature-x']); + + $project->pest('--tia'); + + expect($project->branchKeys())->toBe(['master']); +})->skipOnWindows(); + +test('an unknown cached status is re-run rather than replayed as a failure', function (int $status): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->mutateGraph(function (array $graph) use ($status): array { + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + $graph['baselines']['master']['results'][$id]['status'] = $status; + + return $graph; + }); + + $result = $project->pest('--tia'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed') + ->and($result->uncached())->toBe(1, $result->describe()); +})->with(['unknown' => -1, 'future' => 9, 'garbage' => 99])->skipOnWindows(); + +test('a cached notice, deprecation or warning does not replay as a failure', function (int $status): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->mutateGraph(function (array $graph) use ($status): array { + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + $graph['baselines']['master']['results'][$id]['status'] = $status; + $graph['baselines']['master']['results'][$id]['message'] = 'cached detail'; + + return $graph; + }); + + $result = $project->pest('--tia'); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed'); +})->with(['notice' => 3, 'deprecation' => 4, 'warning' => 6])->skipOnWindows(); + +test('a malformed baseline entry cannot break the run', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->mutateGraph(function (array $graph): array { + $id = Project::testId('tests/Unit/CalculatorTest.php', 'adds two numbers'); + $graph['baselines']['master']['results'][$id] = 'nope'; + $graph['baselines']['master']['tree'] = 'nope'; + $graph['edges']['tests/Unit/GreeterTest.php'] = 'nope'; + + return $graph; + }); + + $result = $project->pest('--tia', ...$arguments); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($result->output)->not->toContain('TypeError') + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed'); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +/* + * Invariant 1 — sequential and parallel must agree — under a process that is + * torn down in the middle of a test file. A worker that flushed what it got to + * before dying has not seen enough of that file to license pruning the + * siblings it never reached. + */ +test('a run torn down mid-file does not prune the tests it never reached', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/CalculatorTest.php', <<<'PHP' + add(1, 2))->toBe(3); + }); + + test('subtracts two numbers', function (): void { + exit(0); + }); + PHP); + + $project->git()->commit('a test that kills its own process'); + + $project->pest('--tia', ...$arguments); + $delta = $project->delta(); + + expect($delta->removed())->toBe(0, $delta->summary()) + ->and($delta->shaMoved())->toBeFalse($delta->summary()) + ->and($delta->structureMoved())->toBeFalse($delta->summary()) + ->and(array_keys($project->graph()['baselines']['master']['results'])) + ->toContain(Project::testId('tests/Unit/CalculatorTest.php', 'subtracts two numbers')); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('a fatal error mid-file is a test error, not a truncation', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->write('tests/Unit/CalculatorTest.php', <<<'PHP' + add(1, 2))->toBe(3); + }); + + test('subtracts two numbers', function (): void { + undefined_function_here(); + }); + PHP); + + $project->git()->commit('a test that fatals'); + + $result = $project->pest('--tia', ...$arguments); + $delta = $project->delta(); + + expect($result->exitCode)->not->toBe(0) + ->and($result->tally())->toContain('1 failed') + ->and($delta->removed())->toBe(0, $delta->summary()) + ->and($delta->structureMoved())->toBeFalse($delta->summary()); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +/* + * Invariant 2 — every command lands in exactly one tier and stays inside it — + * for the combinations that were never exercised. + */ +test('a green complete run leaves the graph exactly as it found it', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--tia', ...$arguments); + $delta = $project->delta(); + + expect($result->replayed())->toBe(Project::TOTAL_TESTS, $result->describe()) + ->and($delta->isHardSuppressed())->toBeTrue($delta->summary()); +})->with([ + 'bail' => [['--bail']], + 'stop-on-failure' => [['--stop-on-failure']], + 'compact' => [['--compact']], + 'parallel bail' => [['--parallel', '--processes=2', '--bail']], + 'parallel one process' => [['--parallel', '--processes=1']], + 'parallel more processes than files' => [['--parallel', '--processes=8']], +])->skipOnWindows(); + +test('--tia --no-tia is a plain run that still refreshes what it executed', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--tia', '--no-tia', ...$arguments); + $delta = $project->delta(); + + expect($result->replayed())->toBe(0, $result->describe()) + ->and($result->tally())->toContain(Project::TOTAL_TESTS.' passed') + ->and($delta->writtenCount())->toBe(Project::TOTAL_TESTS, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('--fresh on a partial run neither purges nor prunes', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master'); + + $result = $project->pest('--tia', '--fresh', '--filter=adds two numbers', ...$arguments); + $delta = $project->delta(); + + expect($result->exitCode)->toBe(0, $result->describe()) + ->and($project->graphExists())->toBeTrue() + ->and($delta->writtenCount())->toBe(1, $delta->summary()) + ->and($delta->isResultsOnly())->toBeTrue($delta->summary()); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); + +test('a second green run on a feature branch writes nothing at all', function (array $arguments): void { + $project = Project::make('master'); + $project->seed('master'); + + $project->git()->switchTo('feature-x', new: true); + $project->pest('--tia', ...$arguments); + + $project->snapshot(); + $result = $project->pest('--tia', ...$arguments); + $delta = $project->delta(); + + expect($result->replayed())->toBe(Project::TOTAL_TESTS, $result->describe()) + ->and($delta->isHardSuppressed())->toBeTrue($delta->summary()); +})->with(Project::SEQUENTIAL_AND_PARALLEL)->skipOnWindows(); diff --git a/tests/Fixtures/Tia/GraphDelta.php b/tests/Fixtures/Tia/GraphDelta.php index 7d930e78..a88e1861 100644 --- a/tests/Fixtures/Tia/GraphDelta.php +++ b/tests/Fixtures/Tia/GraphDelta.php @@ -87,7 +87,7 @@ final readonly class GraphDelta */ public function branchKeys(): array { - return array_keys($this->baselines($this->after)); + return array_map(strval(...), array_keys($this->baselines($this->after))); } /** @@ -95,7 +95,7 @@ final readonly class GraphDelta */ public function branchKeysBefore(): array { - return array_keys($this->baselines($this->before)); + return array_map(strval(...), array_keys($this->baselines($this->before))); } public function branchKeysMoved(): bool @@ -254,9 +254,7 @@ final readonly class GraphDelta $baselines = []; foreach ($this->section($graph, 'baselines') as $branch => $baseline) { - if (is_string($branch)) { - $baselines[$branch] = $baseline; - } + $baselines[(string) $branch] = $baseline; } return $baselines; diff --git a/tests/Fixtures/Tia/Project.php b/tests/Fixtures/Tia/Project.php index 3462913d..d019ec71 100644 --- a/tests/Fixtures/Tia/Project.php +++ b/tests/Fixtures/Tia/Project.php @@ -322,7 +322,8 @@ final class Project { $baselines = $this->graph()['baselines'] ?? []; - return is_array($baselines) ? array_keys($baselines) : []; + // A branch named `12345` comes back from json_decode as an integer key. + return is_array($baselines) ? array_map(strval(...), array_keys($baselines)) : []; } public function graphDir(): string