From 4ac14b25283fa23a2bbe6261f5aabee7ad3c6132 Mon Sep 17 00:00:00 2001 From: nuno maduro Date: Tue, 14 Apr 2026 08:34:41 -0700 Subject: [PATCH 1/8] feat(time-based-sharding): updates plugin --- src/Plugins/Help.php | 4 + src/Plugins/Shard.php | 333 +++++++++++++++++- src/Subscribers/EnsureShardTimingFinished.php | 22 ++ src/Subscribers/EnsureShardTimingStarted.php | 22 ++ .../EnsureShardTimingsAreCollected.php | 75 ++++ ...isual_snapshot_of_help_command_output.snap | 1 + tests/Arch.php | 1 + 7 files changed, 455 insertions(+), 3 deletions(-) create mode 100644 src/Subscribers/EnsureShardTimingFinished.php create mode 100644 src/Subscribers/EnsureShardTimingStarted.php create mode 100644 src/Subscribers/EnsureShardTimingsAreCollected.php diff --git a/src/Plugins/Help.php b/src/Plugins/Help.php index 0795c806..12d03532 100644 --- a/src/Plugins/Help.php +++ b/src/Plugins/Help.php @@ -123,6 +123,10 @@ final readonly class Help implements HandlesArguments 'arg' => '--update-snapshots', 'desc' => 'Update snapshots for tests using the "toMatchSnapshot" expectation', ], + [ + 'arg' => '--update-shards', + 'desc' => 'Update shards.json with test timing data for time-balanced sharding', + ], ], ...$content['Execution']]; $content['Selection'] = [[ diff --git a/src/Plugins/Shard.php b/src/Plugins/Shard.php index f48260bb..abd835ea 100644 --- a/src/Plugins/Shard.php +++ b/src/Plugins/Shard.php @@ -6,7 +6,13 @@ namespace Pest\Plugins; use Pest\Contracts\Plugins\AddsOutput; use Pest\Contracts\Plugins\HandlesArguments; +use Pest\Contracts\Plugins\Terminable; use Pest\Exceptions\InvalidOption; +use Pest\Subscribers\EnsureShardTimingFinished; +use Pest\Subscribers\EnsureShardTimingStarted; +use Pest\Subscribers\EnsureShardTimingsAreCollected; +use Pest\TestSuite; +use PHPUnit\Event; use Symfony\Component\Console\Input\ArgvInput; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -15,7 +21,7 @@ use Symfony\Component\Process\Process; /** * @internal */ -final class Shard implements AddsOutput, HandlesArguments +final class Shard implements AddsOutput, HandlesArguments, Terminable { use Concerns\HandleArguments; @@ -33,6 +39,30 @@ final class Shard implements AddsOutput, HandlesArguments */ private static ?array $shard = null; + /** + * Whether to update the shards.json file. + */ + private static bool $updateShards = false; + + /** + * Whether time-balanced sharding was used. + */ + private static bool $timeBalanced = false; + + /** + * Collected timings from workers or subscribers. + * + * @var array|null + */ + private static ?array $collectedTimings = null; + + /** + * The canonical list of test classes from --list-tests. + * + * @var list|null + */ + private static ?array $knownTests = null; + /** * Creates a new Plugin instance. */ @@ -47,6 +77,19 @@ final class Shard implements AddsOutput, HandlesArguments */ public function handleArguments(array $arguments): array { + if ($this->hasArgument('--update-shards', $arguments)) { + return $this->handleUpdateShards($arguments); + } + + if (Parallel::isWorker() && Parallel::getGlobal('UPDATE_SHARDS') === true) { + self::$updateShards = true; + + Event\Facade::instance()->registerSubscriber(new EnsureShardTimingStarted); + Event\Facade::instance()->registerSubscriber(new EnsureShardTimingFinished); + + return $arguments; + } + if (! $this->hasArgument('--shard', $arguments)) { return $arguments; } @@ -63,7 +106,21 @@ final class Shard implements AddsOutput, HandlesArguments /** @phpstan-ignore-next-line */ $tests = $this->allTests($arguments); - $testsToRun = (array_chunk($tests, max(1, (int) ceil(count($tests) / $total))))[$index - 1] ?? []; + + $timings = self::loadShardsFile(); + if ($timings !== null) { + $missingTests = array_diff($tests, array_keys($timings)); + + if ($missingTests !== []) { + throw new InvalidOption('The [tests/.pest/shards.json] file is out of date. Run [--update-shards] to update it.'); + } + + $partitions = self::partitionByTime($tests, $timings, $total); + $testsToRun = $partitions[$index - 1] ?? []; + self::$timeBalanced = true; + } else { + $testsToRun = (array_chunk($tests, max(1, (int) ceil(count($tests) / $total))))[$index - 1] ?? []; + } self::$shard = [ 'index' => $index, @@ -75,6 +132,36 @@ final class Shard implements AddsOutput, HandlesArguments return [...$arguments, '--filter', $this->buildFilterArgument($testsToRun)]; } + /** + * Handles the --update-shards argument. + * + * @param array $arguments + * @return array + */ + private function handleUpdateShards(array $arguments): array + { + if ($this->hasArgument('--shard', $arguments)) { + throw new InvalidOption('The [--update-shards] option cannot be combined with [--shard].'); + } + + $arguments = $this->popArgument('--update-shards', $arguments); + + self::$updateShards = true; + + /** @phpstan-ignore-next-line */ + self::$knownTests = $this->allTests($arguments); + + if ($this->hasArgument('--parallel', $arguments) || $this->hasArgument('-p', $arguments)) { + Parallel::setGlobal('UPDATE_SHARDS', true); + Parallel::setGlobal('SHARD_RUN_ID', uniqid('pest-shard-', true)); + } else { + Event\Facade::instance()->registerSubscriber(new EnsureShardTimingStarted); + Event\Facade::instance()->registerSubscriber(new EnsureShardTimingFinished); + } + + return $arguments; + } + /** * Returns all tests that the test suite would run. * @@ -116,6 +203,20 @@ final class Shard implements AddsOutput, HandlesArguments */ public function addOutput(int $exitCode): int { + if (self::$updateShards && ! Parallel::isWorker()) { + self::$collectedTimings = self::collectTimings(); + + $count = self::$knownTests !== null + ? count(array_intersect_key(self::$collectedTimings, array_flip(self::$knownTests))) + : count(self::$collectedTimings); + + $this->output->writeln(sprintf( + ' Shards: shards.json updated with timings for %d test class%s.', + $count, + $count === 1 ? '' : 'es', + )); + } + if (self::$shard === null) { return $exitCode; } @@ -128,17 +229,243 @@ final class Shard implements AddsOutput, HandlesArguments ] = self::$shard; $this->output->writeln(sprintf( - ' Shard: %d of %d — %d file%s ran, out of %d.', + ' Shard: %d of %d — %d file%s ran, out of %d%s.', $index, $total, $testsRan, $testsRan === 1 ? '' : 's', $testsCount, + self::$timeBalanced ? ' (time-balanced)' : '', )); return $exitCode; } + /** + * Terminates the plugin. + */ + public function terminate(): void + { + if (! self::$updateShards) { + return; + } + + if (Parallel::isWorker()) { + self::writeWorkerTimings(); + + return; + } + + $timings = self::$collectedTimings ?? self::collectTimings(); + + if ($timings === []) { + return; + } + + self::writeTimings($timings); + } + + /** + * Collects timings from subscribers or worker temp files. + * + * @return array + */ + private static function collectTimings(): array + { + $runId = Parallel::getGlobal('SHARD_RUN_ID'); + + if (is_string($runId)) { + return self::readWorkerTimings($runId); + } + + return EnsureShardTimingsAreCollected::timings(); + } + + /** + * Writes the current worker's timing data to a temp file. + */ + private static function writeWorkerTimings(): void + { + $timings = EnsureShardTimingsAreCollected::timings(); + + if ($timings === []) { + return; + } + + $runId = Parallel::getGlobal('SHARD_RUN_ID'); + + if (! is_string($runId)) { + return; + } + + $path = sys_get_temp_dir().DIRECTORY_SEPARATOR.$runId.'-'.getmypid().'.json'; + + file_put_contents($path, json_encode($timings, JSON_THROW_ON_ERROR)); + } + + /** + * Reads and merges timing data from all worker temp files. + * + * @return array + */ + private static function readWorkerTimings(string $runId): array + { + $pattern = sys_get_temp_dir().DIRECTORY_SEPARATOR.$runId.'-*.json'; + $files = glob($pattern); + + if ($files === false || $files === []) { + return []; + } + + $merged = []; + + foreach ($files as $file) { + $contents = file_get_contents($file); + + if ($contents === false) { + continue; + } + + $timings = json_decode($contents, true); + + if (is_array($timings)) { + $merged = array_merge($merged, $timings); + } + + unlink($file); + } + + return $merged; + } + + /** + * Returns the path to shards.json. + */ + private static function shardsPath(): string + { + $testSuite = TestSuite::getInstance(); + + return implode(DIRECTORY_SEPARATOR, [$testSuite->rootPath, $testSuite->testPath, '.pest', 'shards.json']); + } + + /** + * Loads the timings from shards.json. + * + * @return array|null + */ + private static function loadShardsFile(): ?array + { + $path = self::shardsPath(); + + if (! file_exists($path)) { + return null; + } + + $contents = file_get_contents($path); + + if ($contents === false) { + return null; + } + + $data = json_decode($contents, true); + + if (! is_array($data) || ! isset($data['timings']) || ! is_array($data['timings'])) { + return null; + } + + /** @var array */ + return $data['timings']; + } + + /** + * Partitions tests across shards using the LPT (Longest Processing Time) algorithm. + * + * @param list $tests + * @param array $timings + * @return list> + */ + private static function partitionByTime(array $tests, array $timings, int $total): array + { + $knownTimings = array_filter( + array_map(fn (string $test): ?float => $timings[$test] ?? null, $tests), + fn (?float $t): bool => $t !== null, + ); + + $median = $knownTimings !== [] ? self::median(array_values($knownTimings)) : 1.0; + + $testsWithTimings = array_map( + fn (string $test): array => ['test' => $test, 'time' => $timings[$test] ?? $median], + $tests, + ); + + usort($testsWithTimings, fn (array $a, array $b): int => $b['time'] <=> $a['time']); + + /** @var list> */ + $bins = array_fill(0, $total, []); + /** @var non-empty-list */ + $binTimes = array_fill(0, $total, 0.0); + + foreach ($testsWithTimings as $item) { + $minIndex = array_search(min($binTimes), $binTimes, strict: true); + assert(is_int($minIndex)); + + $bins[$minIndex][] = $item['test']; + $binTimes[$minIndex] += $item['time']; + } + + return $bins; + } + + /** + * Calculates the median of an array of floats. + * + * @param list $values + */ + private static function median(array $values): float + { + sort($values); + + $count = count($values); + $middle = (int) floor($count / 2); + + if ($count % 2 === 0) { + return ($values[$middle - 1] + $values[$middle]) / 2; + } + + return $values[$middle]; + } + + /** + * Writes the timings to shards.json. + * + * @param array $timings + */ + private static function writeTimings(array $timings): void + { + $path = self::shardsPath(); + + $directory = dirname($path); + if (! is_dir($directory)) { + mkdir($directory, 0755, true); + } + + if (self::$knownTests !== null) { + $knownSet = array_flip(self::$knownTests); + $timings = array_intersect_key($timings, $knownSet); + } + + ksort($timings); + + $canonical = self::$knownTests ?? array_keys($timings); + sort($canonical); + + file_put_contents($path, json_encode([ + 'timings' => $timings, + 'checksum' => md5(implode("\n", $canonical)), + 'updated_at' => date('c'), + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)."\n"); + } + /** * Returns the shard information. * diff --git a/src/Subscribers/EnsureShardTimingFinished.php b/src/Subscribers/EnsureShardTimingFinished.php new file mode 100644 index 00000000..f1732d9b --- /dev/null +++ b/src/Subscribers/EnsureShardTimingFinished.php @@ -0,0 +1,22 @@ + + */ + private static array $startTimes = []; + + /** + * The collected timings for each test class. + * + * @var array + */ + private static array $timings = []; + + /** + * Records the start time for a test suite. + */ + public static function started(Started $event): void + { + if (! $event->testSuite()->isForTestClass()) { + return; + } + + $name = preg_replace('/^P\\\\/', '', $event->testSuite()->name()); + + if (is_string($name)) { + self::$startTimes[$name] = $event->telemetryInfo()->time(); + } + } + + /** + * Records the duration for a test suite. + */ + public static function finished(Finished $event): void + { + if (! $event->testSuite()->isForTestClass()) { + return; + } + + $name = preg_replace('/^P\\\\/', '', $event->testSuite()->name()); + + if (! is_string($name) || ! isset(self::$startTimes[$name])) { + return; + } + + $duration = $event->telemetryInfo()->time()->duration(self::$startTimes[$name]); + + self::$timings[$name] = round($duration->asFloat(), 4); + } + + /** + * Returns the collected timings. + * + * @return array + */ + public static function timings(): array + { + return self::$timings; + } +} diff --git a/tests/.pest/snapshots/Visual/Help/visual_snapshot_of_help_command_output.snap b/tests/.pest/snapshots/Visual/Help/visual_snapshot_of_help_command_output.snap index f8eee485..5a7f0c87 100644 --- a/tests/.pest/snapshots/Visual/Help/visual_snapshot_of_help_command_output.snap +++ b/tests/.pest/snapshots/Visual/Help/visual_snapshot_of_help_command_output.snap @@ -49,6 +49,7 @@ EXECUTION OPTIONS: --parallel ........................................... Run tests in parallel --update-snapshots Update snapshots for tests using the "toMatchSnapshot" expectation + --update-shards Update shards.json with test timing data for time-balanced sharding --globals-backup ................. Backup and restore $GLOBALS for each test --static-backup ......... Backup and restore static properties for each test --strict-coverage ................... Be strict about code coverage metadata diff --git a/tests/Arch.php b/tests/Arch.php index 6348a0f3..40bd9b89 100644 --- a/tests/Arch.php +++ b/tests/Arch.php @@ -17,6 +17,7 @@ arch()->preset()->security()->ignoring([ 'eval', 'str_shuffle', 'exec', + 'md5', 'unserialize', 'extract', 'assert', From 10aee6045cd4f91a04b2c1e7e1c16f756482370a Mon Sep 17 00:00:00 2001 From: nuno maduro Date: Tue, 14 Apr 2026 09:08:52 -0700 Subject: [PATCH 2/8] feat(time-based-sharding): updates exception name --- src/Plugins/Shard.php | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/Plugins/Shard.php b/src/Plugins/Shard.php index abd835ea..0a660780 100644 --- a/src/Plugins/Shard.php +++ b/src/Plugins/Shard.php @@ -34,7 +34,8 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable * index: int, * total: int, * testsRan: int, - * testsCount: int + * testsCount: int, + * estimatedTime: float|null * }|null */ private static ?array $shard = null; @@ -122,11 +123,16 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable $testsToRun = (array_chunk($tests, max(1, (int) ceil(count($tests) / $total))))[$index - 1] ?? []; } + $estimatedTime = self::$timeBalanced && $timings !== null + ? array_sum(array_map(fn (string $test): float => $timings[$test] ?? 0.0, $testsToRun)) + : null; + self::$shard = [ 'index' => $index, 'total' => $total, 'testsRan' => count($testsToRun), 'testsCount' => count($tests), + 'estimatedTime' => $estimatedTime, ]; return [...$arguments, '--filter', $this->buildFilterArgument($testsToRun)]; @@ -174,7 +180,7 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable 'php', ...$this->removeParallelArguments($arguments), '--list-tests', - ]))->mustRun()->getOutput(); + ]))->setTimeout(120)->mustRun()->getOutput(); preg_match_all('/ - (?:P\\\\)?(Tests\\\\[^:]+)::/', $output, $matches); @@ -226,8 +232,14 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable 'total' => $total, 'testsRan' => $testsRan, 'testsCount' => $testsCount, + 'estimatedTime' => $estimatedTime, ] = self::$shard; + $suffix = ''; + if (self::$timeBalanced && is_float($estimatedTime)) { + $suffix = sprintf(' (time-balanced, ~%.1fs)', $estimatedTime); + } + $this->output->writeln(sprintf( ' Shard: %d of %d — %d file%s ran, out of %d%s.', $index, @@ -235,7 +247,7 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable $testsRan, $testsRan === 1 ? '' : 's', $testsCount, - self::$timeBalanced ? ' (time-balanced)' : '', + $suffix, )); return $exitCode; @@ -364,13 +376,13 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable $contents = file_get_contents($path); if ($contents === false) { - return null; + throw new InvalidOption('The [tests/.pest/shards.json] file could not be read. Delete it or run [--update-shards] to regenerate.'); } $data = json_decode($contents, true); if (! is_array($data) || ! isset($data['timings']) || ! is_array($data['timings'])) { - return null; + throw new InvalidOption('The [tests/.pest/shards.json] file is corrupted. Delete it or run [--update-shards] to regenerate.'); } /** @var array */ From 985dadd93456b796d00da3db8a8a082aedf86617 Mon Sep 17 00:00:00 2001 From: nuno maduro Date: Tue, 14 Apr 2026 09:16:32 -0700 Subject: [PATCH 3/8] update --- src/Plugins/Shard.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Plugins/Shard.php b/src/Plugins/Shard.php index 0a660780..4ccabb77 100644 --- a/src/Plugins/Shard.php +++ b/src/Plugins/Shard.php @@ -310,7 +310,7 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable return; } - $path = sys_get_temp_dir().DIRECTORY_SEPARATOR.$runId.'-'.getmypid().'.json'; + $path = sys_get_temp_dir().DIRECTORY_SEPARATOR.'__pest_sharding_'.$runId.'-'.getmypid().'.json'; file_put_contents($path, json_encode($timings, JSON_THROW_ON_ERROR)); } @@ -322,7 +322,7 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable */ private static function readWorkerTimings(string $runId): array { - $pattern = sys_get_temp_dir().DIRECTORY_SEPARATOR.$runId.'-*.json'; + $pattern = sys_get_temp_dir().DIRECTORY_SEPARATOR.'__pest_sharding_'.$runId.'-*.json'; $files = glob($pattern); if ($files === false || $files === []) { From cb5f6e1bd28d9e991a34f4c154eaf54ac22e5b9c Mon Sep 17 00:00:00 2001 From: nuno maduro Date: Tue, 14 Apr 2026 09:17:18 -0700 Subject: [PATCH 4/8] chore: style --- src/Plugins/Shard.php | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/src/Plugins/Shard.php b/src/Plugins/Shard.php index 4ccabb77..f6cb3ac1 100644 --- a/src/Plugins/Shard.php +++ b/src/Plugins/Shard.php @@ -9,8 +9,8 @@ use Pest\Contracts\Plugins\HandlesArguments; use Pest\Contracts\Plugins\Terminable; use Pest\Exceptions\InvalidOption; use Pest\Subscribers\EnsureShardTimingFinished; -use Pest\Subscribers\EnsureShardTimingStarted; use Pest\Subscribers\EnsureShardTimingsAreCollected; +use Pest\Subscribers\EnsureShardTimingStarted; use Pest\TestSuite; use PHPUnit\Event; use Symfony\Component\Console\Input\ArgvInput; @@ -108,7 +108,7 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable /** @phpstan-ignore-next-line */ $tests = $this->allTests($arguments); - $timings = self::loadShardsFile(); + $timings = $this->loadShardsFile(); if ($timings !== null) { $missingTests = array_diff($tests, array_keys($timings)); @@ -116,7 +116,7 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable throw new InvalidOption('The [tests/.pest/shards.json] file is out of date. Run [--update-shards] to update it.'); } - $partitions = self::partitionByTime($tests, $timings, $total); + $partitions = $this->partitionByTime($tests, $timings, $total); $testsToRun = $partitions[$index - 1] ?? []; self::$timeBalanced = true; } else { @@ -210,7 +210,7 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable public function addOutput(int $exitCode): int { if (self::$updateShards && ! Parallel::isWorker()) { - self::$collectedTimings = self::collectTimings(); + self::$collectedTimings = $this->collectTimings(); $count = self::$knownTests !== null ? count(array_intersect_key(self::$collectedTimings, array_flip(self::$knownTests))) @@ -263,18 +263,18 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable } if (Parallel::isWorker()) { - self::writeWorkerTimings(); + $this->writeWorkerTimings(); return; } - $timings = self::$collectedTimings ?? self::collectTimings(); + $timings = self::$collectedTimings ?? $this->collectTimings(); if ($timings === []) { return; } - self::writeTimings($timings); + $this->writeTimings($timings); } /** @@ -282,12 +282,12 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable * * @return array */ - private static function collectTimings(): array + private function collectTimings(): array { $runId = Parallel::getGlobal('SHARD_RUN_ID'); if (is_string($runId)) { - return self::readWorkerTimings($runId); + return $this->readWorkerTimings($runId); } return EnsureShardTimingsAreCollected::timings(); @@ -296,7 +296,7 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable /** * Writes the current worker's timing data to a temp file. */ - private static function writeWorkerTimings(): void + private function writeWorkerTimings(): void { $timings = EnsureShardTimingsAreCollected::timings(); @@ -320,7 +320,7 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable * * @return array */ - private static function readWorkerTimings(string $runId): array + private function readWorkerTimings(string $runId): array { $pattern = sys_get_temp_dir().DIRECTORY_SEPARATOR.'__pest_sharding_'.$runId.'-*.json'; $files = glob($pattern); @@ -353,7 +353,7 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable /** * Returns the path to shards.json. */ - private static function shardsPath(): string + private function shardsPath(): string { $testSuite = TestSuite::getInstance(); @@ -365,9 +365,9 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable * * @return array|null */ - private static function loadShardsFile(): ?array + private function loadShardsFile(): ?array { - $path = self::shardsPath(); + $path = $this->shardsPath(); if (! file_exists($path)) { return null; @@ -385,7 +385,6 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable throw new InvalidOption('The [tests/.pest/shards.json] file is corrupted. Delete it or run [--update-shards] to regenerate.'); } - /** @var array */ return $data['timings']; } @@ -396,14 +395,14 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable * @param array $timings * @return list> */ - private static function partitionByTime(array $tests, array $timings, int $total): array + private function partitionByTime(array $tests, array $timings, int $total): array { $knownTimings = array_filter( array_map(fn (string $test): ?float => $timings[$test] ?? null, $tests), fn (?float $t): bool => $t !== null, ); - $median = $knownTimings !== [] ? self::median(array_values($knownTimings)) : 1.0; + $median = $knownTimings !== [] ? $this->median(array_values($knownTimings)) : 1.0; $testsWithTimings = array_map( fn (string $test): array => ['test' => $test, 'time' => $timings[$test] ?? $median], @@ -433,7 +432,7 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable * * @param list $values */ - private static function median(array $values): float + private function median(array $values): float { sort($values); @@ -452,9 +451,9 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable * * @param array $timings */ - private static function writeTimings(array $timings): void + private function writeTimings(array $timings): void { - $path = self::shardsPath(); + $path = $this->shardsPath(); $directory = dirname($path); if (! is_dir($directory)) { From 7cbb1fcdb233281edf11e9796c88c9190f867f94 Mon Sep 17 00:00:00 2001 From: nuno maduro Date: Tue, 14 Apr 2026 09:29:41 -0700 Subject: [PATCH 5/8] wip --- src/Plugins/Shard.php | 20 ++++++++++++++++---- tests/Arch.php | 1 + 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/Plugins/Shard.php b/src/Plugins/Shard.php index f6cb3ac1..e47920d4 100644 --- a/src/Plugins/Shard.php +++ b/src/Plugins/Shard.php @@ -50,6 +50,11 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable */ private static bool $timeBalanced = false; + /** + * Whether the shards.json file is outdated. + */ + private static bool $shardsOutdated = false; + /** * Collected timings from workers or subscribers. * @@ -110,15 +115,18 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable $timings = $this->loadShardsFile(); if ($timings !== null) { - $missingTests = array_diff($tests, array_keys($timings)); + $knownTests = array_values(array_filter($tests, fn (string $test): bool => isset($timings[$test]))); + $newTests = array_values(array_diff($tests, $knownTests)); - if ($missingTests !== []) { - throw new InvalidOption('The [tests/.pest/shards.json] file is out of date. Run [--update-shards] to update it.'); + $partitions = $this->partitionByTime($knownTests, $timings, $total); + + foreach ($newTests as $i => $test) { + $partitions[$i % $total][] = $test; } - $partitions = $this->partitionByTime($tests, $timings, $total); $testsToRun = $partitions[$index - 1] ?? []; self::$timeBalanced = true; + self::$shardsOutdated = $newTests !== []; } else { $testsToRun = (array_chunk($tests, max(1, (int) ceil(count($tests) / $total))))[$index - 1] ?? []; } @@ -250,6 +258,10 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable $suffix, )); + if (self::$shardsOutdated) { + $this->output->writeln(' WARN The [tests/.pest/shards.json] file is out of date. Run [--update-shards] to update it.'); + } + return $exitCode; } diff --git a/tests/Arch.php b/tests/Arch.php index 40bd9b89..d0565216 100644 --- a/tests/Arch.php +++ b/tests/Arch.php @@ -19,6 +19,7 @@ arch()->preset()->security()->ignoring([ 'exec', 'md5', 'unserialize', + 'uniqid', 'extract', 'assert', ]); From e616eab9fb89d3cf83d7939bbeef91b0babec7d9 Mon Sep 17 00:00:00 2001 From: nuno maduro Date: Tue, 14 Apr 2026 09:36:38 -0700 Subject: [PATCH 6/8] wip --- src/ArchPresets/Laravel.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/ArchPresets/Laravel.php b/src/ArchPresets/Laravel.php index 406611a3..3b727340 100644 --- a/src/ArchPresets/Laravel.php +++ b/src/ArchPresets/Laravel.php @@ -176,9 +176,5 @@ final class Laravel extends AbstractPreset ->toImplement('Illuminate\Contracts\Container\ContextualAttribute') ->toHaveAttribute('Attribute') ->toHaveMethod('resolve'); - - $this->expectations[] = expect('App\Rules') - ->classes() - ->toImplement('Illuminate\Contracts\Validation\ValidationRule'); } } From 0acab1cbb40bc7501944097ca99cbef357d7c168 Mon Sep 17 00:00:00 2001 From: nuno maduro Date: Tue, 14 Apr 2026 09:53:57 -0700 Subject: [PATCH 7/8] wip --- src/Plugins/Shard.php | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/Plugins/Shard.php b/src/Plugins/Shard.php index e47920d4..ef11f632 100644 --- a/src/Plugins/Shard.php +++ b/src/Plugins/Shard.php @@ -55,6 +55,11 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable */ private static bool $shardsOutdated = false; + /** + * Whether the test suite passed. + */ + private static bool $passed = false; + /** * Collected timings from workers or subscribers. * @@ -217,7 +222,9 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable */ public function addOutput(int $exitCode): int { - if (self::$updateShards && ! Parallel::isWorker()) { + self::$passed = $exitCode === 0; + + if (self::$updateShards && self::$passed && ! Parallel::isWorker()) { self::$collectedTimings = $this->collectTimings(); $count = self::$knownTests !== null @@ -280,6 +287,10 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable return; } + if (! self::$passed) { + return; + } + $timings = self::$collectedTimings ?? $this->collectTimings(); if ($timings === []) { From 9b64d5425a136ce23b0404d07632c4a1c4d8c257 Mon Sep 17 00:00:00 2001 From: nuno maduro Date: Tue, 14 Apr 2026 10:12:57 -0700 Subject: [PATCH 8/8] removes time balanced --- src/Plugins/Shard.php | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/Plugins/Shard.php b/src/Plugins/Shard.php index ef11f632..9575a906 100644 --- a/src/Plugins/Shard.php +++ b/src/Plugins/Shard.php @@ -34,8 +34,7 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable * index: int, * total: int, * testsRan: int, - * testsCount: int, - * estimatedTime: float|null + * testsCount: int * }|null */ private static ?array $shard = null; @@ -136,16 +135,11 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable $testsToRun = (array_chunk($tests, max(1, (int) ceil(count($tests) / $total))))[$index - 1] ?? []; } - $estimatedTime = self::$timeBalanced && $timings !== null - ? array_sum(array_map(fn (string $test): float => $timings[$test] ?? 0.0, $testsToRun)) - : null; - self::$shard = [ 'index' => $index, 'total' => $total, 'testsRan' => count($testsToRun), 'testsCount' => count($tests), - 'estimatedTime' => $estimatedTime, ]; return [...$arguments, '--filter', $this->buildFilterArgument($testsToRun)]; @@ -247,14 +241,8 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable 'total' => $total, 'testsRan' => $testsRan, 'testsCount' => $testsCount, - 'estimatedTime' => $estimatedTime, ] = self::$shard; - $suffix = ''; - if (self::$timeBalanced && is_float($estimatedTime)) { - $suffix = sprintf(' (time-balanced, ~%.1fs)', $estimatedTime); - } - $this->output->writeln(sprintf( ' Shard: %d of %d — %d file%s ran, out of %d%s.', $index, @@ -262,7 +250,7 @@ final class Shard implements AddsOutput, HandlesArguments, Terminable $testsRan, $testsRan === 1 ? '' : 's', $testsCount, - $suffix, + self::$timeBalanced ? ' (time-balanced)' : '', )); if (self::$shardsOutdated) {