diff --git a/src/ArchPresets/Laravel.php b/src/ArchPresets/Laravel.php index 406611a3..3b727340 100644 --- a/src/ArchPresets/Laravel.php +++ b/src/ArchPresets/Laravel.php @@ -176,9 +176,5 @@ final class Laravel extends AbstractPreset ->toImplement('Illuminate\Contracts\Container\ContextualAttribute') ->toHaveAttribute('Attribute') ->toHaveMethod('resolve'); - - $this->expectations[] = expect('App\Rules') - ->classes() - ->toImplement('Illuminate\Contracts\Validation\ValidationRule'); } } diff --git a/src/Plugins/Help.php b/src/Plugins/Help.php index 0795c806..12d03532 100644 --- a/src/Plugins/Help.php +++ b/src/Plugins/Help.php @@ -123,6 +123,10 @@ final readonly class Help implements HandlesArguments 'arg' => '--update-snapshots', 'desc' => 'Update snapshots for tests using the "toMatchSnapshot" expectation', ], + [ + 'arg' => '--update-shards', + 'desc' => 'Update shards.json with test timing data for time-balanced sharding', + ], ], ...$content['Execution']]; $content['Selection'] = [[ diff --git a/src/Plugins/Shard.php b/src/Plugins/Shard.php index f48260bb..9575a906 100644 --- a/src/Plugins/Shard.php +++ b/src/Plugins/Shard.php @@ -6,7 +6,13 @@ namespace Pest\Plugins; use Pest\Contracts\Plugins\AddsOutput; use Pest\Contracts\Plugins\HandlesArguments; +use Pest\Contracts\Plugins\Terminable; use Pest\Exceptions\InvalidOption; +use Pest\Subscribers\EnsureShardTimingFinished; +use Pest\Subscribers\EnsureShardTimingsAreCollected; +use Pest\Subscribers\EnsureShardTimingStarted; +use Pest\TestSuite; +use PHPUnit\Event; use Symfony\Component\Console\Input\ArgvInput; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -15,7 +21,7 @@ use Symfony\Component\Process\Process; /** * @internal */ -final class Shard implements AddsOutput, HandlesArguments +final class Shard implements AddsOutput, HandlesArguments, Terminable { use Concerns\HandleArguments; @@ -33,6 +39,40 @@ final class Shard implements AddsOutput, HandlesArguments */ private static ?array $shard = null; + /** + * Whether to update the shards.json file. + */ + private static bool $updateShards = false; + + /** + * Whether time-balanced sharding was used. + */ + private static bool $timeBalanced = false; + + /** + * Whether the shards.json file is outdated. + */ + private static bool $shardsOutdated = false; + + /** + * Whether the test suite passed. + */ + private static bool $passed = false; + + /** + * Collected timings from workers or subscribers. + * + * @var array|null + */ + private static ?array $collectedTimings = null; + + /** + * The canonical list of test classes from --list-tests. + * + * @var list|null + */ + private static ?array $knownTests = null; + /** * Creates a new Plugin instance. */ @@ -47,6 +87,19 @@ final class Shard implements AddsOutput, HandlesArguments */ public function handleArguments(array $arguments): array { + if ($this->hasArgument('--update-shards', $arguments)) { + return $this->handleUpdateShards($arguments); + } + + if (Parallel::isWorker() && Parallel::getGlobal('UPDATE_SHARDS') === true) { + self::$updateShards = true; + + Event\Facade::instance()->registerSubscriber(new EnsureShardTimingStarted); + Event\Facade::instance()->registerSubscriber(new EnsureShardTimingFinished); + + return $arguments; + } + if (! $this->hasArgument('--shard', $arguments)) { return $arguments; } @@ -63,7 +116,24 @@ final class Shard implements AddsOutput, HandlesArguments /** @phpstan-ignore-next-line */ $tests = $this->allTests($arguments); - $testsToRun = (array_chunk($tests, max(1, (int) ceil(count($tests) / $total))))[$index - 1] ?? []; + + $timings = $this->loadShardsFile(); + if ($timings !== null) { + $knownTests = array_values(array_filter($tests, fn (string $test): bool => isset($timings[$test]))); + $newTests = array_values(array_diff($tests, $knownTests)); + + $partitions = $this->partitionByTime($knownTests, $timings, $total); + + foreach ($newTests as $i => $test) { + $partitions[$i % $total][] = $test; + } + + $testsToRun = $partitions[$index - 1] ?? []; + self::$timeBalanced = true; + self::$shardsOutdated = $newTests !== []; + } else { + $testsToRun = (array_chunk($tests, max(1, (int) ceil(count($tests) / $total))))[$index - 1] ?? []; + } self::$shard = [ 'index' => $index, @@ -75,6 +145,36 @@ final class Shard implements AddsOutput, HandlesArguments return [...$arguments, '--filter', $this->buildFilterArgument($testsToRun)]; } + /** + * Handles the --update-shards argument. + * + * @param array $arguments + * @return array + */ + private function handleUpdateShards(array $arguments): array + { + if ($this->hasArgument('--shard', $arguments)) { + throw new InvalidOption('The [--update-shards] option cannot be combined with [--shard].'); + } + + $arguments = $this->popArgument('--update-shards', $arguments); + + self::$updateShards = true; + + /** @phpstan-ignore-next-line */ + self::$knownTests = $this->allTests($arguments); + + if ($this->hasArgument('--parallel', $arguments) || $this->hasArgument('-p', $arguments)) { + Parallel::setGlobal('UPDATE_SHARDS', true); + Parallel::setGlobal('SHARD_RUN_ID', uniqid('pest-shard-', true)); + } else { + Event\Facade::instance()->registerSubscriber(new EnsureShardTimingStarted); + Event\Facade::instance()->registerSubscriber(new EnsureShardTimingFinished); + } + + return $arguments; + } + /** * Returns all tests that the test suite would run. * @@ -87,7 +187,7 @@ final class Shard implements AddsOutput, HandlesArguments 'php', ...$this->removeParallelArguments($arguments), '--list-tests', - ]))->mustRun()->getOutput(); + ]))->setTimeout(120)->mustRun()->getOutput(); preg_match_all('/ - (?:P\\\\)?(Tests\\\\[^:]+)::/', $output, $matches); @@ -116,6 +216,22 @@ final class Shard implements AddsOutput, HandlesArguments */ public function addOutput(int $exitCode): int { + self::$passed = $exitCode === 0; + + if (self::$updateShards && self::$passed && ! Parallel::isWorker()) { + self::$collectedTimings = $this->collectTimings(); + + $count = self::$knownTests !== null + ? count(array_intersect_key(self::$collectedTimings, array_flip(self::$knownTests))) + : count(self::$collectedTimings); + + $this->output->writeln(sprintf( + ' Shards: shards.json updated with timings for %d test class%s.', + $count, + $count === 1 ? '' : 'es', + )); + } + if (self::$shard === null) { return $exitCode; } @@ -128,17 +244,250 @@ final class Shard implements AddsOutput, HandlesArguments ] = self::$shard; $this->output->writeln(sprintf( - ' Shard: %d of %d — %d file%s ran, out of %d.', + ' Shard: %d of %d — %d file%s ran, out of %d%s.', $index, $total, $testsRan, $testsRan === 1 ? '' : 's', $testsCount, + self::$timeBalanced ? ' (time-balanced)' : '', )); + if (self::$shardsOutdated) { + $this->output->writeln(' WARN The [tests/.pest/shards.json] file is out of date. Run [--update-shards] to update it.'); + } + return $exitCode; } + /** + * Terminates the plugin. + */ + public function terminate(): void + { + if (! self::$updateShards) { + return; + } + + if (Parallel::isWorker()) { + $this->writeWorkerTimings(); + + return; + } + + if (! self::$passed) { + return; + } + + $timings = self::$collectedTimings ?? $this->collectTimings(); + + if ($timings === []) { + return; + } + + $this->writeTimings($timings); + } + + /** + * Collects timings from subscribers or worker temp files. + * + * @return array + */ + private function collectTimings(): array + { + $runId = Parallel::getGlobal('SHARD_RUN_ID'); + + if (is_string($runId)) { + return $this->readWorkerTimings($runId); + } + + return EnsureShardTimingsAreCollected::timings(); + } + + /** + * Writes the current worker's timing data to a temp file. + */ + private function writeWorkerTimings(): void + { + $timings = EnsureShardTimingsAreCollected::timings(); + + if ($timings === []) { + return; + } + + $runId = Parallel::getGlobal('SHARD_RUN_ID'); + + if (! is_string($runId)) { + return; + } + + $path = sys_get_temp_dir().DIRECTORY_SEPARATOR.'__pest_sharding_'.$runId.'-'.getmypid().'.json'; + + file_put_contents($path, json_encode($timings, JSON_THROW_ON_ERROR)); + } + + /** + * Reads and merges timing data from all worker temp files. + * + * @return array + */ + private function readWorkerTimings(string $runId): array + { + $pattern = sys_get_temp_dir().DIRECTORY_SEPARATOR.'__pest_sharding_'.$runId.'-*.json'; + $files = glob($pattern); + + if ($files === false || $files === []) { + return []; + } + + $merged = []; + + foreach ($files as $file) { + $contents = file_get_contents($file); + + if ($contents === false) { + continue; + } + + $timings = json_decode($contents, true); + + if (is_array($timings)) { + $merged = array_merge($merged, $timings); + } + + unlink($file); + } + + return $merged; + } + + /** + * Returns the path to shards.json. + */ + private function shardsPath(): string + { + $testSuite = TestSuite::getInstance(); + + return implode(DIRECTORY_SEPARATOR, [$testSuite->rootPath, $testSuite->testPath, '.pest', 'shards.json']); + } + + /** + * Loads the timings from shards.json. + * + * @return array|null + */ + private function loadShardsFile(): ?array + { + $path = $this->shardsPath(); + + if (! file_exists($path)) { + return null; + } + + $contents = file_get_contents($path); + + if ($contents === false) { + throw new InvalidOption('The [tests/.pest/shards.json] file could not be read. Delete it or run [--update-shards] to regenerate.'); + } + + $data = json_decode($contents, true); + + if (! is_array($data) || ! isset($data['timings']) || ! is_array($data['timings'])) { + throw new InvalidOption('The [tests/.pest/shards.json] file is corrupted. Delete it or run [--update-shards] to regenerate.'); + } + + return $data['timings']; + } + + /** + * Partitions tests across shards using the LPT (Longest Processing Time) algorithm. + * + * @param list $tests + * @param array $timings + * @return list> + */ + private function partitionByTime(array $tests, array $timings, int $total): array + { + $knownTimings = array_filter( + array_map(fn (string $test): ?float => $timings[$test] ?? null, $tests), + fn (?float $t): bool => $t !== null, + ); + + $median = $knownTimings !== [] ? $this->median(array_values($knownTimings)) : 1.0; + + $testsWithTimings = array_map( + fn (string $test): array => ['test' => $test, 'time' => $timings[$test] ?? $median], + $tests, + ); + + usort($testsWithTimings, fn (array $a, array $b): int => $b['time'] <=> $a['time']); + + /** @var list> */ + $bins = array_fill(0, $total, []); + /** @var non-empty-list */ + $binTimes = array_fill(0, $total, 0.0); + + foreach ($testsWithTimings as $item) { + $minIndex = array_search(min($binTimes), $binTimes, strict: true); + assert(is_int($minIndex)); + + $bins[$minIndex][] = $item['test']; + $binTimes[$minIndex] += $item['time']; + } + + return $bins; + } + + /** + * Calculates the median of an array of floats. + * + * @param list $values + */ + private function median(array $values): float + { + sort($values); + + $count = count($values); + $middle = (int) floor($count / 2); + + if ($count % 2 === 0) { + return ($values[$middle - 1] + $values[$middle]) / 2; + } + + return $values[$middle]; + } + + /** + * Writes the timings to shards.json. + * + * @param array $timings + */ + private function writeTimings(array $timings): void + { + $path = $this->shardsPath(); + + $directory = dirname($path); + if (! is_dir($directory)) { + mkdir($directory, 0755, true); + } + + if (self::$knownTests !== null) { + $knownSet = array_flip(self::$knownTests); + $timings = array_intersect_key($timings, $knownSet); + } + + ksort($timings); + + $canonical = self::$knownTests ?? array_keys($timings); + sort($canonical); + + file_put_contents($path, json_encode([ + 'timings' => $timings, + 'checksum' => md5(implode("\n", $canonical)), + 'updated_at' => date('c'), + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)."\n"); + } + /** * Returns the shard information. * diff --git a/src/Subscribers/EnsureShardTimingFinished.php b/src/Subscribers/EnsureShardTimingFinished.php new file mode 100644 index 00000000..f1732d9b --- /dev/null +++ b/src/Subscribers/EnsureShardTimingFinished.php @@ -0,0 +1,22 @@ + + */ + private static array $startTimes = []; + + /** + * The collected timings for each test class. + * + * @var array + */ + private static array $timings = []; + + /** + * Records the start time for a test suite. + */ + public static function started(Started $event): void + { + if (! $event->testSuite()->isForTestClass()) { + return; + } + + $name = preg_replace('/^P\\\\/', '', $event->testSuite()->name()); + + if (is_string($name)) { + self::$startTimes[$name] = $event->telemetryInfo()->time(); + } + } + + /** + * Records the duration for a test suite. + */ + public static function finished(Finished $event): void + { + if (! $event->testSuite()->isForTestClass()) { + return; + } + + $name = preg_replace('/^P\\\\/', '', $event->testSuite()->name()); + + if (! is_string($name) || ! isset(self::$startTimes[$name])) { + return; + } + + $duration = $event->telemetryInfo()->time()->duration(self::$startTimes[$name]); + + self::$timings[$name] = round($duration->asFloat(), 4); + } + + /** + * Returns the collected timings. + * + * @return array + */ + public static function timings(): array + { + return self::$timings; + } +} diff --git a/tests/.pest/snapshots/Visual/Help/visual_snapshot_of_help_command_output.snap b/tests/.pest/snapshots/Visual/Help/visual_snapshot_of_help_command_output.snap index f8eee485..5a7f0c87 100644 --- a/tests/.pest/snapshots/Visual/Help/visual_snapshot_of_help_command_output.snap +++ b/tests/.pest/snapshots/Visual/Help/visual_snapshot_of_help_command_output.snap @@ -49,6 +49,7 @@ EXECUTION OPTIONS: --parallel ........................................... Run tests in parallel --update-snapshots Update snapshots for tests using the "toMatchSnapshot" expectation + --update-shards Update shards.json with test timing data for time-balanced sharding --globals-backup ................. Backup and restore $GLOBALS for each test --static-backup ......... Backup and restore static properties for each test --strict-coverage ................... Be strict about code coverage metadata diff --git a/tests/Arch.php b/tests/Arch.php index 6348a0f3..d0565216 100644 --- a/tests/Arch.php +++ b/tests/Arch.php @@ -17,7 +17,9 @@ arch()->preset()->security()->ignoring([ 'eval', 'str_shuffle', 'exec', + 'md5', 'unserialize', + 'uniqid', 'extract', 'assert', ]);