diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml index bea844f1..53df58bf 100644 --- a/.github/workflows/static.yml +++ b/.github/workflows/static.yml @@ -11,6 +11,9 @@ concurrency: group: static-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: static: if: github.event_name != 'schedule' || github.repository == 'pestphp/pest' @@ -44,7 +47,7 @@ jobs: uses: actions/cache@v5 with: path: ${{ steps.composer-cache.outputs.dir }} - key: static-php-8.4-${{ matrix.dependency-version }}-composer-${{ hashFiles('**/composer.json') }} + key: static-php-8.4-${{ matrix.dependency-version }}-composer-${{ hashFiles('**/composer.json', '**/composer.lock') }} restore-keys: | static-php-8.4-${{ matrix.dependency-version }}-composer- static-php-8.4-composer- diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 42a0cce9..c63a0d2e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,9 @@ concurrency: group: tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: tests: if: github.event_name != 'schedule' || github.repository == 'pestphp/pest' @@ -48,7 +51,7 @@ jobs: uses: actions/cache@v5 with: path: ${{ steps.composer-cache.outputs.dir }} - key: ${{ matrix.os }}-php-${{ matrix.php }}-symfony-${{ matrix.symfony }}-composer-${{ hashFiles('**/composer.json') }} + key: ${{ matrix.os }}-php-${{ matrix.php }}-symfony-${{ matrix.symfony }}-composer-${{ hashFiles('**/composer.json', '**/composer.lock') }} restore-keys: | ${{ matrix.os }}-php-${{ matrix.php }}-symfony-${{ matrix.symfony }}-composer- ${{ matrix.os }}-php-${{ matrix.php }}-composer- diff --git a/composer.json b/composer.json index 869fac6d..cce82efb 100644 --- a/composer.json +++ b/composer.json @@ -18,19 +18,19 @@ ], "require": { "php": "^8.4", - "brianium/paratest": "^7.22.1", + "brianium/paratest": "^7.22.3", "nunomaduro/collision": "^8.9.3", "nunomaduro/termwind": "^2.4.0", "pestphp/pest-plugin": "^5.0.0", "pestphp/pest-plugin-arch": "^5.0.0", "pestphp/pest-plugin-mutate": "^5.0.0", "pestphp/pest-plugin-profanity": "^5.0.0", - "phpunit/phpunit": "^13.1.0", + "phpunit/phpunit": "^13.1.6", "symfony/process": "^8.1.0" }, "conflict": { "filp/whoops": "<2.18.3", - "phpunit/phpunit": ">13.1.0", + "phpunit/phpunit": ">13.1.6", "sebastian/exporter": "<7.0.0", "webmozart/assert": "<1.11.0" }, diff --git a/overrides/Logging/JUnit/JunitXmlLogger.php b/overrides/Logging/JUnit/JunitXmlLogger.php index 1fc237a6..4636d9fe 100644 --- a/overrides/Logging/JUnit/JunitXmlLogger.php +++ b/overrides/Logging/JUnit/JunitXmlLogger.php @@ -1,6 +1,39 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace PHPUnit\Runner; + +use PHPUnit\Framework\DataProviderTestSuite; +use PHPUnit\Framework\Reorderable; +use PHPUnit\Framework\Test; +use PHPUnit\Framework\TestCase; +use PHPUnit\Framework\TestSuite; +use PHPUnit\Runner\ResultCache\NullResultCache; +use PHPUnit\Runner\ResultCache\ResultCache; +use PHPUnit\Runner\ResultCache\ResultCacheId; + +use function array_diff; +use function array_merge; +use function array_reverse; +use function array_splice; +use function assert; +use function count; +use function in_array; +use function max; +use function shuffle; +use function usort; + +/** + * @internal This class is not covered by the backward compatibility promise for PHPUnit + */ +final class TestSuiteSorter +{ + public const int ORDER_DEFAULT = 0; + + public const int ORDER_RANDOMIZED = 1; + + public const int ORDER_REVERSED = 2; + + public const int ORDER_DEFECTS_FIRST = 3; + + public const int ORDER_DURATION = 4; + + public const int ORDER_SIZE = 5; + + /** + * @var non-empty-array + */ + private const array SIZE_SORT_WEIGHT = [ + 'small' => 1, + 'medium' => 2, + 'large' => 3, + 'unknown' => 4, + ]; + + /** + * @var array Associative array of (string => DEFECT_SORT_WEIGHT) elements + */ + private array $defectSortOrder = []; + + private readonly ResultCache $cache; + + public function __construct(?ResultCache $cache = null) + { + $this->cache = $cache ?? new NullResultCache; + } + + /** + * @throws Exception + */ + public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDependencies, int $orderDefects): void + { + $allowedOrders = [ + self::ORDER_DEFAULT, + self::ORDER_REVERSED, + self::ORDER_RANDOMIZED, + self::ORDER_DURATION, + self::ORDER_SIZE, + ]; + + if (! in_array($order, $allowedOrders, true)) { + // @codeCoverageIgnoreStart + throw new InvalidOrderException; + // @codeCoverageIgnoreEnd + } + + $allowedOrderDefects = [ + self::ORDER_DEFAULT, + self::ORDER_DEFECTS_FIRST, + ]; + + if (! in_array($orderDefects, $allowedOrderDefects, true)) { + // @codeCoverageIgnoreStart + throw new InvalidOrderException; + // @codeCoverageIgnoreEnd + } + + if ($suite instanceof TestSuite) { + foreach ($suite as $_suite) { + $this->reorderTestsInSuite($_suite, $order, $resolveDependencies, $orderDefects); + } + + if ($orderDefects === self::ORDER_DEFECTS_FIRST) { + $this->addSuiteToDefectSortOrder($suite); + } + + $this->sort($suite, $order, $resolveDependencies, $orderDefects); + } + } + + private function sort(TestSuite $suite, int $order, bool $resolveDependencies, int $orderDefects): void + { + if ($suite->tests() === []) { + return; + } + + if ($order === self::ORDER_REVERSED) { + $suite->setTests($this->reverse($suite->tests())); + } elseif ($order === self::ORDER_RANDOMIZED) { + $suite->setTests($this->randomize($suite->tests())); + } elseif ($order === self::ORDER_DURATION) { + $suite->setTests($this->sortByDuration($suite->tests())); + } elseif ($order === self::ORDER_SIZE) { + $suite->setTests($this->sortBySize($suite->tests())); + } + + if ($orderDefects === self::ORDER_DEFECTS_FIRST) { + $suite->setTests($this->sortDefectsFirst($suite->tests())); + } + + if ($resolveDependencies && ! ($suite instanceof DataProviderTestSuite)) { + $tests = $suite->tests(); + + /** @noinspection PhpParamsInspection */ + /** @phpstan-ignore argument.type */ + $suite->setTests($this->resolveDependencies($tests)); + } + } + + private function addSuiteToDefectSortOrder(TestSuite $suite): void + { + $max = 0; + + foreach ($suite->tests() as $test) { + assert($test instanceof Reorderable); + + $sortId = $test->sortId(); + + if (! isset($this->defectSortOrder[$sortId])) { + $this->defectSortOrder[$sortId] = $this->cache->status(ResultCacheId::fromReorderable($test))->asInt(); + $max = max($max, $this->defectSortOrder[$sortId]); + } + } + + $this->defectSortOrder[$suite->sortId()] = $max; + } + + /** + * @param list $tests + * @return list + */ + private function reverse(array $tests): array + { + return array_reverse($tests); + } + + /** + * @param list $tests + * @return list + */ + private function randomize(array $tests): array + { + shuffle($tests); + + return $tests; + } + + /** + * @param list $tests + * @return list + */ + private function sortDefectsFirst(array $tests): array + { + usort( + $tests, + fn (Test $left, Test $right) => $this->cmpDefectPriorityAndTime($left, $right), + ); + + return $tests; + } + + /** + * @param list $tests + * @return list + */ + private function sortByDuration(array $tests): array + { + usort( + $tests, + fn (Test $left, Test $right) => $this->cmpDuration($left, $right), + ); + + return $tests; + } + + /** + * @param list $tests + * @return list + */ + private function sortBySize(array $tests): array + { + usort( + $tests, + fn (Test $left, Test $right) => $this->cmpSize($left, $right), + ); + + return $tests; + } + + /** + * Comparator callback function to sort tests for "reach failure as fast as possible". + * + * 1. sort tests by defect weight defined in self::DEFECT_SORT_WEIGHT + * 2. when tests are equally defective, sort the fastest to the front + * 3. do not reorder successful tests + */ + private function cmpDefectPriorityAndTime(Test $a, Test $b): int + { + assert($a instanceof Reorderable); + assert($b instanceof Reorderable); + + $priorityA = $this->defectSortOrder[$a->sortId()] ?? 0; + $priorityB = $this->defectSortOrder[$b->sortId()] ?? 0; + + if ($priorityA !== $priorityB) { + // Sort defect weight descending + return $priorityB <=> $priorityA; + } + + if ($priorityA > 0 || $priorityB > 0) { + return $this->cmpDuration($a, $b); + } + + // do not change execution order + return 0; + } + + /** + * Compares test duration for sorting tests by duration ascending. + */ + private function cmpDuration(Test $a, Test $b): int + { + if (! ($a instanceof Reorderable && $b instanceof Reorderable)) { + return 0; + } + + return $this->cache->time(ResultCacheId::fromReorderable($a)) <=> $this->cache->time(ResultCacheId::fromReorderable($b)); + } + + /** + * Compares test size for sorting tests small->medium->large->unknown. + */ + private function cmpSize(Test $a, Test $b): int + { + $sizeA = ($a instanceof TestCase || $a instanceof DataProviderTestSuite) + ? $a->size()->asString() + : 'unknown'; + $sizeB = ($b instanceof TestCase || $b instanceof DataProviderTestSuite) + ? $b->size()->asString() + : 'unknown'; + + return self::SIZE_SORT_WEIGHT[$sizeA] <=> self::SIZE_SORT_WEIGHT[$sizeB]; + } + + /** + * Reorder Tests within a TestCase in such a way as to resolve as many dependencies as possible. + * The algorithm will leave the tests in original running order when it can. + * For more details see the documentation for test dependencies. + * + * Short description of algorithm: + * 1. Pick the next Test from remaining tests to be checked for dependencies. + * 2. If the test has no dependencies: mark done, start again from the top + * 3. If the test has dependencies but none left to do: mark done, start again from the top + * 4. When we reach the end add any leftover tests to the end. These will be marked 'skipped' during execution. + * + * @param array $tests + * @return array + */ + private function resolveDependencies(array $tests): array + { + // Pest: Fast-path. If no test in this suite declares dependencies, the + // original O(N^2) algorithm is wasted work — it would splice each test + // one-by-one back into the same order. The check deliberately walks + // TestCase instances directly instead of calling TestSuite::requires(), + // because the latter lazily builds TestSuite::provides() via + // ExecutionOrderDependency::mergeUnique, which is O(N^2) in the total + // number of tests. With thousands of tests that single call alone can + // burn several seconds before the sort even begins. Reading the + // cached TestCase::$dependencies property stays O(N) and costs nothing + // when no test uses `->depends()` / PHPUnit `@depends`. + if (! $this->anyTestHasDependencies($tests)) { + return $tests; + } + + $newTestOrder = []; + $i = 0; + $provided = []; + + do { + if (array_diff($tests[$i]->requires(), $provided) === []) { + $provided = array_merge($provided, $tests[$i]->provides()); + $newTestOrder = array_merge($newTestOrder, array_splice($tests, $i, 1)); + $i = 0; + } else { + $i++; + } + } while ($tests !== [] && ($i < count($tests))); + + return array_merge($newTestOrder, $tests); + } + + /** + * Cheaply determines whether any test in the tree declares @depends. + * + * Walks `TestSuite` containers recursively and inspects each `TestCase` + * directly so it never triggers `TestSuite::provides()`, which is O(N^2) + * in the total number of aggregated tests. + * + * @param iterable $tests + */ + private function anyTestHasDependencies(iterable $tests): bool + { + foreach ($tests as $test) { + if ($test instanceof TestSuite) { + if ($this->anyTestHasDependencies($test->tests())) { + return true; + } + + continue; + } + + if ($test instanceof TestCase && $test->requires() !== []) { + return true; + } + } + + return false; + } +} diff --git a/src/ArchPresets/Laravel.php b/src/ArchPresets/Laravel.php index 406611a3..3b727340 100644 --- a/src/ArchPresets/Laravel.php +++ b/src/ArchPresets/Laravel.php @@ -176,9 +176,5 @@ final class Laravel extends AbstractPreset ->toImplement('Illuminate\Contracts\Container\ContextualAttribute') ->toHaveAttribute('Attribute') ->toHaveMethod('resolve'); - - $this->expectations[] = expect('App\Rules') - ->classes() - ->toImplement('Illuminate\Contracts\Validation\ValidationRule'); } } diff --git a/src/Bootstrappers/BootOverrides.php b/src/Bootstrappers/BootOverrides.php index 28851f3a..4b67bf0a 100644 --- a/src/Bootstrappers/BootOverrides.php +++ b/src/Bootstrappers/BootOverrides.php @@ -21,6 +21,7 @@ final class BootOverrides implements Bootstrapper 'Runner/Filter/NameFilterIterator.php', 'Runner/ResultCache/DefaultResultCache.php', 'Runner/TestSuiteLoader.php', + 'Runner/TestSuiteSorter.php', 'TextUI/Command/Commands/WarmCodeCoverageCacheCommand.php', 'TextUI/Output/Default/ProgressPrinter/Subscriber/TestSkippedSubscriber.php', 'TextUI/TestSuiteFilterProcessor.php', diff --git a/src/Mixins/Expectation.php b/src/Mixins/Expectation.php index c4b7ae9b..0ac44fb1 100644 --- a/src/Mixins/Expectation.php +++ b/src/Mixins/Expectation.php @@ -14,6 +14,7 @@ use InvalidArgumentException; use JsonSerializable; use Pest\Exceptions\InvalidExpectationValue; use Pest\Matchers\Any; +use Pest\Plugins\Snapshot; use Pest\Support\Arr; use Pest\Support\Exporter; use Pest\Support\NullClosure; @@ -851,18 +852,31 @@ final class Expectation default => InvalidExpectationValue::expected('array|object|string'), }; - if ($snapshots->has()) { - [$filename, $content] = $snapshots->get(); - - Assert::assertSame( - strtr($content, ["\r\n" => "\n", "\r" => "\n"]), - strtr($string, ["\r\n" => "\n", "\r" => "\n"]), - $message === '' ? "Failed asserting that the string value matches its snapshot ($filename)." : $message - ); - } else { + if (! $snapshots->has()) { $filename = $snapshots->save($string); TestSuite::getInstance()->registerSnapshotChange("Snapshot created at [$filename]"); + } else { + [$filename, $content] = $snapshots->get(); + + $normalizedContent = strtr($content, ["\r\n" => "\n", "\r" => "\n"]); + $normalizedString = strtr($string, ["\r\n" => "\n", "\r" => "\n"]); + + if (Snapshot::$updateSnapshots && $normalizedContent !== $normalizedString) { + $snapshots->save($string); + + TestSuite::getInstance()->registerSnapshotChange("Snapshot updated at [$filename]"); + } else { + if (Snapshot::$updateSnapshots) { + TestSuite::getInstance()->registerSnapshotChange("Snapshot unchanged at [$filename]"); + } + + Assert::assertSame( + $normalizedContent, + $normalizedString, + $message === '' ? "Failed asserting that the string value matches its snapshot ($filename)." : $message + ); + } } return $this; diff --git a/src/Pest.php b/src/Pest.php index 2b65bc79..0bf9fb06 100644 --- a/src/Pest.php +++ b/src/Pest.php @@ -6,7 +6,7 @@ namespace Pest; function version(): string { - return '5.0.0-rc.3'; + return '5.0.0-rc.4'; } function testDirectory(string $file = ''): string diff --git a/src/Plugins/Help.php b/src/Plugins/Help.php index 0795c806..12d03532 100644 --- a/src/Plugins/Help.php +++ b/src/Plugins/Help.php @@ -123,6 +123,10 @@ final readonly class Help implements HandlesArguments 'arg' => '--update-snapshots', 'desc' => 'Update snapshots for tests using the "toMatchSnapshot" expectation', ], + [ + 'arg' => '--update-shards', + 'desc' => 'Update shards.json with test timing data for time-balanced sharding', + ], ], ...$content['Execution']]; $content['Selection'] = [[ diff --git a/src/Plugins/Shard.php b/src/Plugins/Shard.php index afb2e764..ebbc8097 100644 --- a/src/Plugins/Shard.php +++ b/src/Plugins/Shard.php @@ -6,7 +6,13 @@ namespace Pest\Plugins; use Pest\Contracts\Plugins\AddsOutput; use Pest\Contracts\Plugins\HandlesArguments; +use Pest\Contracts\Plugins\Terminable; use Pest\Exceptions\InvalidOption; +use Pest\Subscribers\EnsureShardTimingFinished; +use Pest\Subscribers\EnsureShardTimingsAreCollected; +use Pest\Subscribers\EnsureShardTimingStarted; +use Pest\TestSuite; +use PHPUnit\Event; use Symfony\Component\Console\Input\ArgvInput; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -15,7 +21,7 @@ use Symfony\Component\Process\Process; /** * @internal */ -final class Shard implements AddsOutput, HandlesArguments +final class Shard implements AddsOutput, HandlesArguments, Terminable { use Concerns\HandleArguments; @@ -33,6 +39,40 @@ final class Shard implements AddsOutput, HandlesArguments */ private static ?array $shard = null; + /** + * Whether to update the shards.json file. + */ + private static bool $updateShards = false; + + /** + * Whether time-balanced sharding was used. + */ + private static bool $timeBalanced = false; + + /** + * Whether the shards.json file is outdated. + */ + private static bool $shardsOutdated = false; + + /** + * Whether the test suite passed. + */ + private static bool $passed = false; + + /** + * Collected timings from workers or subscribers. + * + * @var array|null + */ + private static ?array $collectedTimings = null; + + /** + * The canonical list of test classes from --list-tests. + * + * @var list|null + */ + private static ?array $knownTests = null; + /** * Creates a new Plugin instance. */ @@ -47,6 +87,19 @@ final class Shard implements AddsOutput, HandlesArguments */ public function handleArguments(array $arguments): array { + if ($this->hasArgument('--update-shards', $arguments)) { + return $this->handleUpdateShards($arguments); + } + + if (Parallel::isWorker() && Parallel::getGlobal('UPDATE_SHARDS') === true) { + self::$updateShards = true; + + Event\Facade::instance()->registerSubscriber(new EnsureShardTimingStarted); + Event\Facade::instance()->registerSubscriber(new EnsureShardTimingFinished); + + return $arguments; + } + if (! $this->hasArgument('--shard', $arguments)) { return $arguments; } @@ -63,7 +116,24 @@ final class Shard implements AddsOutput, HandlesArguments /** @phpstan-ignore-next-line */ $tests = $this->allTests($arguments); - $testsToRun = (array_chunk($tests, max(1, (int) ceil(count($tests) / $total))))[$index - 1] ?? []; + + $timings = $this->loadShardsFile(); + if ($timings !== null) { + $knownTests = array_values(array_filter($tests, fn (string $test): bool => isset($timings[$test]))); + $newTests = array_values(array_diff($tests, $knownTests)); + + $partitions = $this->partitionByTime($knownTests, $timings, $total); + + foreach ($newTests as $i => $test) { + $partitions[$i % $total][] = $test; + } + + $testsToRun = $partitions[$index - 1] ?? []; + self::$timeBalanced = true; + self::$shardsOutdated = $newTests !== []; + } else { + $testsToRun = (array_chunk($tests, max(1, (int) ceil(count($tests) / $total))))[$index - 1] ?? []; + } self::$shard = [ 'index' => $index, @@ -72,9 +142,43 @@ final class Shard implements AddsOutput, HandlesArguments 'testsCount' => count($tests), ]; + if ($testsToRun === []) { + return $arguments; + } + return [...$arguments, '--filter', $this->buildFilterArgument($testsToRun)]; } + /** + * Handles the --update-shards argument. + * + * @param array $arguments + * @return array + */ + private function handleUpdateShards(array $arguments): array + { + if ($this->hasArgument('--shard', $arguments)) { + throw new InvalidOption('The [--update-shards] option cannot be combined with [--shard].'); + } + + $arguments = $this->popArgument('--update-shards', $arguments); + + self::$updateShards = true; + + /** @phpstan-ignore-next-line */ + self::$knownTests = $this->allTests($arguments); + + if ($this->hasArgument('--parallel', $arguments) || $this->hasArgument('-p', $arguments)) { + Parallel::setGlobal('UPDATE_SHARDS', true); + Parallel::setGlobal('SHARD_RUN_ID', uniqid('pest-shard-', true)); + } else { + Event\Facade::instance()->registerSubscriber(new EnsureShardTimingStarted); + Event\Facade::instance()->registerSubscriber(new EnsureShardTimingFinished); + } + + return $arguments; + } + /** * Returns all tests that the test suite would run. * @@ -87,7 +191,7 @@ final class Shard implements AddsOutput, HandlesArguments 'php', ...$this->removeParallelArguments($arguments), '--list-tests', - ])->mustRun()->getOutput(); + ])->setTimeout(120)->mustRun()->getOutput(); preg_match_all('/ - (?:P\\\\)?(Tests\\\\[^:]+)::/', $output, $matches); @@ -116,6 +220,22 @@ final class Shard implements AddsOutput, HandlesArguments */ public function addOutput(int $exitCode): int { + self::$passed = $exitCode === 0; + + if (self::$updateShards && self::$passed && ! Parallel::isWorker()) { + self::$collectedTimings = $this->collectTimings(); + + $count = self::$knownTests !== null + ? count(array_intersect_key(self::$collectedTimings, array_flip(self::$knownTests))) + : count(self::$collectedTimings); + + $this->output->writeln(sprintf( + ' Shards: shards.json updated with timings for %d test class%s.', + $count, + $count === 1 ? '' : 'es', + )); + } + if (self::$shard === null) { return $exitCode; } @@ -128,17 +248,250 @@ final class Shard implements AddsOutput, HandlesArguments ] = self::$shard; $this->output->writeln(sprintf( - ' Shard: %d of %d — %d file%s ran, out of %d.', + ' Shard: %d of %d — %d file%s ran, out of %d%s.', $index, $total, $testsRan, $testsRan === 1 ? '' : 's', $testsCount, + self::$timeBalanced ? ' (time-balanced)' : '', )); + if (self::$shardsOutdated) { + $this->output->writeln(' WARN The [tests/.pest/shards.json] file is out of date. Run [--update-shards] to update it.'); + } + return $exitCode; } + /** + * Terminates the plugin. + */ + public function terminate(): void + { + if (! self::$updateShards) { + return; + } + + if (Parallel::isWorker()) { + $this->writeWorkerTimings(); + + return; + } + + if (! self::$passed) { + return; + } + + $timings = self::$collectedTimings ?? $this->collectTimings(); + + if ($timings === []) { + return; + } + + $this->writeTimings($timings); + } + + /** + * Collects timings from subscribers or worker temp files. + * + * @return array + */ + private function collectTimings(): array + { + $runId = Parallel::getGlobal('SHARD_RUN_ID'); + + if (is_string($runId)) { + return $this->readWorkerTimings($runId); + } + + return EnsureShardTimingsAreCollected::timings(); + } + + /** + * Writes the current worker's timing data to a temp file. + */ + private function writeWorkerTimings(): void + { + $timings = EnsureShardTimingsAreCollected::timings(); + + if ($timings === []) { + return; + } + + $runId = Parallel::getGlobal('SHARD_RUN_ID'); + + if (! is_string($runId)) { + return; + } + + $path = sys_get_temp_dir().DIRECTORY_SEPARATOR.'__pest_sharding_'.$runId.'-'.getmypid().'.json'; + + file_put_contents($path, json_encode($timings, JSON_THROW_ON_ERROR)); + } + + /** + * Reads and merges timing data from all worker temp files. + * + * @return array + */ + private function readWorkerTimings(string $runId): array + { + $pattern = sys_get_temp_dir().DIRECTORY_SEPARATOR.'__pest_sharding_'.$runId.'-*.json'; + $files = glob($pattern); + + if ($files === false || $files === []) { + return []; + } + + $merged = []; + + foreach ($files as $file) { + $contents = file_get_contents($file); + + if ($contents === false) { + continue; + } + + $timings = json_decode($contents, true); + + if (is_array($timings)) { + $merged = array_merge($merged, $timings); + } + + unlink($file); + } + + return $merged; + } + + /** + * Returns the path to shards.json. + */ + private function shardsPath(): string + { + $testSuite = TestSuite::getInstance(); + + return implode(DIRECTORY_SEPARATOR, [$testSuite->rootPath, $testSuite->testPath, '.pest', 'shards.json']); + } + + /** + * Loads the timings from shards.json. + * + * @return array|null + */ + private function loadShardsFile(): ?array + { + $path = $this->shardsPath(); + + if (! file_exists($path)) { + return null; + } + + $contents = file_get_contents($path); + + if ($contents === false) { + throw new InvalidOption('The [tests/.pest/shards.json] file could not be read. Delete it or run [--update-shards] to regenerate.'); + } + + $data = json_decode($contents, true); + + if (! is_array($data) || ! isset($data['timings']) || ! is_array($data['timings'])) { + throw new InvalidOption('The [tests/.pest/shards.json] file is corrupted. Delete it or run [--update-shards] to regenerate.'); + } + + return $data['timings']; + } + + /** + * Partitions tests across shards using the LPT (Longest Processing Time) algorithm. + * + * @param list $tests + * @param array $timings + * @return list> + */ + private function partitionByTime(array $tests, array $timings, int $total): array + { + $knownTimings = array_filter( + array_map(fn (string $test): ?float => $timings[$test] ?? null, $tests), + fn (?float $t): bool => $t !== null, + ); + + $median = $knownTimings !== [] ? $this->median(array_values($knownTimings)) : 1.0; + + $testsWithTimings = array_map( + fn (string $test): array => ['test' => $test, 'time' => $timings[$test] ?? $median], + $tests, + ); + + usort($testsWithTimings, fn (array $a, array $b): int => $b['time'] <=> $a['time']); + + /** @var list> */ + $bins = array_fill(0, $total, []); + /** @var non-empty-list */ + $binTimes = array_fill(0, $total, 0.0); + + foreach ($testsWithTimings as $item) { + $minIndex = array_search(min($binTimes), $binTimes, strict: true); + assert(is_int($minIndex)); + + $bins[$minIndex][] = $item['test']; + $binTimes[$minIndex] += $item['time']; + } + + return $bins; + } + + /** + * Calculates the median of an array of floats. + * + * @param list $values + */ + private function median(array $values): float + { + sort($values); + + $count = count($values); + $middle = (int) floor($count / 2); + + if ($count % 2 === 0) { + return ($values[$middle - 1] + $values[$middle]) / 2; + } + + return $values[$middle]; + } + + /** + * Writes the timings to shards.json. + * + * @param array $timings + */ + private function writeTimings(array $timings): void + { + $path = $this->shardsPath(); + + $directory = dirname($path); + if (! is_dir($directory)) { + mkdir($directory, 0755, true); + } + + if (self::$knownTests !== null) { + $knownSet = array_flip(self::$knownTests); + $timings = array_intersect_key($timings, $knownSet); + } + + ksort($timings); + + $canonical = self::$knownTests ?? array_keys($timings); + sort($canonical); + + file_put_contents($path, json_encode([ + 'timings' => $timings, + 'checksum' => md5(implode("\n", $canonical)), + 'updated_at' => date('c'), + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)."\n"); + } + /** * Returns the shard information. * diff --git a/src/Plugins/Snapshot.php b/src/Plugins/Snapshot.php index 717512c2..e0ac0505 100644 --- a/src/Plugins/Snapshot.php +++ b/src/Plugins/Snapshot.php @@ -5,7 +5,6 @@ declare(strict_types=1); namespace Pest\Plugins; use Pest\Contracts\Plugins\HandlesArguments; -use Pest\Exceptions\InvalidOption; use Pest\TestSuite; /** @@ -15,21 +14,116 @@ final class Snapshot implements HandlesArguments { use Concerns\HandleArguments; + /** + * Whether snapshots should be updated on this run. + */ + public static bool $updateSnapshots = false; + /** * {@inheritDoc} */ public function handleArguments(array $arguments): array { + if (Parallel::isWorker() && Parallel::getGlobal('UPDATE_SNAPSHOTS') === true) { + self::$updateSnapshots = true; + + return $arguments; + } + if (! $this->hasArgument('--update-snapshots', $arguments)) { return $arguments; } - if ($this->hasArgument('--parallel', $arguments)) { - throw new InvalidOption('The [--update-snapshots] option is not supported when running in parallel.'); + self::$updateSnapshots = true; + + if ($this->isFullRun($arguments)) { + TestSuite::getInstance()->snapshots->flush(); } - TestSuite::getInstance()->snapshots->flush(); + if ($this->hasArgument('--parallel', $arguments) || $this->hasArgument('-p', $arguments)) { + Parallel::setGlobal('UPDATE_SNAPSHOTS', true); + } return $this->popArgument('--update-snapshots', $arguments); } + + /** + * Options that take a value as the next argument (rather than via "=value"). + * + * @var list + */ + private const array FLAGS_WITH_VALUES = [ + '--filter', + '--group', + '--exclude-group', + '--test-suffix', + '--covers', + '--uses', + '--cache-directory', + '--cache-result-file', + '--configuration', + '--colors', + '--test-directory', + '--bootstrap', + '--order-by', + '--random-order-seed', + '--log-junit', + '--log-teamcity', + '--log-events-text', + '--log-events-verbose-text', + '--coverage-clover', + '--coverage-cobertura', + '--coverage-crap4j', + '--coverage-html', + '--coverage-php', + '--coverage-text', + '--coverage-xml', + '--assignee', + '--issue', + '--ticket', + '--pr', + '--pull-request', + '--retry', + '--shard', + '--repeat', + ]; + + /** + * Determines whether the command targets the entire suite (no filter, no path). + * + * @param array $arguments + */ + private function isFullRun(array $arguments): bool + { + if ($this->hasArgument('--filter', $arguments)) { + return false; + } + + $tokens = array_slice($arguments, 1); + $skipNext = false; + + foreach ($tokens as $arg) { + if ($skipNext) { + $skipNext = false; + + continue; + } + + if ($arg === '') { + continue; + } + + if ($arg[0] === '-') { + if (in_array($arg, self::FLAGS_WITH_VALUES, true)) { + $skipNext = true; + } + + continue; + } + + return false; + } + + return true; + } } diff --git a/src/Repositories/SnapshotRepository.php b/src/Repositories/SnapshotRepository.php index c719f219..30b31546 100644 --- a/src/Repositories/SnapshotRepository.php +++ b/src/Repositories/SnapshotRepository.php @@ -59,8 +59,10 @@ final class SnapshotRepository { $snapshotFilename = $this->getSnapshotFilename(); - if (! file_exists(dirname($snapshotFilename))) { - mkdir(dirname($snapshotFilename), 0755, true); + $directory = dirname($snapshotFilename); + + if (! is_dir($directory)) { + @mkdir($directory, 0755, true); } file_put_contents($snapshotFilename, $snapshot); diff --git a/src/Subscribers/EnsureShardTimingFinished.php b/src/Subscribers/EnsureShardTimingFinished.php new file mode 100644 index 00000000..f1732d9b --- /dev/null +++ b/src/Subscribers/EnsureShardTimingFinished.php @@ -0,0 +1,22 @@ + + */ + private static array $startTimes = []; + + /** + * The collected timings for each test class. + * + * @var array + */ + private static array $timings = []; + + /** + * Records the start time for a test suite. + */ + public static function started(Started $event): void + { + if (! $event->testSuite()->isForTestClass()) { + return; + } + + $name = preg_replace('/^P\\\\/', '', $event->testSuite()->name()); + + if (is_string($name)) { + self::$startTimes[$name] = $event->telemetryInfo()->time(); + } + } + + /** + * Records the duration for a test suite. + */ + public static function finished(Finished $event): void + { + if (! $event->testSuite()->isForTestClass()) { + return; + } + + $name = preg_replace('/^P\\\\/', '', $event->testSuite()->name()); + + if (! is_string($name) || ! isset(self::$startTimes[$name])) { + return; + } + + $duration = $event->telemetryInfo()->time()->duration(self::$startTimes[$name]); + + self::$timings[$name] = round($duration->asFloat(), 4); + } + + /** + * Returns the collected timings. + * + * @return array + */ + public static function timings(): array + { + return self::$timings; + } +} diff --git a/tests/.pest/snapshots/Features/Expect/toMatchSnapshot/failures.snap b/tests/.pest/snapshots/Features/Expect/toMatchSnapshot/failures.snap deleted file mode 100644 index c2b4dc0a..00000000 --- a/tests/.pest/snapshots/Features/Expect/toMatchSnapshot/failures.snap +++ /dev/null @@ -1,7 +0,0 @@ -
-
-
-

Snapshot

-
-
-
\ No newline at end of file diff --git a/tests/.pest/snapshots/Features/Expect/toMatchSnapshot/failures_with_custom_message.snap b/tests/.pest/snapshots/Features/Expect/toMatchSnapshot/failures_with_custom_message.snap deleted file mode 100644 index c2b4dc0a..00000000 --- a/tests/.pest/snapshots/Features/Expect/toMatchSnapshot/failures_with_custom_message.snap +++ /dev/null @@ -1,7 +0,0 @@ -
-
-
-

Snapshot

-
-
-
\ No newline at end of file diff --git a/tests/.pest/snapshots/Visual/Help/visual_snapshot_of_help_command_output.snap b/tests/.pest/snapshots/Visual/Help/visual_snapshot_of_help_command_output.snap index f5307875..39857c68 100644 --- a/tests/.pest/snapshots/Visual/Help/visual_snapshot_of_help_command_output.snap +++ b/tests/.pest/snapshots/Visual/Help/visual_snapshot_of_help_command_output.snap @@ -1,5 +1,5 @@ - Pest Testing Framework 5.0.0-rc.3. + Pest Testing Framework 5.0.0-rc.4. USAGE: pest [options] @@ -50,6 +50,7 @@ EXECUTION OPTIONS: --parallel ........................................... Run tests in parallel --update-snapshots Update snapshots for tests using the "toMatchSnapshot" expectation + --update-shards Update shards.json with test timing data for time-balanced sharding --globals-backup ................. Backup and restore $GLOBALS for each test --static-backup ......... Backup and restore static properties for each test --strict-coverage ................... Be strict about code coverage metadata @@ -91,7 +92,11 @@ --cache-result ............................ Write test results to cache file --do-not-cache-result .............. Do not write test results to cache file --order-by [order] Run tests in order: default|defects|depends|duration|no-depends|random|reverse|size + --resolve-dependencies ...................... Alias for "--order-by depends" + --ignore-dependencies .................... Alias for "--order-by no-depends" + --random-order ............................... Alias for "--order-by random" --random-order-seed [N] Use the specified random seed when running tests in random order + --reverse-order ............................. Alias for "--order-by reverse" REPORTING OPTIONS: --colors=[flag] ......... Use colors in output ("never", "auto" or "always") diff --git a/tests/.pest/snapshots/Visual/Version/visual_snapshot_of_help_command_output.snap b/tests/.pest/snapshots/Visual/Version/visual_snapshot_of_help_command_output.snap index 2f335b21..2007154e 100644 --- a/tests/.pest/snapshots/Visual/Version/visual_snapshot_of_help_command_output.snap +++ b/tests/.pest/snapshots/Visual/Version/visual_snapshot_of_help_command_output.snap @@ -1,3 +1,3 @@ - Pest Testing Framework 5.0.0-rc.3. + Pest Testing Framework 5.0.0-rc.4. diff --git a/tests/.snapshots/success.txt b/tests/.snapshots/success.txt index bd761d83..d30f8c00 100644 --- a/tests/.snapshots/success.txt +++ b/tests/.snapshots/success.txt @@ -1037,8 +1037,6 @@ ✓ pass with toArray ✓ pass with array ✓ pass with toSnapshot - ✓ failures - ✓ failures with custom message ✓ not failures ✓ multiple snapshot expectations ✓ multiple snapshot expectations with datasets with (1) @@ -1905,4 +1903,4 @@ ✓ pass with dataset with ('my-datas-set-value') ✓ within describe → pass with dataset with ('my-datas-set-value') - Tests: 2 deprecated, 4 warnings, 5 incomplete, 2 notices, 40 todos, 35 skipped, 1298 passed (2982 assertions) \ No newline at end of file + Tests: 2 deprecated, 4 warnings, 5 incomplete, 2 notices, 40 todos, 35 skipped, 1296 passed (2976 assertions) \ No newline at end of file diff --git a/tests/Arch.php b/tests/Arch.php index 6348a0f3..d0565216 100644 --- a/tests/Arch.php +++ b/tests/Arch.php @@ -17,7 +17,9 @@ arch()->preset()->security()->ignoring([ 'eval', 'str_shuffle', 'exec', + 'md5', 'unserialize', + 'uniqid', 'extract', 'assert', ]); diff --git a/tests/Features/Expect/toMatchSnapshot.php b/tests/Features/Expect/toMatchSnapshot.php index 0c09be93..6e444eba 100644 --- a/tests/Features/Expect/toMatchSnapshot.php +++ b/tests/Features/Expect/toMatchSnapshot.php @@ -134,18 +134,6 @@ test('pass with `toSnapshot`', function () { expect($object)->toMatchSnapshot(); }); -test('failures', function () { - TestSuite::getInstance()->snapshots->save($this->snapshotable); - - expect('contain that does not match snapshot')->toMatchSnapshot(); -})->throws(ExpectationFailedException::class, 'Failed asserting that two strings are identical.'); - -test('failures with custom message', function () { - TestSuite::getInstance()->snapshots->save($this->snapshotable); - - expect('contain that does not match snapshot')->toMatchSnapshot('oh no'); -})->throws(ExpectationFailedException::class, 'oh no'); - test('not failures', function () { TestSuite::getInstance()->snapshots->save($this->snapshotable); diff --git a/tests/Pest.php b/tests/Pest.php index e498450c..5185f337 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -86,5 +86,12 @@ dataset('dataset_in_pest_file', ['A', 'B']); function removeAnsiEscapeSequences(string $input): ?string { - return preg_replace('#\\x1b[[][^A-Za-z]*[A-Za-z]#', '', $input); + return preg_replace( + [ + '#\\x1b[[][^A-Za-z]*[A-Za-z]#', // CSI (colors, cursor, etc.) + '#\\x1b\\]8;[^\\x1b\\x07]*(?:\\x1b\\\\|\\x07)#', // OSC 8 hyperlinks + ], + '', + $input, + ); } diff --git a/tests/Visual/Parallel.php b/tests/Visual/Parallel.php index 329c6528..60addb9c 100644 --- a/tests/Visual/Parallel.php +++ b/tests/Visual/Parallel.php @@ -23,13 +23,13 @@ test('parallel', function () use ($run) { $file = file_get_contents(__FILE__); $file = preg_replace( '/\$expected = \'.*?\';/', - "\$expected = '2 deprecated, 4 warnings, 5 incomplete, 3 notices, 40 todos, 27 skipped, 1282 passed (2931 assertions)';", + "\$expected = '2 deprecated, 4 warnings, 5 incomplete, 3 notices, 40 todos, 27 skipped, 1280 passed (2925 assertions)';", $file, ); file_put_contents(__FILE__, $file); } - $expected = '2 deprecated, 4 warnings, 5 incomplete, 3 notices, 40 todos, 27 skipped, 1282 passed (2931 assertions)'; + $expected = '2 deprecated, 4 warnings, 5 incomplete, 3 notices, 40 todos, 27 skipped, 1280 passed (2925 assertions)'; expect($output) ->toContain("Tests: {$expected}") diff --git a/tests/Visual/Success.php b/tests/Visual/Success.php index 7906378f..cae012d2 100644 --- a/tests/Visual/Success.php +++ b/tests/Visual/Success.php @@ -21,8 +21,10 @@ test('visual snapshot of test suite on success', function () { return preg_replace([ '#\\x1b[[][^A-Za-z]*[A-Za-z]#', + '#\\x1b\\]8;[^\\x1b\\x07]*(?:\\x1b\\\\|\\x07)#', '/(Tests\\\PHPUnit\\\CustomAffixes\\\InvalidTestName)([A-Za-z0-9]*)/', ], [ + '', '', '$1', ], $process->getOutput());