Fix parallel dataset reporting and nested fixtures

src/Support/DatasetInfo.php

@@ -17,7 +17,7 @@ final class DatasetInfo
 
     public static function isInsideADatasetsDirectory(string $file): bool
     {
-        return basename(dirname($file)) === self::DATASETS_DIR_NAME;
+        return in_array(self::DATASETS_DIR_NAME, self::directorySegmentsInsideTestsDirectory($file), true);
     }
 
     public static function isADatasetsFile(string $file): bool
@@ -32,7 +32,23 @@ final class DatasetInfo
         }
 
         if (self::isInsideADatasetsDirectory($file)) {
-            return dirname($file, 2);
+            $scope = [];
+
+            foreach (self::directorySegmentsInsideTestsDirectory($file) as $segment) {
+                if ($segment === self::DATASETS_DIR_NAME) {
+                    break;
+                }
+
+                $scope[] = $segment;
+            }
+
+            $testsDirectoryPath = self::testsDirectoryPath($file);
+
+            if ($scope === []) {
+                return $testsDirectoryPath;
+            }
+
+            return $testsDirectoryPath.DIRECTORY_SEPARATOR.implode(DIRECTORY_SEPARATOR, $scope);
         }
 
         if (self::isADatasetsFile($file)) {
@@ -41,4 +57,45 @@ final class DatasetInfo
 
         return $file;
     }
+
+    /**
+     * @return list<string>
+     */
+    private static function directorySegmentsInsideTestsDirectory(string $file): array
+    {
+        $directory = dirname(self::pathInsideTestsDirectory($file));
+
+        if ($directory === '.' || $directory === DIRECTORY_SEPARATOR) {
+            return [];
+        }
+
+        return array_values(array_filter(
+            explode(DIRECTORY_SEPARATOR, trim($directory, DIRECTORY_SEPARATOR)),
+            static fn (string $segment): bool => $segment !== '',
+        ));
+    }
+
+    private static function pathInsideTestsDirectory(string $file): string
+    {
+        $testsDirectory = DIRECTORY_SEPARATOR.trim(testDirectory(), DIRECTORY_SEPARATOR).DIRECTORY_SEPARATOR;
+        $position = strrpos($file, $testsDirectory);
+
+        if ($position === false) {
+            return $file;
+        }
+
+        return substr($file, $position + strlen($testsDirectory));
+    }
+
+    private static function testsDirectoryPath(string $file): string
+    {
+        $testsDirectory = DIRECTORY_SEPARATOR.trim(testDirectory(), DIRECTORY_SEPARATOR).DIRECTORY_SEPARATOR;
+        $position = strrpos($file, $testsDirectory);
+
+        if ($position === false) {
+            return dirname($file);
+        }
+
+        return substr($file, 0, $position + strlen($testsDirectory) - 1);
+    }
 }

src/Support/StateGenerator.php

@@ -11,6 +11,10 @@ use PHPUnit\Event\Code\TestDoxBuilder;
 use PHPUnit\Event\Code\TestMethod;
 use PHPUnit\Event\Code\ThrowableBuilder;
 use PHPUnit\Event\Test\Errored;
+use PHPUnit\Event\Test\PhpunitDeprecationTriggered;
+use PHPUnit\Event\Test\PhpunitErrorTriggered;
+use PHPUnit\Event\Test\PhpunitNoticeTriggered;
+use PHPUnit\Event\Test\PhpunitWarningTriggered;
 use PHPUnit\Event\TestData\TestDataCollection;
 use PHPUnit\Framework\SkippedWithMessageException;
 use PHPUnit\Metadata\MetadataCollection;
@@ -43,6 +47,8 @@ final class StateGenerator
             ));
         }
 
+        $this->addTriggeredPhpunitEvents($state, $testResult->testTriggeredPhpunitErrorEvents(), TestResult::FAIL);
+
         foreach ($testResult->testMarkedIncompleteEvents() as $testResultEvent) {
             $state->add(TestResult::fromPestParallelTestCase(
                 $testResultEvent->test(),
@@ -99,6 +105,8 @@ final class StateGenerator
             }
         }
 
+        $this->addTriggeredPhpunitEvents($state, $testResult->testTriggeredPhpunitDeprecationEvents(), TestResult::DEPRECATED);
+
         foreach ($testResult->notices() as $testResultEvent) {
             foreach ($testResultEvent->triggeringTests() as $triggeringTest) {
                 ['test' => $test] = $triggeringTest;
@@ -123,6 +131,8 @@ final class StateGenerator
             }
         }
 
+        $this->addTriggeredPhpunitEvents($state, $testResult->testTriggeredPhpunitNoticeEvents(), TestResult::NOTICE);
+
         foreach ($testResult->warnings() as $testResultEvent) {
             foreach ($testResultEvent->triggeringTests() as $triggeringTest) {
                 ['test' => $test] = $triggeringTest;
@@ -135,6 +145,8 @@ final class StateGenerator
             }
         }
 
+        $this->addTriggeredPhpunitEvents($state, $testResult->testTriggeredPhpunitWarningEvents(), TestResult::WARN);
+
         foreach ($testResult->phpWarnings() as $testResultEvent) {
             foreach ($testResultEvent->triggeringTests() as $triggeringTest) {
                 ['test' => $test] = $triggeringTest;
@@ -165,4 +177,20 @@ final class StateGenerator
 
         return $state;
     }
+
+    /**
+     * @param  array<string, list<PhpunitDeprecationTriggered|PhpunitErrorTriggered|PhpunitNoticeTriggered|PhpunitWarningTriggered>>  $testResultEvents
+     */
+    private function addTriggeredPhpunitEvents(State $state, array $testResultEvents, string $type): void
+    {
+        foreach ($testResultEvents as $events) {
+            foreach ($events as $event) {
+                $state->add(TestResult::fromPestParallelTestCase(
+                    $event->test(),
+                    $type,
+                    ThrowableBuilder::from(new TestOutcome($event->message()))
+                ));
+            }
+        }
+    }
 }

tests/.snapshots/success.txt

@@ -1490,6 +1490,10 @@
    PASS  Tests\Fixtures\ExampleTest
   ✓ it example 2
 
+   PASS  Tests\Fixtures\ParallelNestedDatasets\TestFileWithNestedDataset
+  ✓ loads nested dataset with ('alice')
+  ✓ loads nested dataset with ('bob')
+
    WARN  Tests\Fixtures\UnexpectedOutput
   - output
 
@@ -1640,9 +1644,14 @@
   ✓ it cannot resolve a parameter without type
 
    PASS  Tests\Unit\Support\DatasetInfo
-  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Datase…rs.php', true)
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/Datasets/project/tes…rs.php', true) #1
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/Datasets/project/tes…rs.php', true) #2
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Datase…rs.php', true) #1
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Datase…rs.php', true) #2
   ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Datasets.php', false)
-  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Featur…rs.php', true)
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Featur…rs.php', true) #1
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Featur…rs.php', true) #2
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Featur…rs.php', true) #3
   ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Featur…rs.php', false)
   ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Featur…ts.php', false)
   ✓ it can check if dataset is defined inside a Datasets.php file with ('/var/www/project/tests/Datase…rs.php', false)
@@ -1650,12 +1659,18 @@
   ✓ it can check if dataset is defined inside a Datasets.php file with ('/var/www/project/tests/Featur…rs.php', false) #1
   ✓ it can check if dataset is defined inside a Datasets.php file with ('/var/www/project/tests/Featur…rs.php', false) #2
   ✓ it can check if dataset is defined inside a Datasets.php file with ('/var/www/project/tests/Featur…ts.php', true)
-  ✓ it computes the dataset scope with ('/var/www/project/tests/Datase…rs.php', '/var/www/project/tests')
+  ✓ it computes the dataset scope with ('/var/www/Datasets/project/tes…rs.php', '/var/www/Datasets/project/tests')
+  ✓ it computes the dataset scope with ('/var/www/Datasets/project/tes…rs.php', '/var/www/Datasets/project/tes…atures')
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Datase…rs.php', '/var/www/project/tests') #1
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Datase…rs.php', '/var/www/project/tests') #2
   ✓ it computes the dataset scope with ('/var/www/project/tests/Datasets.php', '/var/www/project/tests')
-  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Features')
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Features') #1
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Features') #2
   ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Featur…rs.php') #1
   ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…ts.php', '/var/www/project/tests/Features')
-  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Featur…ollers')
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Featur…ollers') #1
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Featur…ollers') #2
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Features') #3
   ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Featur…rs.php') #2
   ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…ts.php', '/var/www/project/tests/Featur…ollers')
 
@@ -1739,50 +1754,8 @@
   ✓ it alerts users about tests with arguments but no input
   ✓ it can return an array of all test suite filenames
 
-   PASS  Tests\Visual\BeforeEachTestName
-  ✓ description
-  ✓ latest description
-
-   PASS  Tests\Visual\Collision
-  ✓ collision with ([''])
-
-   PASS  Tests\Visual\Help
-  ✓ visual snapshot of help command output
-
-   WARN  Tests\Visual\JUnit
-  ✓ junit output
-  - junit with parallel → Not working yet
-
-   PASS  Tests\Visual\Parallel
-  ✓ parallel
-  ✓ a parallel test can extend another test with same name
-
-   PASS  Tests\Visual\ParallelNestedDatasets
-  ✓ parallel reports missing nested datasets without a passing summary
-
-   PASS  Tests\Visual\SingleTestOrDirectory
-  ✓ allows to run a single test
-  ✓ allows to run a directory
-  ✓ it disable decorating printer when colors is set to never
-
-   WARN  Tests\Visual\Success
-  - visual snapshot of test suite on success
-
-   WARN  Tests\Visual\TeamCity
-  - visual snapshot of team city with ('Failure.php')
-  - visual snapshot of team city with ('SuccessOnly.php')
-
-   WARN  Tests\Visual\Todo
-  - todos
-  - todos in parallel
-  - todo
-  - todo in parallel
-
-   WARN  Tests\Visual\Version
-  - visual snapshot of help command output
-
    PASS  Testsexternal\Features\Expect\toMatchSnapshot
   ✓ pass with dataset with ('my-datas-set-value')
   ✓ within describe → pass with dataset with ('my-datas-set-value')
 
-  Tests:    2 deprecated, 4 warnings, 5 incomplete, 2 notices, 39 todos, 35 skipped, 1189 passed (2819 assertions)
+  Tests:    2 deprecated, 4 warnings, 5 incomplete, 2 notices, 39 todos, 26 skipped, 1191 passed (2802 assertions)

tests/.tests/ParallelInvalidDataset/MissingDatasetTest.php

@@ -0,0 +1,5 @@
+<?php
+
+test('missing dataset', function (string $value) {
+    expect($value)->toBe('x');
+})->with('missing.dataset');

tests/.tests/ParallelInvalidDataset/PassingTest.php

@@ -0,0 +1,3 @@
+<?php
+
+it('passes')->assertTrue(true);

tests/Fixtures/ParallelNestedDatasets/Datasets/Nested/Users.php

@@ -0,0 +1,6 @@
+<?php
+
+dataset('nested.users', [
+    ['alice'],
+    ['bob'],
+]);

tests/Fixtures/ParallelNestedDatasets/TestFileWithNestedDataset.php

@@ -0,0 +1,5 @@
+<?php
+
+test('loads nested dataset', function (string $name) {
+    expect($name)->not->toBeEmpty();
+})->with('nested.users');

tests/Unit/Support/DatasetInfo.php

@@ -5,9 +5,14 @@ use Pest\Support\DatasetInfo;
 it('can check if dataset is defined inside a Datasets directory', function (string $file, bool $inside) {
     expect(DatasetInfo::isInsideADatasetsDirectory($file))->toBe($inside);
 })->with([
+    ['file' => '/var/www/Datasets/project/tests/Datasets/Nested/Numbers.php', 'inside' => true],
+    ['file' => '/var/www/Datasets/project/tests/Features/Datasets/Nested/Numbers.php', 'inside' => true],
     ['file' => '/var/www/project/tests/Datasets/Numbers.php', 'inside' => true],
+    ['file' => '/var/www/project/tests/Datasets/Nested/Numbers.php', 'inside' => true],
     ['file' => '/var/www/project/tests/Datasets.php', 'inside' => false],
     ['file' => '/var/www/project/tests/Features/Datasets/Numbers.php', 'inside' => true],
+    ['file' => '/var/www/project/tests/Features/Datasets/Nested/Numbers.php', 'inside' => true],
+    ['file' => '/var/www/project/tests/Features/Datasets/Nested/Datasets/Numbers.php', 'inside' => true],
     ['file' => '/var/www/project/tests/Features/Numbers.php', 'inside' => false],
     ['file' => '/var/www/project/tests/Features/Datasets.php', 'inside' => false],
 ]);
@@ -25,12 +30,18 @@ it('can check if dataset is defined inside a Datasets.php file', function (strin
 it('computes the dataset scope', function (string $file, string $scope) {
     expect(DatasetInfo::scope($file))->toBe($scope);
 })->with([
+    ['file' => '/var/www/Datasets/project/tests/Datasets/Nested/Numbers.php', 'scope' => '/var/www/Datasets/project/tests'],
+    ['file' => '/var/www/Datasets/project/tests/Features/Datasets/Nested/Numbers.php', 'scope' => '/var/www/Datasets/project/tests/Features'],
     ['file' => '/var/www/project/tests/Datasets/Numbers.php', 'scope' => '/var/www/project/tests'],
+    ['file' => '/var/www/project/tests/Datasets/Nested/Numbers.php', 'scope' => '/var/www/project/tests'],
     ['file' => '/var/www/project/tests/Datasets.php', 'scope' => '/var/www/project/tests'],
     ['file' => '/var/www/project/tests/Features/Datasets/Numbers.php', 'scope' => '/var/www/project/tests/Features'],
+    ['file' => '/var/www/project/tests/Features/Datasets/Nested/Numbers.php', 'scope' => '/var/www/project/tests/Features'],
     ['file' => '/var/www/project/tests/Features/Numbers.php', 'scope' => '/var/www/project/tests/Features/Numbers.php'],
     ['file' => '/var/www/project/tests/Features/Datasets.php', 'scope' => '/var/www/project/tests/Features'],
     ['file' => '/var/www/project/tests/Features/Controllers/Datasets/Numbers.php', 'scope' => '/var/www/project/tests/Features/Controllers'],
+    ['file' => '/var/www/project/tests/Features/Controllers/Datasets/Nested/Numbers.php', 'scope' => '/var/www/project/tests/Features/Controllers'],
+    ['file' => '/var/www/project/tests/Features/Datasets/Nested/Datasets/Numbers.php', 'scope' => '/var/www/project/tests/Features'],
     ['file' => '/var/www/project/tests/Features/Controllers/Numbers.php', 'scope' => '/var/www/project/tests/Features/Controllers/Numbers.php'],
     ['file' => '/var/www/project/tests/Features/Controllers/Datasets.php', 'scope' => '/var/www/project/tests/Features/Controllers'],
 ]);

tests/Visual/Parallel.php

@@ -16,10 +16,17 @@ $run = function () {
 
 test('parallel', function () use ($run) {
     expect($run('--exclude-group=integration'))
-        ->toContain('Tests:    2 deprecated, 4 warnings, 5 incomplete, 2 notices, 39 todos, 26 skipped, 1177 passed (2789 assertions)')
+        ->toContain('Tests:    2 deprecated, 4 warnings, 5 incomplete, 3 notices, 39 todos, 26 skipped, 1190 passed (2802 assertions)')
         ->toContain('Parallel: 3 processes');
 })->skipOnWindows();
 
 test('a parallel test can extend another test with same name', function () use ($run) {
     expect($run('tests/Fixtures/Inheritance'))->toContain('Tests:    1 skipped, 1 passed (1 assertions)');
-});
+})->skipOnWindows();
+
+test('parallel reports invalid datasets as failures', function () use ($run) {
+    expect($run('tests/.tests/ParallelInvalidDataset'))
+        ->toContain("A dataset with the name `missing.dataset` does not exist. You can create it using `dataset('missing.dataset', ['a', 'b']);`.")
+        ->toContain('Tests:    1 failed, 1 passed (1 assertions)')
+        ->toContain('Parallel: 3 processes');
+})->skipOnWindows();

tests/Visual/ParallelNestedDatasets.php

@@ -2,58 +2,13 @@
 
 use Symfony\Component\Process\Process;
 
-$fixture = function (): array {
+$run = function (bool $parallel = false): array {
-    $directory = dirname(__DIR__).DIRECTORY_SEPARATOR.'Features'.DIRECTORY_SEPARATOR.'ParallelNestedDatasetRepro';
+    $command = [
-    $datasetsDirectory = $directory.DIRECTORY_SEPARATOR.'Datasets'.DIRECTORY_SEPARATOR.'Nested';
+        'php',
-    $target = $directory.DIRECTORY_SEPARATOR.'TestFileWithNestedDataset.php';
+        'bin/pest',
-
+        'tests/Fixtures/ParallelNestedDatasets/TestFileWithNestedDataset.php',
-    if (! is_dir($datasetsDirectory)) {
+        '--colors=never',
-        mkdir($datasetsDirectory, 0777, true);
+    ];
-    }
-
-    file_put_contents($datasetsDirectory.DIRECTORY_SEPARATOR.'Users.php', <<<'PHP'
-<?php
-
-dataset('nested.users', [
-    ['alice'],
-    ['bob'],
-]);
-PHP);
-
-    file_put_contents($target, <<<'PHP'
-<?php
-
-test('loads nested dataset', function (string $name) {
-    expect($name)->not->toBeEmpty();
-})->with('nested.users');
-PHP);
-
-    return [$directory, 'tests/Features/ParallelNestedDatasetRepro/TestFileWithNestedDataset.php'];
-};
-
-$cleanup = function (string $directory): void {
-    if (! is_dir($directory)) {
-        return;
-    }
-
-    $iterator = new RecursiveIteratorIterator(
-        new RecursiveDirectoryIterator($directory, FilesystemIterator::SKIP_DOTS),
-        RecursiveIteratorIterator::CHILD_FIRST,
-    );
-
-    foreach ($iterator as $item) {
-        if ($item->isDir()) {
-            rmdir($item->getPathname());
-        } else {
-            unlink($item->getPathname());
-        }
-    }
-
-    rmdir($directory);
-};
-
-$run = function (string $target, bool $parallel = false): array {
-    $command = ['php', 'bin/pest', $target, '--colors=never'];
 
     if ($parallel) {
         $command[] = '--parallel';
@@ -72,20 +27,14 @@ $run = function (string $target, bool $parallel = false): array {
     ];
 };
 
-test('parallel reports missing nested datasets without a passing summary', function () use ($cleanup, $fixture, $run) {
+test('parallel loads nested datasets from nested directories', function () use ($run) {
-    [$directory, $target] = $fixture();
+    $serial = $run();
+    $parallel = $run(true);
 
-    try {
+    expect($serial['exitCode'])->toBe(0)
-        $serial = $run($target);
+        ->and($parallel['exitCode'])->toBe(0)
-        $parallel = $run($target, true);
+        ->and($serial['output'])->toContain('Tests:    2 passed (2 assertions)')
-
+        ->and($parallel['output'])->toContain('Tests:    2 passed (2 assertions)')
-        expect($serial['exitCode'])->toBe(2)
+        ->and($parallel['output'])->toContain('Parallel: 2 processes')
-            ->and($parallel['exitCode'])->toBe(2)
+        ->and($parallel['output'])->not->toContain('No tests found.');
-            ->and($serial['output'])->toContain('INFO  No tests found.')
-            ->and($parallel['output'])->toContain('INFO  No tests found.')
-            ->and($parallel['output'])->toContain('Parallel: 2 processes')
-            ->and($parallel['output'])->not->toContain('passed');
-    } finally {
-        $cleanup($directory);
-    }
 })->skipOnWindows();

tests/Visual/Success.php

@@ -12,9 +12,9 @@ test('visual snapshot of test suite on success', function () {
 
     $output = function () use ($testsPath) {
         $process = (new Process(
-            ['php', 'bin/pest'],
+            ['php', '-d', 'memory_limit=512M', 'bin/pest', '--exclude-group=integration'],
             dirname($testsPath),
-            ['EXCLUDE' => 'integration', '--exclude-group' => 'integration', 'REBUILD_SNAPSHOTS' => false, 'PARATEST' => 0, 'COLLISION_PRINTER' => 'DefaultPrinter', 'COLLISION_IGNORE_DURATION' => 'true'],
+            ['EXCLUDE' => 'integration', 'REBUILD_SNAPSHOTS' => false, 'PARATEST' => 0, 'COLLISION_PRINTER' => 'DefaultPrinter', 'COLLISION_IGNORE_DURATION' => 'true'],
         ));
 
         $process->run();