Merge pull request #1655 from stsepelin/fix/parallel-empty-suite-reporting

fix: nested dataset discovery and parallel invalid-dataset reporting

src/Plugins/Parallel/Paratest/ResultPrinter.php

@@ -170,6 +170,14 @@ final class ResultPrinter
 
         $state = (new StateGenerator)->fromPhpUnitTestResult($this->passedTests, $testResult);
 
+        if ($testResult->numberOfTestsRun() === 0 && $state->testSuiteTestsCount() === 0) {
+            $this->output->writeln([
+                '',
+                '  <fg=white;options=bold;bg=blue> INFO </> No tests found.',
+                '',
+            ]);
+        }
+
         $this->compactPrinter->errors($state);
         $this->compactPrinter->recap($state, $testResult, $duration, $this->options);
     }

src/Plugins/Parallel/Paratest/WrapperRunner.php

@@ -39,6 +39,7 @@ use function dirname;
 use function file_get_contents;
 use function max;
 use function realpath;
+use function str_starts_with;
 use function unlink;
 use function unserialize;
 use function usleep;
@@ -492,14 +493,52 @@ final class WrapperRunner implements RunnerInterface
     private function getTestFiles(SuiteLoader $suiteLoader): array
     {
         /** @var array<string, non-empty-string> $files */
-        $files = [
-            ...array_values(array_filter(
-                $suiteLoader->tests,
-                fn (string $filename): bool => ! str_ends_with($filename, "eval()'d code")
-            )),
-            ...TestSuite::getInstance()->tests->getFilenames(),
-        ];
+        $files = array_fill_keys(array_values(array_filter(
+            $suiteLoader->tests,
+            fn (string $filename): bool => ! str_ends_with($filename, "eval()'d code")
+        )), null);
 
-        return $files; // @phpstan-ignore-line
+        foreach (TestSuite::getInstance()->tests->getFilenames() as $filename) {
+            if ($this->shouldIncludeBootstrappedTestFile($filename)) {
+                $files[$filename] = null;
+            }
+        }
+
+        return array_keys($files); // @phpstan-ignore-line
+    }
+
+    private function shouldIncludeBootstrappedTestFile(string $filename): bool
+    {
+        if (! $this->options->configuration->hasCliArguments()) {
+            return true;
+        }
+
+        $resolvedFilename = realpath($filename);
+
+        if ($resolvedFilename === false) {
+            $resolvedFilename = realpath($this->options->cwd.DIRECTORY_SEPARATOR.$filename);
+        }
+
+        if ($resolvedFilename === false) {
+            return false;
+        }
+
+        foreach ($this->options->configuration->cliArguments() as $path) {
+            $resolvedPath = realpath($path);
+
+            if ($resolvedPath === false) {
+                continue;
+            }
+
+            if ($resolvedFilename === $resolvedPath) {
+                return true;
+            }
+
+            if (is_dir($resolvedPath) && str_starts_with($resolvedFilename, $resolvedPath.DIRECTORY_SEPARATOR)) {
+                return true;
+            }
+        }
+
+        return false;
     }
 }

src/Support/DatasetInfo.php

@@ -17,7 +17,7 @@ final class DatasetInfo
 
     public static function isInsideADatasetsDirectory(string $file): bool
     {
-        return basename(dirname($file)) === self::DATASETS_DIR_NAME;
+        return in_array(self::DATASETS_DIR_NAME, self::directorySegmentsInsideTestsDirectory($file), true);
     }
 
     public static function isADatasetsFile(string $file): bool
@@ -32,7 +32,23 @@ final class DatasetInfo
         }
 
         if (self::isInsideADatasetsDirectory($file)) {
-            return dirname($file, 2);
+            $scope = [];
+
+            foreach (self::directorySegmentsInsideTestsDirectory($file) as $segment) {
+                if ($segment === self::DATASETS_DIR_NAME) {
+                    break;
+                }
+
+                $scope[] = $segment;
+            }
+
+            $testsDirectoryPath = self::testsDirectoryPath($file);
+
+            if ($scope === []) {
+                return $testsDirectoryPath;
+            }
+
+            return $testsDirectoryPath.DIRECTORY_SEPARATOR.implode(DIRECTORY_SEPARATOR, $scope);
         }
 
         if (self::isADatasetsFile($file)) {
@@ -41,4 +57,45 @@ final class DatasetInfo
 
         return $file;
     }
+
+    /**
+     * @return list<string>
+     */
+    private static function directorySegmentsInsideTestsDirectory(string $file): array
+    {
+        $directory = dirname(self::pathInsideTestsDirectory($file));
+
+        if ($directory === '.' || $directory === DIRECTORY_SEPARATOR) {
+            return [];
+        }
+
+        return array_values(array_filter(
+            explode(DIRECTORY_SEPARATOR, trim($directory, DIRECTORY_SEPARATOR)),
+            static fn (string $segment): bool => $segment !== '',
+        ));
+    }
+
+    private static function pathInsideTestsDirectory(string $file): string
+    {
+        $testsDirectory = DIRECTORY_SEPARATOR.trim(testDirectory(), DIRECTORY_SEPARATOR).DIRECTORY_SEPARATOR;
+        $position = strrpos($file, $testsDirectory);
+
+        if ($position === false) {
+            return $file;
+        }
+
+        return substr($file, $position + strlen($testsDirectory));
+    }
+
+    private static function testsDirectoryPath(string $file): string
+    {
+        $testsDirectory = DIRECTORY_SEPARATOR.trim(testDirectory(), DIRECTORY_SEPARATOR).DIRECTORY_SEPARATOR;
+        $position = strrpos($file, $testsDirectory);
+
+        if ($position === false) {
+            return dirname($file);
+        }
+
+        return substr($file, 0, $position + strlen($testsDirectory) - 1);
+    }
 }

src/Support/StateGenerator.php

@@ -11,6 +11,10 @@ use PHPUnit\Event\Code\TestDoxBuilder;
 use PHPUnit\Event\Code\TestMethod;
 use PHPUnit\Event\Code\ThrowableBuilder;
 use PHPUnit\Event\Test\Errored;
+use PHPUnit\Event\Test\PhpunitDeprecationTriggered;
+use PHPUnit\Event\Test\PhpunitErrorTriggered;
+use PHPUnit\Event\Test\PhpunitNoticeTriggered;
+use PHPUnit\Event\Test\PhpunitWarningTriggered;
 use PHPUnit\Event\TestData\TestDataCollection;
 use PHPUnit\Framework\SkippedWithMessageException;
 use PHPUnit\Metadata\MetadataCollection;
@@ -43,6 +47,8 @@ final class StateGenerator
             ));
         }
 
+        $this->addTriggeredPhpunitEvents($state, $testResult->testTriggeredPhpunitErrorEvents(), TestResult::FAIL);
+
         foreach ($testResult->testMarkedIncompleteEvents() as $testResultEvent) {
             $state->add(TestResult::fromPestParallelTestCase(
                 $testResultEvent->test(),
@@ -99,6 +105,8 @@ final class StateGenerator
             }
         }
 
+        $this->addTriggeredPhpunitEvents($state, $testResult->testTriggeredPhpunitDeprecationEvents(), TestResult::DEPRECATED);
+
         foreach ($testResult->notices() as $testResultEvent) {
             foreach ($testResultEvent->triggeringTests() as $triggeringTest) {
                 ['test' => $test] = $triggeringTest;
@@ -123,6 +131,8 @@ final class StateGenerator
             }
         }
 
+        $this->addTriggeredPhpunitEvents($state, $testResult->testTriggeredPhpunitNoticeEvents(), TestResult::NOTICE);
+
         foreach ($testResult->warnings() as $testResultEvent) {
             foreach ($testResultEvent->triggeringTests() as $triggeringTest) {
                 ['test' => $test] = $triggeringTest;
@@ -135,6 +145,8 @@ final class StateGenerator
             }
         }
 
+        $this->addTriggeredPhpunitEvents($state, $testResult->testTriggeredPhpunitWarningEvents(), TestResult::WARN);
+
         foreach ($testResult->phpWarnings() as $testResultEvent) {
             foreach ($testResultEvent->triggeringTests() as $triggeringTest) {
                 ['test' => $test] = $triggeringTest;
@@ -165,4 +177,20 @@ final class StateGenerator
 
         return $state;
     }
+
+    /**
+     * @param  array<string, list<PhpunitDeprecationTriggered|PhpunitErrorTriggered|PhpunitNoticeTriggered|PhpunitWarningTriggered>>  $testResultEvents
+     */
+    private function addTriggeredPhpunitEvents(State $state, array $testResultEvents, string $type): void
+    {
+        foreach ($testResultEvents as $events) {
+            foreach ($events as $event) {
+                $state->add(TestResult::fromPestParallelTestCase(
+                    $event->test(),
+                    $type,
+                    ThrowableBuilder::from(new TestOutcome($event->message()))
+                ));
+            }
+        }
+    }
 }

tests/.snapshots/success.txt

@@ -1490,6 +1490,10 @@
    PASS  Tests\Fixtures\ExampleTest
   ✓ it example 2
 
+   PASS  Tests\Fixtures\ParallelNestedDatasets\TestFileWithNestedDataset
+  ✓ loads nested dataset with ('alice')
+  ✓ loads nested dataset with ('bob')
+
    WARN  Tests\Fixtures\UnexpectedOutput
   - output
 
@@ -1640,9 +1644,14 @@
   ✓ it cannot resolve a parameter without type
 
    PASS  Tests\Unit\Support\DatasetInfo
-  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Datase…rs.php', true)
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/Datasets/project/tes…rs.php', true) #1
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/Datasets/project/tes…rs.php', true) #2
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Datase…rs.php', true) #1
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Datase…rs.php', true) #2
   ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Datasets.php', false)
-  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Featur…rs.php', true)
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Featur…rs.php', true) #1
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Featur…rs.php', true) #2
+  ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Featur…rs.php', true) #3
   ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Featur…rs.php', false)
   ✓ it can check if dataset is defined inside a Datasets directory with ('/var/www/project/tests/Featur…ts.php', false)
   ✓ it can check if dataset is defined inside a Datasets.php file with ('/var/www/project/tests/Datase…rs.php', false)
@@ -1650,12 +1659,18 @@
   ✓ it can check if dataset is defined inside a Datasets.php file with ('/var/www/project/tests/Featur…rs.php', false) #1
   ✓ it can check if dataset is defined inside a Datasets.php file with ('/var/www/project/tests/Featur…rs.php', false) #2
   ✓ it can check if dataset is defined inside a Datasets.php file with ('/var/www/project/tests/Featur…ts.php', true)
-  ✓ it computes the dataset scope with ('/var/www/project/tests/Datase…rs.php', '/var/www/project/tests')
+  ✓ it computes the dataset scope with ('/var/www/Datasets/project/tes…rs.php', '/var/www/Datasets/project/tests')
+  ✓ it computes the dataset scope with ('/var/www/Datasets/project/tes…rs.php', '/var/www/Datasets/project/tes…atures')
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Datase…rs.php', '/var/www/project/tests') #1
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Datase…rs.php', '/var/www/project/tests') #2
   ✓ it computes the dataset scope with ('/var/www/project/tests/Datasets.php', '/var/www/project/tests')
-  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Features')
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Features') #1
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Features') #2
   ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Featur…rs.php') #1
   ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…ts.php', '/var/www/project/tests/Features')
-  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Featur…ollers')
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Featur…ollers') #1
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Featur…ollers') #2
+  ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Features') #3
   ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…rs.php', '/var/www/project/tests/Featur…rs.php') #2
   ✓ it computes the dataset scope with ('/var/www/project/tests/Featur…ts.php', '/var/www/project/tests/Featur…ollers')
 
@@ -1756,6 +1771,10 @@
    PASS  Tests\Visual\Parallel
   ✓ parallel
   ✓ a parallel test can extend another test with same name
+  ✓ parallel reports invalid datasets as failures
+
+   PASS  Tests\Visual\ParallelNestedDatasets
+  ✓ parallel loads nested datasets from nested directories
 
    PASS  Tests\Visual\SingleTestOrDirectory
   ✓ allows to run a single test
@@ -1782,4 +1801,4 @@
   ✓ pass with dataset with ('my-datas-set-value')
   ✓ within describe → pass with dataset with ('my-datas-set-value')
 
-  Tests:    2 deprecated, 4 warnings, 5 incomplete, 2 notices, 39 todos, 35 skipped, 1188 passed (2813 assertions)
+  Tests:    2 deprecated, 4 warnings, 5 incomplete, 2 notices, 39 todos, 35 skipped, 1203 passed (2835 assertions)

tests/.tests/ParallelInvalidDataset/MissingDatasetTest.php

@@ -0,0 +1,5 @@
+<?php
+
+test('missing dataset', function (string $value) {
+    expect($value)->toBe('x');
+})->with('missing.dataset');

tests/.tests/ParallelInvalidDataset/PassingTest.php

@@ -0,0 +1,3 @@
+<?php
+
+it('passes')->assertTrue(true);

tests/Fixtures/ParallelNestedDatasets/Datasets/Nested/Users.php

@@ -0,0 +1,6 @@
+<?php
+
+dataset('nested.users', [
+    ['alice'],
+    ['bob'],
+]);

tests/Fixtures/ParallelNestedDatasets/TestFileWithNestedDataset.php

@@ -0,0 +1,5 @@
+<?php
+
+test('loads nested dataset', function (string $name) {
+    expect($name)->not->toBeEmpty();
+})->with('nested.users');

tests/Unit/Support/DatasetInfo.php

@@ -5,9 +5,14 @@ use Pest\Support\DatasetInfo;
 it('can check if dataset is defined inside a Datasets directory', function (string $file, bool $inside) {
     expect(DatasetInfo::isInsideADatasetsDirectory($file))->toBe($inside);
 })->with([
+    ['file' => '/var/www/Datasets/project/tests/Datasets/Nested/Numbers.php', 'inside' => true],
+    ['file' => '/var/www/Datasets/project/tests/Features/Datasets/Nested/Numbers.php', 'inside' => true],
     ['file' => '/var/www/project/tests/Datasets/Numbers.php', 'inside' => true],
+    ['file' => '/var/www/project/tests/Datasets/Nested/Numbers.php', 'inside' => true],
     ['file' => '/var/www/project/tests/Datasets.php', 'inside' => false],
     ['file' => '/var/www/project/tests/Features/Datasets/Numbers.php', 'inside' => true],
+    ['file' => '/var/www/project/tests/Features/Datasets/Nested/Numbers.php', 'inside' => true],
+    ['file' => '/var/www/project/tests/Features/Datasets/Nested/Datasets/Numbers.php', 'inside' => true],
     ['file' => '/var/www/project/tests/Features/Numbers.php', 'inside' => false],
     ['file' => '/var/www/project/tests/Features/Datasets.php', 'inside' => false],
 ]);
@@ -25,12 +30,18 @@ it('can check if dataset is defined inside a Datasets.php file', function (strin
 it('computes the dataset scope', function (string $file, string $scope) {
     expect(DatasetInfo::scope($file))->toBe($scope);
 })->with([
+    ['file' => '/var/www/Datasets/project/tests/Datasets/Nested/Numbers.php', 'scope' => '/var/www/Datasets/project/tests'],
+    ['file' => '/var/www/Datasets/project/tests/Features/Datasets/Nested/Numbers.php', 'scope' => '/var/www/Datasets/project/tests/Features'],
     ['file' => '/var/www/project/tests/Datasets/Numbers.php', 'scope' => '/var/www/project/tests'],
+    ['file' => '/var/www/project/tests/Datasets/Nested/Numbers.php', 'scope' => '/var/www/project/tests'],
     ['file' => '/var/www/project/tests/Datasets.php', 'scope' => '/var/www/project/tests'],
     ['file' => '/var/www/project/tests/Features/Datasets/Numbers.php', 'scope' => '/var/www/project/tests/Features'],
+    ['file' => '/var/www/project/tests/Features/Datasets/Nested/Numbers.php', 'scope' => '/var/www/project/tests/Features'],
     ['file' => '/var/www/project/tests/Features/Numbers.php', 'scope' => '/var/www/project/tests/Features/Numbers.php'],
     ['file' => '/var/www/project/tests/Features/Datasets.php', 'scope' => '/var/www/project/tests/Features'],
     ['file' => '/var/www/project/tests/Features/Controllers/Datasets/Numbers.php', 'scope' => '/var/www/project/tests/Features/Controllers'],
+    ['file' => '/var/www/project/tests/Features/Controllers/Datasets/Nested/Numbers.php', 'scope' => '/var/www/project/tests/Features/Controllers'],
+    ['file' => '/var/www/project/tests/Features/Datasets/Nested/Datasets/Numbers.php', 'scope' => '/var/www/project/tests/Features'],
     ['file' => '/var/www/project/tests/Features/Controllers/Numbers.php', 'scope' => '/var/www/project/tests/Features/Controllers/Numbers.php'],
     ['file' => '/var/www/project/tests/Features/Controllers/Datasets.php', 'scope' => '/var/www/project/tests/Features/Controllers'],
 ]);

tests/Visual/Parallel.php

@@ -16,10 +16,17 @@ $run = function () {
 
 test('parallel', function () use ($run) {
     expect($run('--exclude-group=integration'))
-        ->toContain('Tests:    2 deprecated, 4 warnings, 5 incomplete, 2 notices, 39 todos, 26 skipped, 1177 passed (2789 assertions)')
+        ->toContain('Tests:    2 deprecated, 4 warnings, 5 incomplete, 3 notices, 39 todos, 26 skipped, 1190 passed (2802 assertions)')
         ->toContain('Parallel: 3 processes');
 })->skipOnWindows();
 
 test('a parallel test can extend another test with same name', function () use ($run) {
-    expect($run('tests/Fixtures/Inheritance'))->toContain('Tests:    1 skipped, 2 passed (2 assertions)');
-});
+    expect($run('tests/Fixtures/Inheritance'))->toContain('Tests:    1 skipped, 1 passed (1 assertions)');
+})->skipOnWindows();
+
+test('parallel reports invalid datasets as failures', function () use ($run) {
+    expect($run('tests/.tests/ParallelInvalidDataset'))
+        ->toContain("A dataset with the name `missing.dataset` does not exist. You can create it using `dataset('missing.dataset', ['a', 'b']);`.")
+        ->toContain('Tests:    1 failed, 1 passed (1 assertions)')
+        ->toContain('Parallel: 3 processes');
+})->skipOnWindows();

tests/Visual/ParallelNestedDatasets.php

@@ -0,0 +1,40 @@
+<?php
+
+use Symfony\Component\Process\Process;
+
+$run = function (bool $parallel = false): array {
+    $command = [
+        'php',
+        'bin/pest',
+        'tests/Fixtures/ParallelNestedDatasets/TestFileWithNestedDataset.php',
+        '--colors=never',
+    ];
+
+    if ($parallel) {
+        $command[] = '--parallel';
+        $command[] = '--processes=2';
+    }
+
+    $process = new Process($command, dirname(__DIR__, 2),
+        ['COLLISION_PRINTER' => 'DefaultPrinter', 'COLLISION_IGNORE_DURATION' => 'true'],
+    );
+
+    $process->run();
+
+    return [
+        'exitCode' => $process->getExitCode(),
+        'output' => removeAnsiEscapeSequences($process->getOutput().$process->getErrorOutput()),
+    ];
+};
+
+test('parallel loads nested datasets from nested directories', function () use ($run) {
+    $serial = $run();
+    $parallel = $run(true);
+
+    expect($serial['exitCode'])->toBe(0)
+        ->and($parallel['exitCode'])->toBe(0)
+        ->and($serial['output'])->toContain('Tests:    2 passed (2 assertions)')
+        ->and($parallel['output'])->toContain('Tests:    2 passed (2 assertions)')
+        ->and($parallel['output'])->toContain('Parallel: 2 processes')
+        ->and($parallel['output'])->not->toContain('No tests found.');
+})->skipOnWindows();

tests/Visual/Success.php

@@ -12,7 +12,7 @@ test('visual snapshot of test suite on success', function () {
 
     $output = function () use ($testsPath) {
         $process = (new Process(
-            ['php', 'bin/pest'],
+            ['php', '-d', 'memory_limit=256M', 'bin/pest'],
             dirname($testsPath),
             ['EXCLUDE' => 'integration', '--exclude-group' => 'integration', 'REBUILD_SNAPSHOTS' => false, 'PARATEST' => 0, 'COLLISION_PRINTER' => 'DefaultPrinter', 'COLLISION_IGNORE_DURATION' => 'true'],
         ));