diff --git a/src/Plugins/Cache.php b/src/Plugins/Cache.php
index 7312691b..a9d881ab 100644
--- a/src/Plugins/Cache.php
+++ b/src/Plugins/Cache.php
@@ -6,6 +6,10 @@ namespace Pest\Plugins;
 
 use Pest\Contracts\Plugins\HandlesArguments;
 use Pest\Plugins\Concerns\HandleArguments;
+use PHPUnit\TextUI\CliArguments\Builder as CliConfigurationBuilder;
+use PHPUnit\TextUI\CliArguments\XmlConfigurationFileFinder;
+use PHPUnit\TextUI\XmlConfiguration\DefaultConfiguration;
+use PHPUnit\TextUI\XmlConfiguration\Loader;
 
 /**
  * @internal
@@ -31,9 +35,19 @@ final class Cache implements HandlesArguments
     public function handleArguments(array $arguments): array
     {
         if (! $this->hasArgument('--cache-directory', $arguments)) {
-            $arguments = $this->pushArgument('--cache-directory', $arguments);
 
-            $arguments = $this->pushArgument((string) realpath(self::TEMPORARY_FOLDER), $arguments);
+            $cliConfiguration  = (new CliConfigurationBuilder)->fromParameters([]);
+            $configurationFile = (new XmlConfigurationFileFinder)->find($cliConfiguration);
+            $xmlConfiguration  = DefaultConfiguration::create();
+
+            if ($configurationFile) {
+                $xmlConfiguration = (new Loader)->load($configurationFile);
+            }
+
+            if (! $xmlConfiguration->phpunit()->hasCacheDirectory()) {
+                $arguments = $this->pushArgument('--cache-directory', $arguments);
+                $arguments = $this->pushArgument((string) realpath(self::TEMPORARY_FOLDER), $arguments);
+            }
         }
 
         if (! $this->hasArgument('--parallel', $arguments)) {
diff --git a/tests/.cache/test-results b/tests/.cache/test-results
new file mode 100644
index 00000000..9c485c33
--- /dev/null
+++ b/tests/.cache/test-results
@@ -0,0 +1 @@
+{"version":"pest_2.32.2","defects":[],"times":{"P\\Tests\\Playground::__pest_evaluable_basic":0.005}}
\ No newline at end of file