diff --git a/test/dev.js b/test/dev.js
index 04d5e4c483..d08f3d9eba 100644
--- a/test/dev.js
+++ b/test/dev.js
@@ -1,3 +1,4 @@
+const fs = require('fs');
 const path = require('path');
 const babelConfig = require('../babel.config');
 
@@ -15,6 +16,12 @@ require('@babel/register')({
 const [testSuiteDir] = process.argv.slice(2);
 
 const configPath = path.resolve(__dirname, testSuiteDir, 'config.ts');
+
+if (!fs.existsSync(configPath)) {
+  console.error('ERROR: You must pass a valid directory under test/ that contains a config.ts');
+  process.exit(1);
+}
+
 process.env.PAYLOAD_CONFIG_PATH = configPath;
 process.env.PAYLOAD_DROP_DATABASE = 'true';