fix(test runner): do not load test files not matching --test-list

packages/playwright/src/common/config.ts

@@ -22,7 +22,7 @@ import { getPackageJsonPath, mergeObjects } from '../util';
 
 import type { Config, Fixtures, Metadata, Project, ReporterDescription } from '../../types/test';
 import type { TestRunnerPluginRegistration } from '../plugins';
-import type { TestCaseFilter } from '../util';
+import type { Matcher, TestCaseFilter } from '../util';
 import type { ConfigCLIOverrides } from './ipc';
 import type { Location } from '../../types/testReporter';
 import type { FullConfig, FullProject } from '../../types/testReporter';
@@ -59,6 +59,7 @@ export class FullConfigInternal {
   cliLastFailed?: boolean;
   cliTestList?: string;
   cliTestListInvert?: string;
+  loadFileFilters: Matcher[] = [];
   preOnlyTestFilters: TestCaseFilter[] = [];
   postShardTestFilters: TestCaseFilter[] = [];
   defineConfigWasUsed = false;

packages/playwright/src/runner/loadUtils.ts

@@ -19,7 +19,7 @@ import fs from 'fs';
 import { toPosixPath } from 'playwright-core/lib/utils';
 
 import { InProcessLoaderHost, OutOfProcessLoaderHost } from './loaderHost';
-import { createFileFiltersFromArguments, createFileMatcherFromArguments, createTitleMatcher, errorWithFile, forceRegExp, parseLocationArg } from '../util';
+import { createFileFiltersFromArguments, createTitleMatcher, errorWithFile, forceRegExp, parseLocationArg } from '../util';
 import { buildProjectsClosure, collectFilesForProject, filterProjects } from './projectUtils';
 import {  createTestGroups, filterForShard } from './testGroups';
 import { applyRepeatEachIndex, bindFileSuiteToProject, filterByFocusedLine, filterOnly, filterTestsRemoveEmptySuites } from '../common/suiteUtils';
@@ -42,7 +42,6 @@ export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTest
   const config = testRun.config;
   const fsCache = new Map();
   const sourceMapCache = new Map();
-  const cliFileMatcher = config.cliArgs.length ? createFileMatcherFromArguments(config.cliArgs) : null;
 
   // First collect all files for the projects in the command line, don't apply any file filters.
   const allFilesForProject = new Map<FullProjectInternal, string[]>();
@@ -56,10 +55,13 @@ export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTest
   const filesToRunByProject = new Map<FullProjectInternal, string[]>();
   for (const [project, files] of allFilesForProject) {
     const matchedFiles = files.filter(file => {
-      const hasMatchingSources = sourceMapSources(file, sourceMapCache).some(source => {
-        if (cliFileMatcher && !cliFileMatcher(source))
-          return false;
+      if (!config.loadFileFilters.length) {
+        // Avoid loading source maps.
         return true;
+      }
+      const hasMatchingSources = sourceMapSources(file, sourceMapCache).some(source => {
+        const matchesAllFileFilters = config.loadFileFilters.every(filter => filter(source));
+        return matchesAllFileFilters;
       });
       return hasMatchingSources;
     });
@@ -350,7 +352,7 @@ function sourceMapSources(file: string, cache: Map<string, string[]>): string[]
   }
 }
 
-export async function loadTestList(config: FullConfigInternal, filePath: string): Promise<TestCaseFilter> {
+export async function loadTestList(config: FullConfigInternal, filePath: string): Promise<{ testFilter: TestCaseFilter, fileFilter: Matcher }> {
   try {
     const content = await fs.promises.readFile(filePath, 'utf-8');
     const lines = content.split('\n').map(line => line.trim()).filter(line => line && !line.startsWith('#'));
@@ -366,7 +368,7 @@ export async function loadTestList(config: FullConfigInternal, filePath: string)
       }
       return { project, file: toPosixPath(parseLocationArg(tokens[0]).file), titlePath: tokens.slice(1) };
     });
-    return (test: TestCase) => descriptions.some(d => {
+    const testFilter = (test: TestCase) => descriptions.some(d => {
       // Note: there is no root yet at the time of filtering.
       const [projectName, , ...titles] = test.titlePath();
       if (d.project !== undefined && d.project !== projectName)
@@ -376,6 +378,11 @@ export async function loadTestList(config: FullConfigInternal, filePath: string)
         return false;
       return d.titlePath.length <= titles.length && d.titlePath.every((_, index) => titles[index] === d.titlePath[index]);
     });
+    const fileFilter = (file: string) => {
+      const relativeFile = toPosixPath(path.relative(config.config.rootDir, file));
+      return descriptions.some(d => d.file === relativeFile);
+    };
+    return { testFilter, fileFilter };
   } catch (e) {
     throw errorWithFile(filePath, 'Cannot read test list file: ' + e.message);
   }

packages/playwright/src/runner/tasks.ts

@@ -31,7 +31,7 @@ import { detectChangedTestFiles } from './vcs';
 import { Suite } from '../common/test';
 import { createTestGroups } from '../runner/testGroups';
 import { cacheDir } from '../transform/compilationCache';
-import { removeDirAndLogToConsole } from '../util';
+import { createFileMatcherFromArguments, removeDirAndLogToConsole } from '../util';
 
 import type { TestGroup } from '../runner/testGroups';
 import type { EnvByProjectId } from './dispatcher';
@@ -258,6 +258,24 @@ export function createLoadTask(mode: 'out-of-process' | 'in-process', options: {
   return {
     title: 'load tests',
     setup: async (testRun, errors, softErrors) => {
+      if (testRun.config.cliArgs.length)
+        testRun.config.loadFileFilters.push(createFileMatcherFromArguments(testRun.config.cliArgs));
+
+      if (testRun.config.cliTestList) {
+        const { testFilter, fileFilter } = await loadTestList(testRun.config, testRun.config.cliTestList);
+        testRun.config.preOnlyTestFilters.push(testFilter);
+        testRun.config.loadFileFilters.push(fileFilter);
+      }
+
+      if (testRun.config.cliTestListInvert) {
+        // Note: invert list does not mean we can filter files. For example, the following invert list
+        // can still run tests from foo.spec.ts:
+        //
+        // foo.spec.ts > some test
+        const { testFilter } = await loadTestList(testRun.config, testRun.config.cliTestListInvert);
+        testRun.config.preOnlyTestFilters.push(test => !testFilter(test));
+      }
+
       await collectProjectsAndTestFiles(testRun, !!options.doNotRunDepsOutsideProjectFilter);
       await loadFileSuites(testRun, mode, options.failOnLoadErrors ? errors : softErrors);
 
@@ -271,16 +289,6 @@ export function createLoadTask(mode: 'out-of-process' | 'in-process', options: {
         testRun.config.preOnlyTestFilters.push(test => changedFiles.has(test.location.file));
       }
 
-      if (testRun.config.cliTestList) {
-        const testListFilter = await loadTestList(testRun.config, testRun.config.cliTestList);
-        testRun.config.preOnlyTestFilters.push(testListFilter);
-      }
-
-      if (testRun.config.cliTestListInvert) {
-        const testListInvertFilter = await loadTestList(testRun.config, testRun.config.cliTestListInvert);
-        testRun.config.preOnlyTestFilters.push(test => !testListInvertFilter(test));
-      }
-
       const { rootSuite, topLevelProjects } = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);
       testRun.rootSuite = rootSuite;
       testRun.failureTracker.onRootSuite(rootSuite, topLevelProjects);

tests/playwright-test/test-list.spec.ts

@@ -226,3 +226,23 @@ test('--test-list with nested group entry should run only tests in that nested g
     'd-test2-p2',
   ]);
 });
+
+test('--test-list should not load files not in the list', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { testDir: '.' };
+    `,
+    'test.list': `
+      compiles.test.ts
+    `,
+    'compiles.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('test', async () => {});
+    `,
+    'fails.test.ts': `
+      await await;
+    `,
+  }, { 'test-list': 'test.list' });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});