perf(cache): make the file watcher respect scan filters; smarter defaults

README.md

@@ -29,8 +29,20 @@ They can be set in user preferences (`ctrl+,` or `cmd+,`) or workspace settings
 
 // An array of glob keys to ignore when searching.
 "relativePath.ignore": [
+	"**/.git/**",
 	"**/node_modules/**",
+	"**/.husky/**",
+	"**/.next/**",
+	"**/dist/**",
+	"**/build/**",
+	"**/coverage/**",
+	"**/.turbo/**",
+	"**/.cache/**",
+	"**/.venv/**",
+	"**/__pycache__/**",
 	"**/*.dll",
+	"**/*.swp",
+	"**/*.un~",
 	"**/obj/**",
 	"**/objd/**"
 ],
@@ -44,9 +56,9 @@ They can be set in user preferences (`ctrl+,` or `cmd+,`) or workspace settings
 	".ts"
 ],
 
-// For performance optimization the default limit for quick filter is 1,000 files.
-// Extending this may lead to performance issues
-"relativePath.searchCountLimit": 1000,
+// Max number of files shown directly in the quick filter picker.
+// Above this, a search box is shown first to narrow results.
+"relativePath.searchCountLimit": 10000,
 
 // Max number of files scanned and cached from the workspace. Bounds memory and
 // scan time on very large workspaces; the picker indicates when results were

package.json

@@ -63,6 +63,13 @@
                         "**/node_modules/**",
                         "**/.husky/**",
                         "**/.next/**",
+                        "**/dist/**",
+                        "**/build/**",
+                        "**/coverage/**",
+                        "**/.turbo/**",
+                        "**/.cache/**",
+                        "**/.venv/**",
+                        "**/__pycache__/**",
                         "**/*.dll",
                         "**/*.swp",
                         "**/*.un~",
@@ -101,8 +108,8 @@
                 },
                 "relativePath.searchCountLimit": {
                     "type": "integer",
-                    "default": 1000,
-                    "description": "Max number of files searched in quick filter. Extending this may lead to performance issues."
+                    "default": 10000,
+                    "description": "Max number of files shown directly in the quick filter picker. Above this, a search box is shown first to narrow results."
                 },
                 "relativePath.maxFilesCached": {
                     "type": "integer",

src/extension.ts

@@ -15,6 +15,7 @@ import {
     WorkspaceConfiguration,
 } from "vscode";
 import { getClosestMatches } from "./closest-match";
+import { shouldAddToCache } from "./glob-match";
 import { buildExcludeGlob, normalizeIncludeGlob } from "./globs";
 import { isTruncated, resolveMaxResults } from "./search-limit";
 
@@ -69,9 +70,39 @@ class RelativePath {
             IGNORE_DELETE_EVENTS
         );
 
-        // Add a file on creation
+        // Add a file on creation, applying the same include/ignore/cap
+        // filters as the findFiles scan so bulk creations in ignored folders
+        // (e.g. npm install) don't flood the cache.
         this._watcher.onDidCreate((e) => {
-            this._fileNames.push(e.fsPath.replace(/\\/g, "/"));
+            if (!this._fileNames) {
+                // Initial scan hasn't finished; it will pick this file up.
+                return;
+            }
+
+            const filePath = e.fsPath.replace(/\\/g, "/");
+            const workspacePrefix = `${this._workspacePath}/`;
+            if (!filePath.startsWith(workspacePrefix)) {
+                return;
+            }
+
+            const maxResults = resolveMaxResults(
+                this._configuration.get("maxFilesCached")
+            );
+            if (isTruncated(this._fileNames.length, maxResults)) {
+                this._truncated = true;
+                return;
+            }
+
+            const includeGlob: string = this._configuration.get("includeGlob");
+            if (
+                shouldAddToCache(
+                    filePath.slice(workspacePrefix.length),
+                    normalizeIncludeGlob(includeGlob),
+                    this._configuration.get("ignore")
+                )
+            ) {
+                this._fileNames.push(filePath);
+            }
         });
 
         // on change active text editor refresh the cache
@@ -89,6 +120,10 @@ class RelativePath {
 
         // Remove a file on deletion
         this._watcher.onDidDelete((e) => {
+            if (!this._fileNames) {
+                return;
+            }
+
             let item = e.fsPath.replace(/\\/g, "/");
             let index = this._fileNames.indexOf(item);
             if (index > -1) {
@@ -202,7 +237,9 @@ class RelativePath {
                     this._configuration.ignore,
                     lastConfig.ignore
                 ) ||
-                this._configuration.maxFilesCached !== lastConfig.maxFilesCached
+                this._configuration.maxFilesCached !==
+                    lastConfig.maxFilesCached ||
+                this._configuration.includeGlob !== lastConfig.includeGlob
             ) {
                 this.updateFiles(true);
             }

src/glob-match.ts

@@ -0,0 +1,76 @@
+// Matches a single workspace-relative path against the include/ignore globs,
+// so the file watcher can apply the same filters as the findFiles scan when a
+// file is created. Supports the glob subset VS Code documents for these
+// settings: `**` (any depth, including none), `*` and `?` (within one
+// segment), and non-nested `{a,b}` alternation.
+
+export function globToRegExp(glob: string): RegExp {
+    return new RegExp(`^${convert(glob)}$`);
+}
+
+// Should a newly created file enter the cache? Mirrors what findFiles would
+// decide for the same path, so creations in ignored folders (e.g. an
+// `npm install` filling node_modules) don't flood the cache.
+export function shouldAddToCache(
+    relativePath: string,
+    includeGlob: string,
+    ignoreGlobs: string[] | undefined
+): boolean {
+    if (!globToRegExp(includeGlob).test(relativePath)) {
+        return false;
+    }
+
+    return !(ignoreGlobs ?? []).some((glob) =>
+        globToRegExp(glob).test(relativePath)
+    );
+}
+
+function convert(glob: string): string {
+    let source = "";
+    let index = 0;
+
+    while (index < glob.length) {
+        const char = glob.charAt(index);
+
+        if (char === "*") {
+            if (glob.charAt(index + 1) === "*") {
+                if (glob.charAt(index + 2) === "/") {
+                    // "**/" spans zero or more whole segments
+                    source += "(?:[^/]+/)*";
+                    index += 3;
+                } else {
+                    source += ".*";
+                    index += 2;
+                }
+            } else {
+                source += "[^/]*";
+                index += 1;
+            }
+        } else if (char === "?") {
+            source += "[^/]";
+            index += 1;
+        } else if (char === "{") {
+            const end = glob.indexOf("}", index);
+            if (end === -1) {
+                source += "\\{";
+                index += 1;
+            } else {
+                const options = glob
+                    .slice(index + 1, end)
+                    .split(",")
+                    .map(convert);
+                source += `(?:${options.join("|")})`;
+                index = end + 1;
+            }
+        } else {
+            source += escapeRegExp(char);
+            index += 1;
+        }
+    }
+
+    return source;
+}
+
+function escapeRegExp(char: string): string {
+    return /[.+^$()|[\]\\}]/.test(char) ? `\\${char}` : char;
+}

test/glob-match.test.ts

@@ -0,0 +1,78 @@
+import * as assert from "node:assert/strict";
+import { test } from "node:test";
+import { globToRegExp, shouldAddToCache } from "../src/glob-match";
+
+const DEFAULT_IGNORE = [
+    "**/.git/**",
+    "**/node_modules/**",
+    "**/*.dll",
+    "**/obj/**",
+];
+
+test("** crosses directories, including zero of them", () => {
+    const re = globToRegExp("**/node_modules/**");
+    assert.equal(re.test("node_modules/react/index.js"), true);
+    assert.equal(re.test("packages/app/node_modules/lodash/a.js"), true);
+    assert.equal(re.test("src/app.ts"), false);
+    assert.equal(re.test("src/my-node_modules-notes.md"), false);
+});
+
+test("* and ? stay within a single path segment", () => {
+    assert.equal(globToRegExp("**/*.dll").test("bin/app.dll"), true);
+    assert.equal(globToRegExp("**/*.dll").test("app.dll"), true);
+    assert.equal(globToRegExp("**/*.dll").test("app.dll.txt"), false);
+    assert.equal(globToRegExp("*.ts").test("src/a.ts"), false);
+    assert.equal(globToRegExp("a?.ts").test("ab.ts"), true);
+    assert.equal(globToRegExp("a?.ts").test("a/b.ts"), false);
+});
+
+test("supports single-level brace alternation", () => {
+    const re = globToRegExp("**/*.{js,ts}");
+    assert.equal(re.test("src/a.js"), true);
+    assert.equal(re.test("src/b.ts"), true);
+    assert.equal(re.test("src/c.css"), false);
+});
+
+test("escapes regex metacharacters in literals", () => {
+    assert.equal(globToRegExp("a+b/c.ts").test("a+b/c.ts"), true);
+    assert.equal(globToRegExp("a+b/c.ts").test("aab/cxts"), false);
+});
+
+test("prefix globs like src/**/*.ts match at any depth under the prefix", () => {
+    const re = globToRegExp("src/**/*.ts");
+    assert.equal(re.test("src/a.ts"), true);
+    assert.equal(re.test("src/deep/nested/a.ts"), true);
+    assert.equal(re.test("test/a.ts"), false);
+});
+
+test("shouldAddToCache rejects ignored paths", () => {
+    assert.equal(
+        shouldAddToCache(
+            "node_modules/react/index.js",
+            "**/*.*",
+            DEFAULT_IGNORE
+        ),
+        false
+    );
+    assert.equal(
+        shouldAddToCache("bin/app.dll", "**/*.*", DEFAULT_IGNORE),
+        false
+    );
+});
+
+test("shouldAddToCache rejects paths outside the include glob", () => {
+    assert.equal(shouldAddToCache("Makefile", "**/*.*", DEFAULT_IGNORE), false);
+    assert.equal(
+        shouldAddToCache("test/a.ts", "src/**/*.ts", DEFAULT_IGNORE),
+        false
+    );
+});
+
+test("shouldAddToCache accepts a normal source file", () => {
+    assert.equal(
+        shouldAddToCache("src/app.ts", "**/*.*", DEFAULT_IGNORE),
+        true
+    );
+    assert.equal(shouldAddToCache("src/app.ts", "**/*.*", []), true);
+    assert.equal(shouldAddToCache("src/app.ts", "**/*.*", undefined), true);
+});