fix(server): rescan the workspace index once its file watcher goes live

apps/server/src/workspace/WorkspaceSearchIndex.test.ts

@@ -1,3 +1,4 @@
+// @effect-diagnostics nodeBuiltinImport:off
 import {
   FileFinder,
   type FileItem,
@@ -6,9 +7,16 @@ import {
   type GrepResult,
 } from "@ff-labs/fff-node";
 import { afterEach, expect, it } from "@effect/vitest";
+import * as NodeFS from "node:fs";
+import * as NodeOS from "node:os";
+import * as NodePath from "node:path";
 import * as Cause from "effect/Cause";
+import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
 import * as Exit from "effect/Exit";
+import * as Logger from "effect/Logger";
+import * as Scope from "effect/Scope";
+import * as TestClock from "effect/testing/TestClock";
 import { vi } from "vite-plus/test";
 
 import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts";
@@ -17,6 +25,35 @@ afterEach(() => {
   vi.restoreAllMocks();
 });
 
+function scanProgress(isWatcherReady: boolean) {
+  return {
+    ok: true as const,
+    value: {
+      scannedFilesCount: 1,
+      isScanning: false,
+      isWatcherReady,
+      isWarmupComplete: true,
+    },
+  };
+}
+
+/** A scan-progress mock reporting a background filesystem watcher already live. */
+function watcherReadyProgress() {
+  return vi.fn(() => scanProgress(true));
+}
+
+function mockFinder(getScanProgress: () => ReturnType<typeof scanProgress>) {
+  const scanFiles = vi.fn(() => ({ ok: true as const, value: undefined }));
+  const finder = {
+    destroy: vi.fn(),
+    waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
+    getScanProgress,
+    scanFiles,
+  } as unknown as FileFinder;
+  vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
+  return { finder, scanFiles };
+}
+
 function fileItem(relativePath: string): FileItem {
   return {
     relativePath,
@@ -49,6 +86,7 @@ it.effect("filters image searches before applying the result limit", () =>
       const finder = {
         destroy: vi.fn(),
         waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
+        getScanProgress: watcherReadyProgress(),
         fileSearch,
       } as unknown as FileFinder;
       vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
@@ -148,6 +186,7 @@ it.effect("preserves FileFinder destroy failures as structured defects", () =>
         throw cause;
       }),
       waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
+      getScanProgress: watcherReadyProgress(),
     } as unknown as FileFinder;
     vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
 
@@ -178,6 +217,7 @@ it.effect("preserves search and refresh failures with operation context", () =>
       const finder = {
         destroy: vi.fn(),
         waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
+        getScanProgress: watcherReadyProgress(),
         mixedSearch: vi.fn(() => {
           throw searchCause;
         }),
@@ -240,6 +280,7 @@ it.effect("keeps returned search diagnostics out of the cause chain", () =>
       const finder = {
         destroy: vi.fn(),
         waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
+        getScanProgress: watcherReadyProgress(),
         mixedSearch: vi.fn(() => ({ ok: false, error: "native query rejected" })),
         scanFiles: vi.fn(() => ({ ok: false, error: "native refresh rejected" })),
       } as unknown as FileFinder;
@@ -317,6 +358,7 @@ it.effect("continues whole-word searches after a filtered grep page", () =>
       const finder = {
         destroy: vi.fn(),
         waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
+        getScanProgress: watcherReadyProgress(),
         grep,
       } as unknown as FileFinder;
       vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
@@ -346,3 +388,123 @@ it.effect("continues whole-word searches after a filtered grep page", () =>
     }),
   ),
 );
+
+it.effect("rescans once the filesystem watcher goes live, without blocking creation", () =>
+  Effect.gen(function* () {
+    let polls = 0;
+    const { scanFiles } = mockFinder(vi.fn(() => scanProgress(++polls >= 3)));
+
+    const scope = yield* Scope.make();
+    // Creation must not wait on the watcher: the file tree would sit empty
+    // behind it on every workspace open.
+    yield* Effect.provideService(
+      WorkspaceSearchIndex.make("/workspace/project"),
+      Scope.Scope,
+      scope,
+    );
+    expect(scanFiles).not.toHaveBeenCalled();
+
+    yield* TestClock.adjust(Duration.seconds(1));
+    expect(scanFiles).toHaveBeenCalledTimes(1);
+
+    yield* Scope.close(scope, Exit.void);
+  }),
+);
+
+it.effect("does not rescan when the watcher was already live at creation", () =>
+  Effect.gen(function* () {
+    const { scanFiles } = mockFinder(watcherReadyProgress());
+
+    const scope = yield* Scope.make();
+    yield* Effect.provideService(
+      WorkspaceSearchIndex.make("/workspace/project"),
+      Scope.Scope,
+      scope,
+    );
+
+    yield* TestClock.adjust(Duration.seconds(15));
+    expect(scanFiles).not.toHaveBeenCalled();
+
+    yield* Scope.close(scope, Exit.void);
+  }),
+);
+
+it.effect("serves a non-live index when the watcher never becomes ready", () => {
+  const warnings: string[] = [];
+  const logger = Logger.make(({ logLevel, message }) => {
+    if (logLevel === "Warn") warnings.push(String(message));
+  });
+
+  return Effect.gen(function* () {
+    const { scanFiles } = mockFinder(vi.fn(() => scanProgress(false)));
+
+    const scope = yield* Scope.make();
+    yield* Effect.provideService(
+      WorkspaceSearchIndex.make("/workspace/project"),
+      Scope.Scope,
+      scope,
+    );
+
+    yield* TestClock.adjust(Duration.seconds(15));
+    expect(scanFiles).not.toHaveBeenCalled();
+    expect(warnings).toEqual([
+      expect.stringContaining("serving without a live file watcher") as unknown as string,
+    ]);
+
+    yield* Scope.close(scope, Exit.void);
+  }).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false })));
+});
+
+it.effect("stops waiting for a watcher whose readiness cannot be read", () =>
+  Effect.gen(function* () {
+    const getScanProgress = vi.fn<() => ReturnType<typeof scanProgress>>(() => {
+      throw new Error("native scan progress unavailable");
+    });
+    const { scanFiles } = mockFinder(getScanProgress);
+
+    const scope = yield* Scope.make();
+    yield* Effect.provideService(
+      WorkspaceSearchIndex.make("/workspace/project"),
+      Scope.Scope,
+      scope,
+    );
+
+    yield* TestClock.adjust(Duration.seconds(15));
+    expect(getScanProgress).toHaveBeenCalledTimes(1);
+    expect(scanFiles).not.toHaveBeenCalled();
+
+    yield* Scope.close(scope, Exit.void);
+  }),
+);
+
+// it.live: this drives the real native finder and its background watcher, so
+// it needs real elapsed time rather than TestClock's virtual clock.
+it.live("indexes a file created externally right after construction, without a refresh", () =>
+  Effect.gen(function* () {
+    const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-workspace-index-"));
+    // 20k files, so the scan finishes well before the watcher goes live. A
+    // smaller tree does not separate the two and the test stops guarding the
+    // window the index used to serve blind.
+    for (let directory = 0; directory < 200; directory++) {
+      const directoryPath = NodePath.join(root, `dir-${directory}`);
+      NodeFS.mkdirSync(directoryPath);
+      for (let file = 0; file < 100; file++) {
+        NodeFS.writeFileSync(NodePath.join(directoryPath, `file-${file}.ts`), "export {};\n");
+      }
+    }
+
+    yield* Effect.scoped(
+      Effect.gen(function* () {
+        const searchIndex = yield* WorkspaceSearchIndex.make(root);
+        NodeFS.writeFileSync(NodePath.join(root, "created-externally.txt"), "hi");
+
+        // No refresh() here on purpose: only a live watcher can surface this.
+        yield* Effect.sleep("1 second");
+        const { entries } = yield* searchIndex.list();
+        expect(entries.map((entry) => entry.path)).toContain("created-externally.txt");
+      }),
+    ).pipe(
+      Effect.ensuring(Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true }))),
+    );
+  }),
+);

apps/server/src/workspace/WorkspaceSearchIndex.ts

@@ -10,9 +10,12 @@ import {
   type SearchResult,
 } from "@ff-labs/fff-node";
 import * as Context from "effect/Context";
+import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
 import * as LayerMap from "effect/LayerMap";
+import * as Option from "effect/Option";
+import * as Schedule from "effect/Schedule";
 import * as Schema from "effect/Schema";
 
 import type {
@@ -30,6 +33,8 @@ const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2;
 const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds";
 const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000;
 const WORKSPACE_INDEX_IDLE_TTL = "15 minutes";
+const WORKSPACE_INDEX_WATCHER_POLL_INTERVAL = Duration.millis(50);
+const WORKSPACE_INDEX_WATCHER_TIMEOUT = Duration.millis(WORKSPACE_INDEX_SCAN_TIMEOUT_MS);
 const CONTENT_SEARCH_TIME_BUDGET_MS = 250;
 const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100;
 
@@ -352,6 +357,79 @@ const waitForIndexReady = Effect.fn("WorkspaceSearchIndex.waitForIndexReady")(fu
   }
 });
 
+/**
+ * `waitForIndexReady` covers only the scan and warmup phases; the background
+ * filesystem watcher goes live up to a second later, and every external change
+ * in that window is lost permanently because nothing rescans once the watcher
+ * does start.
+ *
+ * Blocking creation on the watcher would put that second in front of the file
+ * tree on every workspace open, so the index serves immediately and this
+ * closes the window from behind instead: wait for the watcher off the critical
+ * path, then rescan once to pick up whatever the window swallowed. A watcher
+ * that never arrives degrades to a non-live index with a warning rather than
+ * taking the index down - search still answers correctly, it just stops
+ * tracking edits until the next explicit refresh.
+ *
+ * Polling is what the native index itself offers - `waitForIndexReady` polls
+ * the same snapshot - so there is no watcher-ready event to wait on instead.
+ */
+const rescanWhenWatcherReady = Effect.fn("WorkspaceSearchIndex.rescanWhenWatcherReady")(function* (
+  cwd: string,
+  finder: FileFinder,
+): Effect.fn.Return<void> {
+  const warn = (message: string) =>
+    Effect.logWarning(message).pipe(
+      Effect.annotateLogs({ cwd, watcherTimeout: WORKSPACE_INDEX_SCAN_TIMEOUT }),
+    );
+
+  const pollWatcher = Effect.try(() => finder.getScanProgress()).pipe(
+    Effect.match({
+      onFailure: () => "unreadable" as const,
+      onSuccess: (progress) => {
+        if (!progress.ok) return "unreadable" as const;
+        return progress.value.isWatcherReady ? ("ready" as const) : ("starting" as const);
+      },
+    }),
+  );
+
+  // The watcher was already live when the scan finished, so nothing can have
+  // been missed and a rescan would be pure cost.
+  const initialStatus = yield* pollWatcher;
+  if (initialStatus === "ready") return;
+  if (initialStatus === "unreadable") {
+    return yield* warn(
+      "Workspace search index is serving without a confirmed file watcher: its readiness could not be read.",
+    );
+  }
+
+  const outcome = yield* pollWatcher.pipe(
+    Effect.repeat({
+      // A watcher whose readiness cannot be read is not going to start
+      // reporting it later, so stop polling instead of burning the budget.
+      until: (status) => status !== "starting",
+      schedule: Schedule.spaced(WORKSPACE_INDEX_WATCHER_POLL_INTERVAL),
+    }),
+    Effect.timeoutOption(WORKSPACE_INDEX_WATCHER_TIMEOUT),
+    Effect.map(Option.getOrElse(() => "timedOut" as const)),
+  );
+  if (outcome !== "ready") {
+    return yield* warn(
+      "Workspace search index is serving without a live file watcher; external changes will not appear until the next refresh.",
+    );
+  }
+
+  const rescanned = yield* Effect.try(() => finder.scanFiles()).pipe(
+    Effect.map((result) => result.ok),
+    Effect.orElseSucceed(() => false),
+  );
+  if (!rescanned) {
+    yield* warn(
+      "Workspace search index could not rescan after its file watcher went live; changes made during startup may be missing until the next refresh.",
+    );
+  }
+});
+
 export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
   cwd: string,
   variant: WorkspaceSearchIndexVariant = "paths",
@@ -372,6 +450,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
         cause,
       }),
   );
+  yield* Effect.forkScoped(rescanWhenWatcherReady(cwd, finder));
 
   const runSearch = Effect.fn("WorkspaceSearchIndex.runSearch")(function* <A>(
     query: string,