From 8d73d98ca637fd28fe9fa9685f2a52d19a4f111b Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi <17528887+harshithmullapudi@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:14:32 +0000 Subject: [PATCH 1/4] fix(host-service): stop deleted workspaces reappearing on large worktrees Workspace delete archives the row first, then re-reads the worktree's dirty state with the delete dialog's 15 s budget. On a large worktree on a cold or busy disk (a batch delete, a nearly full APFS volume) `git status` needs longer, the pool times out, and the saga un-archives the row: the workspace pops back with "Couldn't verify worktree state ... timed out after 15000ms" and has to be deleted again (SUPER-3376). - The post-archive preflight gets its own 120 s budget. Nobody waits on it, because the row is already gone from every list. The dialog's inspect keeps 15 s. - The dirty check runs `git status --porcelain --untracked-files=normal` instead of simple-git's `status()`, which passes `-u` (all) and lists every file in every untracked directory. Same yes/no answer; on a 200k-file worktree it is 31 ms vs 359 ms and 14 bytes vs 6.5 MB. - `git worktree remove` gets 30 min instead of 120 s. A timeout SIGKILLs git partway through its delete and leaves a half-deleted worktree, still registered, that the saga then restores as a normal workspace. - A failed delete now logs before it restores the row, so the failure is in host-service.log and not only in a renderer toast. --- .../trpc/router/workspace-cleanup/git-ops.ts | 25 +++++---- .../workspace-cleanup/workspace-cleanup.ts | 25 ++++++--- .../workers/tasks/git-worktree-state.test.ts | 52 +++++++++++++++++++ .../host-service/src/workers/tasks/git.ts | 10 +++- .../test/workspace-cleanup.test.ts | 48 +++++++++++++++++ 5 files changed, 142 insertions(+), 18 deletions(-) create mode 100644 packages/host-service/src/workers/tasks/git-worktree-state.test.ts diff --git a/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts b/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts index 7fcee653888..38da228f02f 100644 --- a/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts +++ b/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts @@ -33,6 +33,19 @@ export function isIndeterminateGitTaskFailure(err: unknown): boolean { return err instanceof WorkerTaskError && err.name === "WorkerTaskError"; } +/** The delete dialog waits on this read before it can show its warnings. */ +export const INSPECT_WORKTREE_STATE_TIMEOUT_MS = 15_000; + +/** Destroy reads the state after the row is archived, so nobody waits on + * it, and a timeout restores the row. A cold or busy disk (a batch delete) + * can need far more than the dialog's budget. */ +export const DESTROY_WORKTREE_STATE_TIMEOUT_MS = 120_000; + +/** A timeout kills `git worktree remove` partway through its delete and + * leaves a half-deleted worktree. A multi-GB pnpm tree on a full disk + * needs minutes, so this budget only stops a removal that is hung. */ +export const WORKTREE_REMOVE_TIMEOUT_MS = 30 * 60_000; + export const cleanupGitOps = { /** Same failure surface as `ctx.git(repoPath)` — a throw here maps to the * saga's "failed to open project repo" handling. */ @@ -49,12 +62,9 @@ export const cleanupGitOps = { gitEnv: GitTaskEnv; ignoreInitialCommit?: boolean; }, - signal?: AbortSignal, + options: { timeoutMs: number; signal?: AbortSignal }, ): Promise<{ hasChanges: boolean; hasUnpushedCommits: boolean }> { - return getHostWorkerPool().run(gitWorktreeStateTask, input, { - timeoutMs: 15_000, - signal, - }); + return getHostWorkerPool().run(gitWorktreeStateTask, input, options); }, removeWorktree(input: { @@ -63,11 +73,8 @@ export const cleanupGitOps = { gitEnv: GitTaskEnv; force?: boolean; }): Promise<{ stillRegistered: boolean; removeError?: string }> { - // Generous timeout: removal recursively deletes the worktree - // directory, which can take a while for large trees (node_modules - // etc.). return getHostWorkerPool().run(gitWorktreeRemoveTask, input, { - timeoutMs: 120_000, + timeoutMs: WORKTREE_REMOVE_TIMEOUT_MS, }); }, diff --git a/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts b/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts index 9030a78f89f..00342c9636a 100644 --- a/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts +++ b/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts @@ -30,7 +30,12 @@ import { protectedProcedure, router } from "../../index"; import { getHostWorktreeBaseDir } from "../settings/worktree-location"; import { isInsideSessionsRoot } from "../workspace-creation/shared/session-paths"; import { isInsideProjectWorktreesRoot } from "../workspace-creation/shared/worktree-paths"; -import { cleanupGitOps, isIndeterminateGitTaskFailure } from "./git-ops"; +import { + cleanupGitOps, + DESTROY_WORKTREE_STATE_TIMEOUT_MS, + INSPECT_WORKTREE_STATE_TIMEOUT_MS, + isIndeterminateGitTaskFailure, +} from "./git-ops"; import { isLocalCheckoutWorkspace } from "./is-local-checkout-workspace"; import { removeDirectoryTree } from "./remove-directory-tree"; @@ -155,7 +160,7 @@ export const workspaceCleanupRouter = router({ gitEnv, ignoreInitialCommit: local.type === "session", }, - signal, + { timeoutMs: INSPECT_WORKTREE_STATE_TIMEOUT_MS, signal }, ); return { canDelete: true, @@ -317,10 +322,10 @@ async function runDestroy( ctx, local.worktreePath, ); - const state = await cleanupGitOps.readWorktreeState({ - worktreePath: local.worktreePath, - gitEnv, - }); + const state = await cleanupGitOps.readWorktreeState( + { worktreePath: local.worktreePath, gitEnv }, + { timeoutMs: DESTROY_WORKTREE_STATE_TIMEOUT_MS }, + ); if (state.hasChanges) { throw new TRPCError({ code: "CONFLICT", @@ -411,7 +416,13 @@ async function runDestroy( if (marked && local) trackWorkspaceDeleted(ctx, local); return result; } catch (err) { - if (marked) unarchiveLocalWorkspace(ctx, input.workspaceId); + if (marked) { + console.warn("[workspace-cleanup] delete failed; restoring workspace", { + workspaceId: input.workspaceId, + error: err instanceof Error ? err.message : String(err), + }); + unarchiveLocalWorkspace(ctx, input.workspaceId); + } throw err; } } diff --git a/packages/host-service/src/workers/tasks/git-worktree-state.test.ts b/packages/host-service/src/workers/tasks/git-worktree-state.test.ts new file mode 100644 index 00000000000..9012a1c90fb --- /dev/null +++ b/packages/host-service/src/workers/tasks/git-worktree-state.test.ts @@ -0,0 +1,52 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { simpleGit } from "simple-git"; +import { gitWorktreeStateTask } from "./git.ts"; + +const gitEnv = { + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_NOSYSTEM: "1", +}; + +describe("gitWorktreeStateTask", () => { + let repoPath: string; + + beforeEach(async () => { + repoPath = mkdtempSync(join(tmpdir(), "worktree-state-")); + const git = simpleGit(repoPath); + await git.init(); + await git.raw(["config", "user.email", "test@example.com"]); + await git.raw(["config", "user.name", "test"]); + await git.raw(["config", "commit.gpgsign", "false"]); + writeFileSync(join(repoPath, ".gitignore"), "node_modules/\n"); + writeFileSync(join(repoPath, "tracked.txt"), "committed\n"); + await git.add([".gitignore", "tracked.txt"]); + await git.commit("initial"); + }); + + afterEach(() => { + rmSync(repoPath, { recursive: true, force: true }); + }); + + const readState = () => + gitWorktreeStateTask.handler({ worktreePath: repoPath, gitEnv }); + + test("ignored files do not make the worktree dirty", async () => { + mkdirSync(join(repoPath, "node_modules", "pkg"), { recursive: true }); + writeFileSync(join(repoPath, "node_modules", "pkg", "index.js"), "x"); + expect((await readState()).hasChanges).toBe(false); + }); + + test("a file deep inside an untracked directory makes the worktree dirty", async () => { + mkdirSync(join(repoPath, "generated", "a", "b"), { recursive: true }); + writeFileSync(join(repoPath, "generated", "a", "b", "out.js"), "x"); + expect((await readState()).hasChanges).toBe(true); + }); + + test("a modified tracked file makes the worktree dirty", async () => { + writeFileSync(join(repoPath, "tracked.txt"), "edited\n"); + expect((await readState()).hasChanges).toBe(true); + }); +}); diff --git a/packages/host-service/src/workers/tasks/git.ts b/packages/host-service/src/workers/tasks/git.ts index 36b21758a22..d0718029836 100644 --- a/packages/host-service/src/workers/tasks/git.ts +++ b/packages/host-service/src/workers/tasks/git.ts @@ -328,7 +328,13 @@ export const gitWorktreeStateTask = defineWorkerTask< type: "git/worktreeState", handler: async ({ worktreePath, gitEnv, ignoreInitialCommit }) => { const git = createUserSimpleGit(worktreePath, { env: gitEnv }); - const status = await git.status(); + // `git.status()` passes `-u` (all), which lists every file in every + // untracked directory. One entry per directory is enough to say "dirty". + const status = await git.raw([ + "status", + "--porcelain", + "--untracked-files=normal", + ]); let hasUnpushedCommits = false; try { const result = await git.raw([ @@ -344,7 +350,7 @@ export const gitWorktreeStateTask = defineWorkerTask< } catch { // Leave false — `rev-list` failure isn't a signal we can act on. } - return { hasChanges: !status.isClean(), hasUnpushedCommits }; + return { hasChanges: status.trim() !== "", hasUnpushedCommits }; }, }); diff --git a/packages/host-service/test/workspace-cleanup.test.ts b/packages/host-service/test/workspace-cleanup.test.ts index f6b8f99bc26..4075c272a9a 100644 --- a/packages/host-service/test/workspace-cleanup.test.ts +++ b/packages/host-service/test/workspace-cleanup.test.ts @@ -22,6 +22,7 @@ import { workspaceCleanupRouter, } from "../src/trpc/router/workspace-cleanup/workspace-cleanup"; import type { HostServiceContext } from "../src/types"; +import { getHostWorkerPool } from "../src/workers/host-worker-pool"; import { WorkerTaskError } from "../src/workers/WorkerTaskRunner"; type WorkspaceRow = { @@ -1043,6 +1044,53 @@ describe("workspaceCleanup.destroy cleanup ordering", () => { ).rejects.toThrow(/Couldn't verify worktree state/); }); + test("a preflight slower than the delete dialog's budget deletes instead of restoring the row", async () => { + // `git status` that needs 20 s, as on a cold or busy disk: the pool + // rejects any shorter budget, exactly as it does in the field. + const pool = getHostWorkerPool(); + const realRun = pool.run; + const patchedReadWorktreeState = cleanupGitOps.readWorktreeState; + pool.run = (async (_def, _input, options) => { + const timeoutMs = options?.timeoutMs ?? 0; + if (timeoutMs < 20_000) { + throw new WorkerTaskError( + `[host-worker] Task "git/worktreeState" timed out after ${timeoutMs}ms`, + ); + } + return { hasChanges: false, hasUnpushedCommits: false }; + }) as typeof pool.run; + cleanupGitOps.readWorktreeState = realGitOps.readWorktreeState; + const worktree = mkdtempSync(join(tmpdir(), "workspace-delete-")); + const repo = mkdtempSync(join(tmpdir(), "workspace-delete-repo-")); + try { + const ctx = makeCtx({ + workspace: { + id: "ws-1", + projectId: "p-1", + worktreePath: worktree, + branch: "feature", + }, + project: { id: "p-1", repoPath: repo }, + }); + const caller = workspaceCleanupRouter.createCaller(ctx); + + const result = await caller.destroy({ + workspaceId: "ws-1", + deleteBranch: false, + force: false, + }); + + expect(result.success).toBe(true); + // One broadcast is the archive; an un-archive would add a second. + expect(ctx.__mocks.broadcastWorkspaceChanged).toHaveBeenCalledTimes(1); + } finally { + pool.run = realRun; + cleanupGitOps.readWorktreeState = patchedReadWorktreeState; + rmSync(worktree, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true }); + } + }); + test("preflight git failure (missing worktree) still proceeds idempotently", async () => { const ctx = makeCtx({ workspace: { From 8e38ede345acc393d5a1c60c5606c02b33debd0b Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi <17528887+harshithmullapudi@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:26:43 +0000 Subject: [PATCH 2/4] fix(host-service): cap concurrent worktree removals; address review - Removals run at most two at a time. With a 30 min budget, a batch delete on a slow disk could otherwise hold every shared host-worker slot and queue status and diff reads behind it. The wait happens before dispatch, so it does not use the removal's budget. - Project purge removes its worktrees through the same limit instead of one after another, and one failed removal no longer skips the rest. The 30 min budget is kept there on purpose: purge deletes the rows anyway, so a killed removal would leave a half-deleted folder that nothing ever cleans up. - The restore log line carries the tRPC code, so a routine CONFLICT or teardown PRECONDITION_FAILED reads apart from a real failure. - Constant docs cut to one line each; the rationale is in the PR. --- .../src/projects/project-deletion.test.ts | 16 +++++++ .../src/projects/project-deletion.ts | 42 ++++++++++++------- .../trpc/router/workspace-cleanup/git-ops.ts | 39 ++++++++++++----- .../workspace-cleanup/workspace-cleanup.ts | 1 + .../test/workspace-cleanup.test.ts | 36 +++++++++++++++- 5 files changed, 107 insertions(+), 27 deletions(-) diff --git a/packages/host-service/src/projects/project-deletion.test.ts b/packages/host-service/src/projects/project-deletion.test.ts index 936acc4c872..2973d3dab00 100644 --- a/packages/host-service/src/projects/project-deletion.test.ts +++ b/packages/host-service/src/projects/project-deletion.test.ts @@ -333,6 +333,22 @@ describe("delete permanently", () => { expect(existsSync(repoPath)).toBe(true); }); + test("a worktree whose removal fails does not stop the others", async () => { + const { ctx, addWorkspace, project } = setup(); + const stuck = addWorkspace("stuck"); + const other = addWorkspace("other"); + const removeAndRecord = cleanupGitOps.removeWorktree; + cleanupGitOps.removeWorktree = async (input) => { + if (input.worktreePath === stuck) throw new Error("timed out"); + return removeAndRecord(input); + }; + await softDeleteProject(ctx, PROJECT_ID); + expect(await purgeDeletedProject(ctx, PROJECT_ID)).toBe(true); + expect(project()).toBeUndefined(); + expect(existsSync(stuck)).toBe(true); + expect(existsSync(other)).toBe(false); + }); + test("refuses a project that was never deleted", async () => { const { ctx, addWorkspace, gitCalls, project } = setup(); addWorkspace("live"); diff --git a/packages/host-service/src/projects/project-deletion.ts b/packages/host-service/src/projects/project-deletion.ts index 78f8b86d4a2..bd3457b6656 100644 --- a/packages/host-service/src/projects/project-deletion.ts +++ b/packages/host-service/src/projects/project-deletion.ts @@ -243,22 +243,32 @@ async function purgeProject( if (worktrees.length > 0) { try { const gitEnv = await cleanupGitOps.resolveGitEnv(ctx, project.repoPath); - for (const row of worktrees) { - const { stillRegistered, removeError } = - await cleanupGitOps.removeWorktree({ - repoPath: project.repoPath, - worktreePath: row.worktreePath, - gitEnv, - force: false, - }); - if (stillRegistered) { - console.warn("[project-deletion] left worktree on disk", { - projectId: project.id, - worktreePath: row.worktreePath, - removeError, - }); - } - } + await Promise.all( + worktrees.map(async (row) => { + try { + const { stillRegistered, removeError } = + await cleanupGitOps.removeWorktree({ + repoPath: project.repoPath, + worktreePath: row.worktreePath, + gitEnv, + force: false, + }); + if (stillRegistered) { + console.warn("[project-deletion] left worktree on disk", { + projectId: project.id, + worktreePath: row.worktreePath, + removeError, + }); + } + } catch (err) { + console.warn("[project-deletion] left worktree on disk", { + projectId: project.id, + worktreePath: row.worktreePath, + err, + }); + } + }), + ); } catch (err) { console.warn("[project-deletion] left worktrees on disk", { projectId: project.id, diff --git a/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts b/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts index 38da228f02f..1daa3879e75 100644 --- a/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts +++ b/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts @@ -33,19 +33,36 @@ export function isIndeterminateGitTaskFailure(err: unknown): boolean { return err instanceof WorkerTaskError && err.name === "WorkerTaskError"; } -/** The delete dialog waits on this read before it can show its warnings. */ +/** The delete dialog waits on this read. */ export const INSPECT_WORKTREE_STATE_TIMEOUT_MS = 15_000; -/** Destroy reads the state after the row is archived, so nobody waits on - * it, and a timeout restores the row. A cold or busy disk (a batch delete) - * can need far more than the dialog's budget. */ +/** Runs after the row is archived, so nobody waits on it. */ export const DESTROY_WORKTREE_STATE_TIMEOUT_MS = 120_000; -/** A timeout kills `git worktree remove` partway through its delete and - * leaves a half-deleted worktree. A multi-GB pnpm tree on a full disk - * needs minutes, so this budget only stops a removal that is hung. */ +/** A timeout kills git mid-delete, so this only stops a hung removal. */ export const WORKTREE_REMOVE_TIMEOUT_MS = 30 * 60_000; +/** Long removals must not take every shared worker slot from status and diff reads. */ +export const MAX_CONCURRENT_WORKTREE_REMOVALS = 2; + +let activeWorktreeRemovals = 0; +const queuedWorktreeRemovals: Array<() => void> = []; + +async function withWorktreeRemovalSlot(run: () => Promise): Promise { + if (activeWorktreeRemovals < MAX_CONCURRENT_WORKTREE_REMOVALS) { + activeWorktreeRemovals++; + } else { + await new Promise((resolve) => queuedWorktreeRemovals.push(resolve)); + } + try { + return await run(); + } finally { + const next = queuedWorktreeRemovals.shift(); + if (next) next(); + else activeWorktreeRemovals--; + } +} + export const cleanupGitOps = { /** Same failure surface as `ctx.git(repoPath)` — a throw here maps to the * saga's "failed to open project repo" handling. */ @@ -73,9 +90,11 @@ export const cleanupGitOps = { gitEnv: GitTaskEnv; force?: boolean; }): Promise<{ stillRegistered: boolean; removeError?: string }> { - return getHostWorkerPool().run(gitWorktreeRemoveTask, input, { - timeoutMs: WORKTREE_REMOVE_TIMEOUT_MS, - }); + return withWorktreeRemovalSlot(() => + getHostWorkerPool().run(gitWorktreeRemoveTask, input, { + timeoutMs: WORKTREE_REMOVE_TIMEOUT_MS, + }), + ); }, deleteLocalBranch(input: { diff --git a/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts b/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts index 00342c9636a..cc735ff3f03 100644 --- a/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts +++ b/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts @@ -419,6 +419,7 @@ async function runDestroy( if (marked) { console.warn("[workspace-cleanup] delete failed; restoring workspace", { workspaceId: input.workspaceId, + code: err instanceof TRPCError ? err.code : undefined, error: err instanceof Error ? err.message : String(err), }); unarchiveLocalWorkspace(ctx, input.workspaceId); diff --git a/packages/host-service/test/workspace-cleanup.test.ts b/packages/host-service/test/workspace-cleanup.test.ts index 4075c272a9a..dd133e36743 100644 --- a/packages/host-service/test/workspace-cleanup.test.ts +++ b/packages/host-service/test/workspace-cleanup.test.ts @@ -14,7 +14,10 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { workspaceRouter } from "../src/trpc/router/workspace"; -import { cleanupGitOps } from "../src/trpc/router/workspace-cleanup/git-ops"; +import { + cleanupGitOps, + MAX_CONCURRENT_WORKTREE_REMOVALS, +} from "../src/trpc/router/workspace-cleanup/git-ops"; import { isLocalCheckoutWorkspace } from "../src/trpc/router/workspace-cleanup/is-local-checkout-workspace"; import { __testDestroysInFlight, @@ -1091,6 +1094,37 @@ describe("workspaceCleanup.destroy cleanup ordering", () => { } }); + test("worktree removals beyond the limit wait for a slot instead of taking another worker", async () => { + const pool = getHostWorkerPool(); + const realRun = pool.run; + const finishers: Array<() => void> = []; + pool.run = (() => + new Promise((resolve) => + finishers.push(() => resolve({ stillRegistered: false })), + )) as typeof pool.run; + const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + try { + const removals = [1, 2, 3].map((n) => + realGitOps.removeWorktree({ + repoPath: "/repo", + worktreePath: `/wt-${n}`, + gitEnv: {}, + }), + ); + await flush(); + expect(finishers).toHaveLength(MAX_CONCURRENT_WORKTREE_REMOVALS); + + finishers[0]?.(); + await flush(); + expect(finishers).toHaveLength(3); + + for (const finish of finishers.slice(1)) finish(); + await Promise.all(removals); + } finally { + pool.run = realRun; + } + }); + test("preflight git failure (missing worktree) still proceeds idempotently", async () => { const ctx = makeCtx({ workspace: { From dccbc7d974bbb15c8a4cee5437d8b8a303b754fd Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi <17528887+harshithmullapudi@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:03:56 +0000 Subject: [PATCH 3/4] fix(host-service): hold delete timeouts on 2GB+ worktrees - Destroy preflight budget 120 s -> 10 min. It runs after the archive and only needs to catch a hung git; a timeout un-archives the row, which is the original bug. On a 450k-file / 3.6 GB worktree, cold `git status` is 1.2 s (3.3 s single-threaded) on NVMe, so a 40x slower disk already passes 120 s. - inspect throws on a timeout or pool failure instead of reporting a clean worktree. With the longer preflight, a dirty worktree whose inspect timed out reached CONFLICT, and the dialog then silently force-deleted it. - The delete dialog silently force-retries a CONFLICT only when inspect finished. Otherwise it reopens with the uncommitted-changes warning. --- .../useDestroyDialogState.ts | 9 ++++--- .../utils/canSilentlyForceRetry.test.ts | 27 +++++++++++++++++++ .../utils/canSilentlyForceRetry.ts | 9 +++++++ .../trpc/router/workspace-cleanup/git-ops.ts | 2 +- .../workspace-cleanup/workspace-cleanup.ts | 9 ++++++- .../test/workspace-cleanup.test.ts | 24 ++++++++++++++--- 6 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.test.ts create mode 100644 apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.ts diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/hooks/useDestroyDialogState/useDestroyDialogState.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/hooks/useDestroyDialogState/useDestroyDialogState.ts index cdaea015a61..3c121c5e3b1 100644 --- a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/hooks/useDestroyDialogState/useDestroyDialogState.ts +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/hooks/useDestroyDialogState/useDestroyDialogState.ts @@ -13,6 +13,7 @@ import { useV2UserPreferences } from "renderer/hooks/useV2UserPreferences/useV2U import { useNavigateAwayFromWorkspace } from "renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/hooks/useNavigateAwayFromWorkspace"; import { useDeletingWorkspacesStore } from "renderer/routes/_authenticated/_dashboard/stores/deletingWorkspacesStore"; import { useHostWorkspaces } from "renderer/routes/_authenticated/providers/HostWorkspacesProvider"; +import { canSilentlyForceRetry } from "../../utils/canSilentlyForceRetry"; interface UseDestroyDialogStateOptions { workspaceId: string; @@ -110,6 +111,7 @@ export function useDestroyDialogState({ }) => { if (inFlight.current) return; inFlight.current = true; + const checkedClean = inspectState.status === "ready"; setError(null); onOpenChange(false); @@ -139,7 +141,7 @@ export function useDestroyDialogState({ // retry. Do NOT extend this to `in-progress` (that's a // different CONFLICT cause; retrying just races the same // guard). - if (e.kind === "conflict" && !force) { + if (canSilentlyForceRetry(e, { force, checkedClean })) { result = await destroy({ deleteBranch, force: true, skipTeardown }); } else { throw firstErr; @@ -155,7 +157,7 @@ export function useDestroyDialogState({ onDeleted?.(); } catch (err) { const e = err as DestroyWorkspaceError; - if (e.kind === "teardown-failed") { + if (e.kind === "teardown-failed" || e.kind === "conflict") { setError(e); onOpenChange(true); } else if (e.kind === "in-progress") { @@ -179,6 +181,7 @@ export function useDestroyDialogState({ [ destroy, deleteBranch, + inspectState.status, workspaceName, workspaceId, onOpenChange, @@ -193,7 +196,7 @@ export function useDestroyDialogState({ return { deleteBranch, setDeleteBranch, - hasChanges: preview?.hasChanges ?? false, + hasChanges: preview?.hasChanges === true || error?.kind === "conflict", hasUnpushedCommits: preview?.hasUnpushedCommits ?? false, sharesProjectCheckout: preview?.canDelete === true && preview.sharesProjectCheckout === true, diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.test.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.test.ts new file mode 100644 index 00000000000..5c2b1d151f1 --- /dev/null +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, test } from "bun:test"; +import { canSilentlyForceRetry } from "./canSilentlyForceRetry"; + +const conflict = { kind: "conflict", message: "dirty" } as const; + +describe("canSilentlyForceRetry", () => { + test("retries a conflict after the dialog checked the worktree clean", () => { + expect( + canSilentlyForceRetry(conflict, { force: false, checkedClean: true }), + ).toBe(true); + }); + + test("does not retry a conflict when the check never finished", () => { + expect( + canSilentlyForceRetry(conflict, { force: false, checkedClean: false }), + ).toBe(false); + }); + + test("does not retry other failures", () => { + expect( + canSilentlyForceRetry( + { kind: "in-progress", message: "busy" }, + { force: false, checkedClean: true }, + ), + ).toBe(false); + }); +}); diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.ts new file mode 100644 index 00000000000..dc77e20b1cd --- /dev/null +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.ts @@ -0,0 +1,9 @@ +import type { DestroyWorkspaceError } from "renderer/hooks/host-service/useDestroyWorkspace"; + +/** A dirty worktree the dialog never checked has had no warning shown. */ +export function canSilentlyForceRetry( + error: DestroyWorkspaceError, + { force, checkedClean }: { force: boolean; checkedClean: boolean }, +): boolean { + return error.kind === "conflict" && !force && checkedClean; +} diff --git a/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts b/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts index 1daa3879e75..29b7bdbc771 100644 --- a/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts +++ b/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts @@ -37,7 +37,7 @@ export function isIndeterminateGitTaskFailure(err: unknown): boolean { export const INSPECT_WORKTREE_STATE_TIMEOUT_MS = 15_000; /** Runs after the row is archived, so nobody waits on it. */ -export const DESTROY_WORKTREE_STATE_TIMEOUT_MS = 120_000; +export const DESTROY_WORKTREE_STATE_TIMEOUT_MS = 10 * 60_000; /** A timeout kills git mid-delete, so this only stops a hung removal. */ export const WORKTREE_REMOVE_TIMEOUT_MS = 30 * 60_000; diff --git a/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts b/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts index cc735ff3f03..1d7972c51d3 100644 --- a/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts +++ b/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts @@ -126,6 +126,7 @@ export const workspaceCleanupRouter = router({ * - hasChanges/Unpushed → render as warnings; user can still confirm. * - git failures (missing worktree, broken repo) → return as canDelete * with no warnings; the destroy saga handles those states best-effort. + * - timeout / pool failure → throw: the state is unknown, not clean. * * The git reads run in the host worker pool (gitWorktreeStateTask) so a * slow status on a large worktree can't block the event loop. @@ -169,7 +170,13 @@ export const workspaceCleanupRouter = router({ hasUnpushedCommits: state.hasUnpushedCommits, sharesProjectCheckout: false, }; - } catch { + } catch (err) { + if (isIndeterminateGitTaskFailure(err)) { + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: `Couldn't verify worktree state at ${local.worktreePath}: ${err instanceof Error ? err.message : String(err)}`, + }); + } return { canDelete: true, reason: null, diff --git a/packages/host-service/test/workspace-cleanup.test.ts b/packages/host-service/test/workspace-cleanup.test.ts index dd133e36743..66ab9c1649e 100644 --- a/packages/host-service/test/workspace-cleanup.test.ts +++ b/packages/host-service/test/workspace-cleanup.test.ts @@ -340,6 +340,22 @@ describe("workspaceCleanup.inspect", () => { }); }); + test("a timed-out worktree-state read fails instead of reporting a clean worktree", async () => { + const ctx = makeCtx({ + ...wsAndProject, + worktreeState: () => + Promise.reject( + new WorkerTaskError( + '[host-worker] Task "git/worktreeState" timed out after 15000ms', + ), + ), + }); + const caller = workspaceCleanupRouter.createCaller(ctx); + await expect(caller.inspect({ workspaceId: "ws-1" })).rejects.toThrow( + /Couldn't verify worktree state/, + ); + }); + test("swallows git env-resolution failures and returns canDelete: true with no warnings", async () => { const ctx = makeCtx({ ...wsAndProject, @@ -1047,15 +1063,15 @@ describe("workspaceCleanup.destroy cleanup ordering", () => { ).rejects.toThrow(/Couldn't verify worktree state/); }); - test("a preflight slower than the delete dialog's budget deletes instead of restoring the row", async () => { - // `git status` that needs 20 s, as on a cold or busy disk: the pool - // rejects any shorter budget, exactly as it does in the field. + test("a preflight that needs minutes deletes instead of restoring the row", async () => { + // `git status` that needs 5 min, as on a large worktree on a cold or + // busy disk: the pool rejects any shorter budget, as it does in the field. const pool = getHostWorkerPool(); const realRun = pool.run; const patchedReadWorktreeState = cleanupGitOps.readWorktreeState; pool.run = (async (_def, _input, options) => { const timeoutMs = options?.timeoutMs ?? 0; - if (timeoutMs < 20_000) { + if (timeoutMs < 5 * 60_000) { throw new WorkerTaskError( `[host-worker] Task "git/worktreeState" timed out after ${timeoutMs}ms`, ); From 057577060d77d617d655b7cd8d6881390fc43303 Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi <17528887+harshithmullapudi@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:36:00 +0000 Subject: [PATCH 4/4] fix(host-service): keep the long delete preflight to callers that can wait for it - destroy takes an optional longPreflight. Only the new desktop sends it, and only to this machine's host. Released desktops silently force-retry any CONFLICT, so on a slow disk a long check could delete uncommitted work the dialog never checked. Through the relay (~65 s limit) the client would give up while the host kept deleting. Without the flag the check keeps the dialog's 15 s budget and fails closed as before. - The startup reconciler passes longPreflight; nobody waits on it. - Long preflights share the 2 long-delete-task slots with removals, so they can't take every host worker. - A cancelled inspect returns CLIENT_CLOSED_REQUEST, not a 500. - The limiter test releases its held slots on failure. - Rename checkedClean to inspectFinished. Co-Authored-By: Claude --- .../useDestroyWorkspace.ts | 8 +- .../useBulkWorkspaceDelete.ts | 12 +- .../useDestroyDialogState.ts | 4 +- .../utils/canSilentlyForceRetry.test.ts | 8 +- .../utils/canSilentlyForceRetry.ts | 4 +- .../src/projects/project-deletion.test.ts | 6 +- .../runtime/archived-workspace-reconcile.ts | 1 + .../trpc/router/workspace-cleanup/git-ops.ts | 34 ++++-- .../workspace-cleanup/workspace-cleanup.ts | 26 ++++- .../host-service/src/workers/tasks/git.ts | 3 +- .../test/workspace-cleanup.test.ts | 109 +++++++++++++----- 11 files changed, 151 insertions(+), 64 deletions(-) diff --git a/apps/desktop/src/renderer/hooks/host-service/useDestroyWorkspace/useDestroyWorkspace.ts b/apps/desktop/src/renderer/hooks/host-service/useDestroyWorkspace/useDestroyWorkspace.ts index 745fdeb215a..c9eac9bce10 100644 --- a/apps/desktop/src/renderer/hooks/host-service/useDestroyWorkspace/useDestroyWorkspace.ts +++ b/apps/desktop/src/renderer/hooks/host-service/useDestroyWorkspace/useDestroyWorkspace.ts @@ -22,6 +22,9 @@ export interface DestroyWorkspaceInput { /** Consent to abandon the teardown script — only the teardown-failed * retry sets this. */ skipTeardown?: boolean; + /** Only for this machine's host: the relay gives up on a request after + * about a minute, and a long dirty check can outlast that. */ + longPreflight?: boolean; } export interface DestroyWorkspaceSuccess { @@ -84,6 +87,7 @@ export async function destroyWorkspaceAtHost( deleteBranch: input.deleteBranch ?? false, force: input.force ?? false, skipTeardown: input.skipTeardown ?? false, + longPreflight: input.longPreflight ?? false, }); } catch (error) { throw normalizeDestroyWorkspaceError(error); @@ -163,10 +167,10 @@ export function useDestroyWorkspace(workspaceId: string): UseDestroyWorkspace { } return destroyWorkspaceAtHost( { workspaceId, hostUrl, hostStatus }, - input, + { ...input, longPreflight: hostUrl === activeHostUrl }, ); }, - [hostUrl, hostStatus, isSandbox, utils, workspaceId], + [activeHostUrl, hostUrl, hostStatus, isSandbox, utils, workspaceId], ); const inspect = useCallback(async (): Promise => { diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarBulkDeleteDialog/hooks/useBulkWorkspaceDelete/useBulkWorkspaceDelete.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarBulkDeleteDialog/hooks/useBulkWorkspaceDelete/useBulkWorkspaceDelete.ts index b7b992eb169..28229219d49 100644 --- a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarBulkDeleteDialog/hooks/useBulkWorkspaceDelete/useBulkWorkspaceDelete.ts +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarBulkDeleteDialog/hooks/useBulkWorkspaceDelete/useBulkWorkspaceDelete.ts @@ -19,6 +19,7 @@ import type { DashboardSidebarWorkspace } from "renderer/routes/_authenticated/_ import { useDeletingWorkspacesStore } from "renderer/routes/_authenticated/_dashboard/stores/deletingWorkspacesStore"; import { useDashboardSidebarState } from "renderer/routes/_authenticated/hooks/useDashboardSidebarState"; import { useHostWorkspaces } from "renderer/routes/_authenticated/providers/HostWorkspacesProvider"; +import { useLocalHostService } from "renderer/routes/_authenticated/providers/LocalHostServiceProvider"; import { type BulkWorkspaceInspectionState, buildBulkWorkspaceInspectionSummary, @@ -75,6 +76,7 @@ export function useBulkWorkspaceDelete({ }: UseBulkWorkspaceDeleteOptions) { const { t } = useLingui(); const { cache: hostWorkspacesCache } = useHostWorkspaces(); + const { activeHostUrl } = useLocalHostService(); const { navigateAwayFromWorkspace } = useNavigateAwayFromWorkspace(); const { removeWorkspaceFromSidebar } = useDashboardSidebarState(); const { preferences, setDeleteLocalBranch } = useV2UserPreferences(); @@ -217,12 +219,15 @@ export function useBulkWorkspaceDelete({ shouldRetryWithForce: (workspace, error) => error.kind === "conflict" && inspections.get(workspace.id)?.status !== "ready", - destroy: (workspace, force) => - destroyWorkspaceAtHost(targetFor(workspace), { + destroy: (workspace, force) => { + const target = targetFor(workspace); + return destroyWorkspaceAtHost(target, { deleteBranch: preferences.deleteLocalBranch, force, skipTeardown, - }), + longPreflight: target.hostUrl === activeHostUrl, + }); + }, onSettled: () => { completedCount += 1; if (completedCount < total) { @@ -323,6 +328,7 @@ export function useBulkWorkspaceDelete({ } }, [ + activeHostUrl, hostWorkspacesCache, inspections, navigateAwayFromWorkspace, diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/hooks/useDestroyDialogState/useDestroyDialogState.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/hooks/useDestroyDialogState/useDestroyDialogState.ts index 3c121c5e3b1..84038a586a4 100644 --- a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/hooks/useDestroyDialogState/useDestroyDialogState.ts +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/hooks/useDestroyDialogState/useDestroyDialogState.ts @@ -111,7 +111,7 @@ export function useDestroyDialogState({ }) => { if (inFlight.current) return; inFlight.current = true; - const checkedClean = inspectState.status === "ready"; + const inspectFinished = inspectState.status === "ready"; setError(null); onOpenChange(false); @@ -141,7 +141,7 @@ export function useDestroyDialogState({ // retry. Do NOT extend this to `in-progress` (that's a // different CONFLICT cause; retrying just races the same // guard). - if (canSilentlyForceRetry(e, { force, checkedClean })) { + if (canSilentlyForceRetry(e, { force, inspectFinished })) { result = await destroy({ deleteBranch, force: true, skipTeardown }); } else { throw firstErr; diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.test.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.test.ts index 5c2b1d151f1..e44f5eba17a 100644 --- a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.test.ts +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.test.ts @@ -4,15 +4,15 @@ import { canSilentlyForceRetry } from "./canSilentlyForceRetry"; const conflict = { kind: "conflict", message: "dirty" } as const; describe("canSilentlyForceRetry", () => { - test("retries a conflict after the dialog checked the worktree clean", () => { + test("retries a conflict after the dialog finished checking the worktree", () => { expect( - canSilentlyForceRetry(conflict, { force: false, checkedClean: true }), + canSilentlyForceRetry(conflict, { force: false, inspectFinished: true }), ).toBe(true); }); test("does not retry a conflict when the check never finished", () => { expect( - canSilentlyForceRetry(conflict, { force: false, checkedClean: false }), + canSilentlyForceRetry(conflict, { force: false, inspectFinished: false }), ).toBe(false); }); @@ -20,7 +20,7 @@ describe("canSilentlyForceRetry", () => { expect( canSilentlyForceRetry( { kind: "in-progress", message: "busy" }, - { force: false, checkedClean: true }, + { force: false, inspectFinished: true }, ), ).toBe(false); }); diff --git a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.ts b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.ts index dc77e20b1cd..de982be0822 100644 --- a/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.ts +++ b/apps/desktop/src/renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/components/DashboardSidebarDeleteDialog/utils/canSilentlyForceRetry.ts @@ -3,7 +3,7 @@ import type { DestroyWorkspaceError } from "renderer/hooks/host-service/useDestr /** A dirty worktree the dialog never checked has had no warning shown. */ export function canSilentlyForceRetry( error: DestroyWorkspaceError, - { force, checkedClean }: { force: boolean; checkedClean: boolean }, + { force, inspectFinished }: { force: boolean; inspectFinished: boolean }, ): boolean { - return error.kind === "conflict" && !force && checkedClean; + return error.kind === "conflict" && !force && inspectFinished; } diff --git a/packages/host-service/src/projects/project-deletion.test.ts b/packages/host-service/src/projects/project-deletion.test.ts index 2973d3dab00..ff8208fbcd1 100644 --- a/packages/host-service/src/projects/project-deletion.test.ts +++ b/packages/host-service/src/projects/project-deletion.test.ts @@ -242,7 +242,7 @@ describe("interrupted-delete reconciler", () => { test("never deletes uncommitted files", async () => { const { ctx, addWorkspace, workspace } = setup(); const dirty = strand(addWorkspace, "dirty"); - cleanupGitOps.readWorktreeState = async () => + cleanupGitOps.readWorktreeStateUnattended = async () => ({ hasChanges: true }) as never; await runArchivedWorkspaceReconcile(ctx); expect(existsSync(dirty)).toBe(true); @@ -252,7 +252,7 @@ describe("interrupted-delete reconciler", () => { test("finishes a clean interrupted delete", async () => { const { ctx, addWorkspace, workspace } = setup(); const stranded = strand(addWorkspace, "stranded"); - cleanupGitOps.readWorktreeState = async () => + cleanupGitOps.readWorktreeStateUnattended = async () => ({ hasChanges: false }) as never; await runArchivedWorkspaceReconcile(ctx); expect(existsSync(stranded)).toBe(false); @@ -269,7 +269,7 @@ describe("interrupted-delete reconciler", () => { archivedAt: Date.now() + 1_000, archiveReason: "deleted", }); - cleanupGitOps.readWorktreeState = async () => + cleanupGitOps.readWorktreeStateUnattended = async () => ({ hasChanges: false }) as never; await runArchivedWorkspaceReconcile(ctx); expect(gitCalls).toEqual([]); diff --git a/packages/host-service/src/runtime/archived-workspace-reconcile.ts b/packages/host-service/src/runtime/archived-workspace-reconcile.ts index e8855bf40b8..2d44754072d 100644 --- a/packages/host-service/src/runtime/archived-workspace-reconcile.ts +++ b/packages/host-service/src/runtime/archived-workspace-reconcile.ts @@ -72,6 +72,7 @@ export async function runArchivedWorkspaceReconcile( // uncommitted files block the delete and un-archive the row. force: false, teardownMode: "best-effort", + longPreflight: true, }); resumed++; } catch (err) { diff --git a/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts b/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts index 29b7bdbc771..832743e2c49 100644 --- a/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts +++ b/packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts @@ -42,24 +42,24 @@ export const DESTROY_WORKTREE_STATE_TIMEOUT_MS = 10 * 60_000; /** A timeout kills git mid-delete, so this only stops a hung removal. */ export const WORKTREE_REMOVE_TIMEOUT_MS = 30 * 60_000; -/** Long removals must not take every shared worker slot from status and diff reads. */ -export const MAX_CONCURRENT_WORKTREE_REMOVALS = 2; +/** Long delete tasks must not take every shared worker slot from status and diff reads. */ +export const MAX_CONCURRENT_LONG_DELETE_TASKS = 2; -let activeWorktreeRemovals = 0; -const queuedWorktreeRemovals: Array<() => void> = []; +let activeLongDeleteTasks = 0; +const queuedLongDeleteTasks: Array<() => void> = []; -async function withWorktreeRemovalSlot(run: () => Promise): Promise { - if (activeWorktreeRemovals < MAX_CONCURRENT_WORKTREE_REMOVALS) { - activeWorktreeRemovals++; +async function withLongDeleteTaskSlot(run: () => Promise): Promise { + if (activeLongDeleteTasks < MAX_CONCURRENT_LONG_DELETE_TASKS) { + activeLongDeleteTasks++; } else { - await new Promise((resolve) => queuedWorktreeRemovals.push(resolve)); + await new Promise((resolve) => queuedLongDeleteTasks.push(resolve)); } try { return await run(); } finally { - const next = queuedWorktreeRemovals.shift(); + const next = queuedLongDeleteTasks.shift(); if (next) next(); - else activeWorktreeRemovals--; + else activeLongDeleteTasks--; } } @@ -84,13 +84,25 @@ export const cleanupGitOps = { return getHostWorkerPool().run(gitWorktreeStateTask, input, options); }, + /** For a caller that does not wait on the answer (no dialog, no relay). */ + readWorktreeStateUnattended(input: { + worktreePath: string; + gitEnv: GitTaskEnv; + }): Promise<{ hasChanges: boolean; hasUnpushedCommits: boolean }> { + return withLongDeleteTaskSlot(() => + getHostWorkerPool().run(gitWorktreeStateTask, input, { + timeoutMs: DESTROY_WORKTREE_STATE_TIMEOUT_MS, + }), + ); + }, + removeWorktree(input: { repoPath: string; worktreePath: string; gitEnv: GitTaskEnv; force?: boolean; }): Promise<{ stillRegistered: boolean; removeError?: string }> { - return withWorktreeRemovalSlot(() => + return withLongDeleteTaskSlot(() => getHostWorkerPool().run(gitWorktreeRemoveTask, input, { timeoutMs: WORKTREE_REMOVE_TIMEOUT_MS, }), diff --git a/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts b/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts index 1d7972c51d3..7f6e7033cdc 100644 --- a/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts +++ b/packages/host-service/src/trpc/router/workspace-cleanup/workspace-cleanup.ts @@ -16,6 +16,7 @@ import { import { disposeSessionsByWorkspaceId } from "../../../terminal/terminal"; import type { HostServiceContext } from "../../../types"; import type { GitTaskEnv } from "../../../workers/tasks/git"; +import { WorkerTaskAbortedError } from "../../../workers/WorkerTaskRunner"; import { archiveLocalWorkspace, trackWorkspaceDeleted, @@ -32,7 +33,6 @@ import { isInsideSessionsRoot } from "../workspace-creation/shared/session-paths import { isInsideProjectWorktreesRoot } from "../workspace-creation/shared/worktree-paths"; import { cleanupGitOps, - DESTROY_WORKTREE_STATE_TIMEOUT_MS, INSPECT_WORKTREE_STATE_TIMEOUT_MS, isIndeterminateGitTaskFailure, } from "./git-ops"; @@ -91,6 +91,12 @@ export interface DestroyWorkspaceInput { * - "skip": don't run — the interactive force-retry contract. */ teardownMode: "blocking" | "best-effort" | "skip"; + /** + * The caller does not wait on a relay or show an unchecked dialog, so the + * dirty check may take minutes. Released desktops silently force-retry any + * CONFLICT, so without this the check keeps the dialog's budget. + */ + longPreflight?: boolean; } /** @@ -171,6 +177,9 @@ export const workspaceCleanupRouter = router({ sharesProjectCheckout: false, }; } catch (err) { + if (err instanceof WorkerTaskAbortedError && err.kind === "cancelled") { + throw new TRPCError({ code: "CLIENT_CLOSED_REQUEST", cause: err }); + } if (isIndeterminateGitTaskFailure(err)) { throw new TRPCError({ code: "INTERNAL_SERVER_ERROR", @@ -244,6 +253,7 @@ export const workspaceCleanupRouter = router({ // anyway" must still run teardown, otherwise editing any // tracked file silently disables the user's cleanup script. skipTeardown: z.boolean().default(false), + longPreflight: z.boolean().default(false), }), ) .mutation(async ({ ctx, input }) => @@ -252,6 +262,7 @@ export const workspaceCleanupRouter = router({ deleteBranch: input.deleteBranch, force: input.force, teardownMode: input.skipTeardown ? "skip" : "blocking", + longPreflight: input.longPreflight, }), ), }); @@ -329,10 +340,15 @@ async function runDestroy( ctx, local.worktreePath, ); - const state = await cleanupGitOps.readWorktreeState( - { worktreePath: local.worktreePath, gitEnv }, - { timeoutMs: DESTROY_WORKTREE_STATE_TIMEOUT_MS }, - ); + const state = input.longPreflight + ? await cleanupGitOps.readWorktreeStateUnattended({ + worktreePath: local.worktreePath, + gitEnv, + }) + : await cleanupGitOps.readWorktreeState( + { worktreePath: local.worktreePath, gitEnv }, + { timeoutMs: INSPECT_WORKTREE_STATE_TIMEOUT_MS }, + ); if (state.hasChanges) { throw new TRPCError({ code: "CONFLICT", diff --git a/packages/host-service/src/workers/tasks/git.ts b/packages/host-service/src/workers/tasks/git.ts index d0718029836..ab704d11dbc 100644 --- a/packages/host-service/src/workers/tasks/git.ts +++ b/packages/host-service/src/workers/tasks/git.ts @@ -328,8 +328,7 @@ export const gitWorktreeStateTask = defineWorkerTask< type: "git/worktreeState", handler: async ({ worktreePath, gitEnv, ignoreInitialCommit }) => { const git = createUserSimpleGit(worktreePath, { env: gitEnv }); - // `git.status()` passes `-u` (all), which lists every file in every - // untracked directory. One entry per directory is enough to say "dirty". + // Not `git.status()`: its `-u` lists every file in untracked directories. const status = await git.raw([ "status", "--porcelain", diff --git a/packages/host-service/test/workspace-cleanup.test.ts b/packages/host-service/test/workspace-cleanup.test.ts index 66ab9c1649e..f82671b0a9d 100644 --- a/packages/host-service/test/workspace-cleanup.test.ts +++ b/packages/host-service/test/workspace-cleanup.test.ts @@ -16,7 +16,7 @@ import { join } from "node:path"; import { workspaceRouter } from "../src/trpc/router/workspace"; import { cleanupGitOps, - MAX_CONCURRENT_WORKTREE_REMOVALS, + MAX_CONCURRENT_LONG_DELETE_TASKS, } from "../src/trpc/router/workspace-cleanup/git-ops"; import { isLocalCheckoutWorkspace } from "../src/trpc/router/workspace-cleanup/is-local-checkout-workspace"; import { @@ -26,7 +26,10 @@ import { } from "../src/trpc/router/workspace-cleanup/workspace-cleanup"; import type { HostServiceContext } from "../src/types"; import { getHostWorkerPool } from "../src/workers/host-worker-pool"; -import { WorkerTaskError } from "../src/workers/WorkerTaskRunner"; +import { + WorkerTaskAbortedError, + WorkerTaskError, +} from "../src/workers/WorkerTaskRunner"; type WorkspaceRow = { id: string; @@ -72,6 +75,12 @@ interface ContextSpec { let gitOpsSpec: ContextSpec = {}; const realGitOps = { ...cleanupGitOps }; + +function readSpecWorktreeState(): Promise | WorktreeState { + const state = gitOpsSpec.worktreeState; + if (typeof state === "function") return state(); + return state ?? { hasChanges: false, hasUnpushedCommits: false }; +} const realTeardownOps = { ...cleanupTeardownOps }; afterAll(() => { Object.assign(cleanupGitOps, realGitOps); @@ -83,11 +92,8 @@ Object.assign(cleanupGitOps, { if (gitOpsSpec.resolveGitEnvThrows) throw new Error("git env boom"); return {}; }, - readWorktreeState: async () => { - const state = gitOpsSpec.worktreeState; - if (typeof state === "function") return state(); - return state ?? { hasChanges: false, hasUnpushedCommits: false }; - }, + readWorktreeState: async () => readSpecWorktreeState(), + readWorktreeStateUnattended: async () => readSpecWorktreeState(), removeWorktree: async () => gitOpsSpec.removeWorktree ? gitOpsSpec.removeWorktree() @@ -356,6 +362,20 @@ describe("workspaceCleanup.inspect", () => { ); }); + test("a cancelled worktree-state read reports the closed request, not a server error", async () => { + const ctx = makeCtx({ + ...wsAndProject, + worktreeState: () => + Promise.reject(new WorkerTaskAbortedError("cancelled")), + }); + const caller = workspaceCleanupRouter.createCaller(ctx); + const error = await caller + .inspect({ workspaceId: "ws-1" }) + .then(() => null) + .catch((err: { code?: string }) => err); + expect(error?.code).toBe("CLIENT_CLOSED_REQUEST"); + }); + test("swallows git env-resolution failures and returns canDelete: true with no warnings", async () => { const ctx = makeCtx({ ...wsAndProject, @@ -1063,12 +1083,12 @@ describe("workspaceCleanup.destroy cleanup ordering", () => { ).rejects.toThrow(/Couldn't verify worktree state/); }); - test("a preflight that needs minutes deletes instead of restoring the row", async () => { - // `git status` that needs 5 min, as on a large worktree on a cold or - // busy disk: the pool rejects any shorter budget, as it does in the field. + async function destroyWithSlowStatus(longPreflight: boolean) { + // A `git status` that needs 5 min, as on a large worktree on a cold or + // busy disk: the pool rejects any shorter budget. const pool = getHostWorkerPool(); const realRun = pool.run; - const patchedReadWorktreeState = cleanupGitOps.readWorktreeState; + const patched = { ...cleanupGitOps }; pool.run = (async (_def, _input, options) => { const timeoutMs = options?.timeoutMs ?? 0; if (timeoutMs < 5 * 60_000) { @@ -1079,6 +1099,8 @@ describe("workspaceCleanup.destroy cleanup ordering", () => { return { hasChanges: false, hasUnpushedCommits: false }; }) as typeof pool.run; cleanupGitOps.readWorktreeState = realGitOps.readWorktreeState; + cleanupGitOps.readWorktreeStateUnattended = + realGitOps.readWorktreeStateUnattended; const worktree = mkdtempSync(join(tmpdir(), "workspace-delete-")); const repo = mkdtempSync(join(tmpdir(), "workspace-delete-repo-")); try { @@ -1091,26 +1113,46 @@ describe("workspaceCleanup.destroy cleanup ordering", () => { }, project: { id: "p-1", repoPath: repo }, }); - const caller = workspaceCleanupRouter.createCaller(ctx); - - const result = await caller.destroy({ - workspaceId: "ws-1", - deleteBranch: false, - force: false, - }); - - expect(result.success).toBe(true); - // One broadcast is the archive; an un-archive would add a second. - expect(ctx.__mocks.broadcastWorkspaceChanged).toHaveBeenCalledTimes(1); + const outcome = await workspaceCleanupRouter + .createCaller(ctx) + .destroy({ + workspaceId: "ws-1", + deleteBranch: false, + force: false, + longPreflight, + }) + .then( + (result) => ({ result, error: null }), + (error: Error) => ({ result: null, error }), + ); + return { + ...outcome, + broadcasts: ctx.__mocks.broadcastWorkspaceChanged.mock.calls.length, + }; } finally { pool.run = realRun; - cleanupGitOps.readWorktreeState = patchedReadWorktreeState; + Object.assign(cleanupGitOps, patched); rmSync(worktree, { recursive: true, force: true }); rmSync(repo, { recursive: true, force: true }); } + } + + test("a long preflight that needs minutes deletes instead of restoring the row", async () => { + const { result, broadcasts } = await destroyWithSlowStatus(true); + + expect(result?.success).toBe(true); + // One broadcast is the archive; an un-archive would add a second. + expect(broadcasts).toBe(1); + }); + + test("without longPreflight the preflight keeps the dialog's budget and fails closed", async () => { + const { error, broadcasts } = await destroyWithSlowStatus(false); + + expect(error?.message).toMatch(/Couldn't verify worktree state/); + expect(broadcasts).toBe(2); }); - test("worktree removals beyond the limit wait for a slot instead of taking another worker", async () => { + test("long delete tasks beyond the limit wait for a slot instead of taking another worker", async () => { const pool = getHostWorkerPool(); const realRun = pool.run; const finishers: Array<() => void> = []; @@ -1120,15 +1162,21 @@ describe("workspaceCleanup.destroy cleanup ordering", () => { )) as typeof pool.run; const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); try { - const removals = [1, 2, 3].map((n) => - realGitOps.removeWorktree({ - repoPath: "/repo", - worktreePath: `/wt-${n}`, + const removals = [ + ...[1, 2].map((n) => + realGitOps.removeWorktree({ + repoPath: "/repo", + worktreePath: `/wt-${n}`, + gitEnv: {}, + }), + ), + realGitOps.readWorktreeStateUnattended({ + worktreePath: "/wt-3", gitEnv: {}, }), - ); + ]; await flush(); - expect(finishers).toHaveLength(MAX_CONCURRENT_WORKTREE_REMOVALS); + expect(finishers).toHaveLength(MAX_CONCURRENT_LONG_DELETE_TASKS); finishers[0]?.(); await flush(); @@ -1137,6 +1185,7 @@ describe("workspaceCleanup.destroy cleanup ordering", () => { for (const finish of finishers.slice(1)) finish(); await Promise.all(removals); } finally { + for (const finish of finishers) finish(); pool.run = realRun; } });