Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ export interface DestroyWorkspaceInput {
/** Consent to abandon the teardown script — only the teardown-failed
* retry sets this. */
skipTeardown?: boolean;
/** Only for this machine's host: the relay gives up on a request after
* about a minute, and a long dirty check can outlast that. */
longPreflight?: boolean;
}

export interface DestroyWorkspaceSuccess {
Expand Down Expand Up @@ -84,6 +87,7 @@ export async function destroyWorkspaceAtHost(
deleteBranch: input.deleteBranch ?? false,
force: input.force ?? false,
skipTeardown: input.skipTeardown ?? false,
longPreflight: input.longPreflight ?? false,
});
} catch (error) {
throw normalizeDestroyWorkspaceError(error);
Expand Down Expand Up @@ -163,10 +167,10 @@ export function useDestroyWorkspace(workspaceId: string): UseDestroyWorkspace {
}
return destroyWorkspaceAtHost(
{ workspaceId, hostUrl, hostStatus },
input,
{ ...input, longPreflight: hostUrl === activeHostUrl },
);
},
[hostUrl, hostStatus, isSandbox, utils, workspaceId],
[activeHostUrl, hostUrl, hostStatus, isSandbox, utils, workspaceId],
);

const inspect = useCallback(async (): Promise<DestroyWorkspacePreview> => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import type { DashboardSidebarWorkspace } from "renderer/routes/_authenticated/_
import { useDeletingWorkspacesStore } from "renderer/routes/_authenticated/_dashboard/stores/deletingWorkspacesStore";
import { useDashboardSidebarState } from "renderer/routes/_authenticated/hooks/useDashboardSidebarState";
import { useHostWorkspaces } from "renderer/routes/_authenticated/providers/HostWorkspacesProvider";
import { useLocalHostService } from "renderer/routes/_authenticated/providers/LocalHostServiceProvider";
import {
type BulkWorkspaceInspectionState,
buildBulkWorkspaceInspectionSummary,
Expand Down Expand Up @@ -75,6 +76,7 @@ export function useBulkWorkspaceDelete({
}: UseBulkWorkspaceDeleteOptions) {
const { t } = useLingui();
const { cache: hostWorkspacesCache } = useHostWorkspaces();
const { activeHostUrl } = useLocalHostService();
const { navigateAwayFromWorkspace } = useNavigateAwayFromWorkspace();
const { removeWorkspaceFromSidebar } = useDashboardSidebarState();
const { preferences, setDeleteLocalBranch } = useV2UserPreferences();
Expand Down Expand Up @@ -217,12 +219,15 @@ export function useBulkWorkspaceDelete({
shouldRetryWithForce: (workspace, error) =>
error.kind === "conflict" &&
inspections.get(workspace.id)?.status !== "ready",
destroy: (workspace, force) =>
destroyWorkspaceAtHost(targetFor(workspace), {
destroy: (workspace, force) => {
const target = targetFor(workspace);
return destroyWorkspaceAtHost(target, {
deleteBranch: preferences.deleteLocalBranch,
force,
skipTeardown,
}),
longPreflight: target.hostUrl === activeHostUrl,
});
},
onSettled: () => {
completedCount += 1;
if (completedCount < total) {
Expand Down Expand Up @@ -323,6 +328,7 @@ export function useBulkWorkspaceDelete({
}
},
[
activeHostUrl,
hostWorkspacesCache,
inspections,
navigateAwayFromWorkspace,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { useV2UserPreferences } from "renderer/hooks/useV2UserPreferences/useV2U
import { useNavigateAwayFromWorkspace } from "renderer/routes/_authenticated/_dashboard/components/DashboardSidebar/hooks/useNavigateAwayFromWorkspace";
import { useDeletingWorkspacesStore } from "renderer/routes/_authenticated/_dashboard/stores/deletingWorkspacesStore";
import { useHostWorkspaces } from "renderer/routes/_authenticated/providers/HostWorkspacesProvider";
import { canSilentlyForceRetry } from "../../utils/canSilentlyForceRetry";

interface UseDestroyDialogStateOptions {
workspaceId: string;
Expand Down Expand Up @@ -110,6 +111,7 @@ export function useDestroyDialogState({
}) => {
if (inFlight.current) return;
inFlight.current = true;
const inspectFinished = inspectState.status === "ready";

setError(null);
onOpenChange(false);
Expand Down Expand Up @@ -139,7 +141,7 @@ export function useDestroyDialogState({
// retry. Do NOT extend this to `in-progress` (that's a
// different CONFLICT cause; retrying just races the same
// guard).
if (e.kind === "conflict" && !force) {
if (canSilentlyForceRetry(e, { force, inspectFinished })) {
result = await destroy({ deleteBranch, force: true, skipTeardown });
} else {
throw firstErr;
Expand All @@ -155,7 +157,7 @@ export function useDestroyDialogState({
onDeleted?.();
} catch (err) {
const e = err as DestroyWorkspaceError;
if (e.kind === "teardown-failed") {
if (e.kind === "teardown-failed" || e.kind === "conflict") {
setError(e);
onOpenChange(true);
} else if (e.kind === "in-progress") {
Expand All @@ -179,6 +181,7 @@ export function useDestroyDialogState({
[
destroy,
deleteBranch,
inspectState.status,
workspaceName,
workspaceId,
onOpenChange,
Expand All @@ -193,7 +196,7 @@ export function useDestroyDialogState({
return {
deleteBranch,
setDeleteBranch,
hasChanges: preview?.hasChanges ?? false,
hasChanges: preview?.hasChanges === true || error?.kind === "conflict",
hasUnpushedCommits: preview?.hasUnpushedCommits ?? false,
sharesProjectCheckout:
preview?.canDelete === true && preview.sharesProjectCheckout === true,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import { describe, expect, test } from "bun:test";
import { canSilentlyForceRetry } from "./canSilentlyForceRetry";

const conflict = { kind: "conflict", message: "dirty" } as const;

describe("canSilentlyForceRetry", () => {
test("retries a conflict after the dialog finished checking the worktree", () => {
expect(
canSilentlyForceRetry(conflict, { force: false, inspectFinished: true }),
).toBe(true);
});

test("does not retry a conflict when the check never finished", () => {
expect(
canSilentlyForceRetry(conflict, { force: false, inspectFinished: false }),
).toBe(false);
});

test("does not retry other failures", () => {
expect(
canSilentlyForceRetry(
{ kind: "in-progress", message: "busy" },
{ force: false, inspectFinished: true },
),
).toBe(false);
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import type { DestroyWorkspaceError } from "renderer/hooks/host-service/useDestroyWorkspace";

/** A dirty worktree the dialog never checked has had no warning shown. */
export function canSilentlyForceRetry(
error: DestroyWorkspaceError,
{ force, inspectFinished }: { force: boolean; inspectFinished: boolean },
): boolean {
return error.kind === "conflict" && !force && inspectFinished;
}
22 changes: 19 additions & 3 deletions packages/host-service/src/projects/project-deletion.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ describe("interrupted-delete reconciler", () => {
test("never deletes uncommitted files", async () => {
const { ctx, addWorkspace, workspace } = setup();
const dirty = strand(addWorkspace, "dirty");
cleanupGitOps.readWorktreeState = async () =>
cleanupGitOps.readWorktreeStateUnattended = async () =>
({ hasChanges: true }) as never;
await runArchivedWorkspaceReconcile(ctx);
expect(existsSync(dirty)).toBe(true);
Expand All @@ -252,7 +252,7 @@ describe("interrupted-delete reconciler", () => {
test("finishes a clean interrupted delete", async () => {
const { ctx, addWorkspace, workspace } = setup();
const stranded = strand(addWorkspace, "stranded");
cleanupGitOps.readWorktreeState = async () =>
cleanupGitOps.readWorktreeStateUnattended = async () =>
({ hasChanges: false }) as never;
await runArchivedWorkspaceReconcile(ctx);
expect(existsSync(stranded)).toBe(false);
Expand All @@ -269,7 +269,7 @@ describe("interrupted-delete reconciler", () => {
archivedAt: Date.now() + 1_000,
archiveReason: "deleted",
});
cleanupGitOps.readWorktreeState = async () =>
cleanupGitOps.readWorktreeStateUnattended = async () =>
({ hasChanges: false }) as never;
await runArchivedWorkspaceReconcile(ctx);
expect(gitCalls).toEqual([]);
Expand Down Expand Up @@ -333,6 +333,22 @@ describe("delete permanently", () => {
expect(existsSync(repoPath)).toBe(true);
});

test("a worktree whose removal fails does not stop the others", async () => {
const { ctx, addWorkspace, project } = setup();
const stuck = addWorkspace("stuck");
const other = addWorkspace("other");
const removeAndRecord = cleanupGitOps.removeWorktree;
cleanupGitOps.removeWorktree = async (input) => {
if (input.worktreePath === stuck) throw new Error("timed out");
return removeAndRecord(input);
};
await softDeleteProject(ctx, PROJECT_ID);
expect(await purgeDeletedProject(ctx, PROJECT_ID)).toBe(true);
expect(project()).toBeUndefined();
expect(existsSync(stuck)).toBe(true);
expect(existsSync(other)).toBe(false);
});

test("refuses a project that was never deleted", async () => {
const { ctx, addWorkspace, gitCalls, project } = setup();
addWorkspace("live");
Expand Down
42 changes: 26 additions & 16 deletions packages/host-service/src/projects/project-deletion.ts
Original file line number Diff line number Diff line change
Expand Up @@ -243,22 +243,32 @@ async function purgeProject(
if (worktrees.length > 0) {
try {
const gitEnv = await cleanupGitOps.resolveGitEnv(ctx, project.repoPath);
for (const row of worktrees) {
const { stillRegistered, removeError } =
await cleanupGitOps.removeWorktree({
repoPath: project.repoPath,
worktreePath: row.worktreePath,
gitEnv,
force: false,
});
if (stillRegistered) {
console.warn("[project-deletion] left worktree on disk", {
projectId: project.id,
worktreePath: row.worktreePath,
removeError,
});
}
}
await Promise.all(
worktrees.map(async (row) => {
try {
const { stillRegistered, removeError } =
await cleanupGitOps.removeWorktree({
repoPath: project.repoPath,
worktreePath: row.worktreePath,
gitEnv,
force: false,
});
if (stillRegistered) {
console.warn("[project-deletion] left worktree on disk", {
projectId: project.id,
worktreePath: row.worktreePath,
removeError,
});
}
} catch (err) {
console.warn("[project-deletion] left worktree on disk", {
projectId: project.id,
worktreePath: row.worktreePath,
err,
});
}
}),
);
} catch (err) {
console.warn("[project-deletion] left worktrees on disk", {
projectId: project.id,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ export async function runArchivedWorkspaceReconcile(
// uncommitted files block the delete and un-archive the row.
force: false,
teardownMode: "best-effort",
longPreflight: true,
});
resumed++;
} catch (err) {
Expand Down
60 changes: 49 additions & 11 deletions packages/host-service/src/trpc/router/workspace-cleanup/git-ops.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,36 @@ export function isIndeterminateGitTaskFailure(err: unknown): boolean {
return err instanceof WorkerTaskError && err.name === "WorkerTaskError";
}

/** The delete dialog waits on this read. */
export const INSPECT_WORKTREE_STATE_TIMEOUT_MS = 15_000;

/** Runs after the row is archived, so nobody waits on it. */
export const DESTROY_WORKTREE_STATE_TIMEOUT_MS = 10 * 60_000;

/** A timeout kills git mid-delete, so this only stops a hung removal. */
export const WORKTREE_REMOVE_TIMEOUT_MS = 30 * 60_000;

/** Long delete tasks must not take every shared worker slot from status and diff reads. */
export const MAX_CONCURRENT_LONG_DELETE_TASKS = 2;

let activeLongDeleteTasks = 0;
const queuedLongDeleteTasks: Array<() => void> = [];

async function withLongDeleteTaskSlot<T>(run: () => Promise<T>): Promise<T> {
if (activeLongDeleteTasks < MAX_CONCURRENT_LONG_DELETE_TASKS) {
activeLongDeleteTasks++;
} else {
await new Promise<void>((resolve) => queuedLongDeleteTasks.push(resolve));
}
try {
return await run();
} finally {
const next = queuedLongDeleteTasks.shift();
if (next) next();
else activeLongDeleteTasks--;
}
}

export const cleanupGitOps = {
/** Same failure surface as `ctx.git(repoPath)` — a throw here maps to the
* saga's "failed to open project repo" handling. */
Expand All @@ -49,12 +79,21 @@ export const cleanupGitOps = {
gitEnv: GitTaskEnv;
ignoreInitialCommit?: boolean;
},
signal?: AbortSignal,
options: { timeoutMs: number; signal?: AbortSignal },
): Promise<{ hasChanges: boolean; hasUnpushedCommits: boolean }> {
return getHostWorkerPool().run(gitWorktreeStateTask, input, {
timeoutMs: 15_000,
signal,
});
return getHostWorkerPool().run(gitWorktreeStateTask, input, options);
},

/** For a caller that does not wait on the answer (no dialog, no relay). */
readWorktreeStateUnattended(input: {
worktreePath: string;
gitEnv: GitTaskEnv;
}): Promise<{ hasChanges: boolean; hasUnpushedCommits: boolean }> {
return withLongDeleteTaskSlot(() =>
getHostWorkerPool().run(gitWorktreeStateTask, input, {
timeoutMs: DESTROY_WORKTREE_STATE_TIMEOUT_MS,
}),
);
},

removeWorktree(input: {
Expand All @@ -63,12 +102,11 @@ export const cleanupGitOps = {
gitEnv: GitTaskEnv;
force?: boolean;
}): Promise<{ stillRegistered: boolean; removeError?: string }> {
// Generous timeout: removal recursively deletes the worktree
// directory, which can take a while for large trees (node_modules
// etc.).
return getHostWorkerPool().run(gitWorktreeRemoveTask, input, {
timeoutMs: 120_000,
});
return withLongDeleteTaskSlot(() =>
getHostWorkerPool().run(gitWorktreeRemoveTask, input, {
timeoutMs: WORKTREE_REMOVE_TIMEOUT_MS,
}),
);
},

deleteLocalBranch(input: {
Expand Down
Loading
Loading