From 94f3b9033f06a75914c724569b4402da20e6d194 Mon Sep 17 00:00:00 2001 From: Phil J Date: Fri, 11 Sep 2026 09:12:00 +0200 Subject: [PATCH 1/2] fix(vcs): deadline + backoff self-heal for checkpoint capture timeouts `git add -A` during checkpoint capture ran on the generic 30s VcsProcess deadline. In large repositories (many worktrees, submodules, thousands of files) it routinely exceeds that under disk pressure - most of all while an auto-gc repacks the shared object store - so every turn surfaced a user-visible "Checkpoint capture failed" activity. - give the checkpoint index step (whole-worktree add / ls-files) its own 120s deadline instead of the generic 30s default - on a VcsProcessTimeoutError, re-run the whole capture from a fresh temp index (idempotent by design: temp index seeded from HEAD, update-ref is deterministic) with the same backoff-ladder pattern as the interrupted-turn re-drive: 5s, then 30s, then surface the failure - regression test (it.live, real clock): first add times out, the capture succeeds via the retry, and both index steps carry the dedicated deadline --- apps/server/src/vcs/GitVcsDriver.ts | 184 ++++++++++++------ ...t3team-GitVcsDriverCheckpointIndex.test.ts | 106 ++++++++++ .../vcs/t3team-GitVcsDriverCheckpointIndex.ts | 14 +- 3 files changed, 240 insertions(+), 64 deletions(-) diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index 4f50b7713a..f4a3908438 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -2,16 +2,20 @@ import * as NodeCrypto from "node:crypto"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; import { ChildProcessSpawner } from "effect/unstable/process"; import { GitCommandError, + type VcsError, VcsProcessExitError, + VcsProcessTimeoutError, type VcsSwitchRefInput, type VcsSwitchRefResult, type VcsCreateRefInput, @@ -712,89 +716,145 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( return path.isAbsolute(gitCommonDir) ? gitCommonDir : path.resolve(cwd, gitCommonDir); }); + /** + * `git add -A` re-indexes the whole worktree. In large repositories (many + * worktrees, submodules, thousands of files) it routinely exceeds the + * generic 30s VcsProcess default under disk pressure — and it can stall + * further behind an auto-gc repacking the shared object store. The index + * step gets its own budget; a timeout that still fires is self-healed by + * re-running the whole capture on a fresh temp index with the same backoff + * ladder pattern as the interrupted-turn re-drive + * (t3team-workflowEngineTurnRetrySupport.ts). + */ + const CHECKPOINT_INDEX_TIMEOUT_MS = 120_000; + /** Backoff ladder (ms) before capture attempts 2 and 3. */ + const CHECKPOINT_TIMEOUT_RETRY_BACKOFF_MS = [5_000, 30_000] as const; + /** Total capture attempts before a timed-out capture surfaces to the caller. */ + const CHECKPOINT_MAX_CAPTURE_ATTEMPTS = CHECKPOINT_TIMEOUT_RETRY_BACKOFF_MS.length + 1; + const isVcsProcessTimeoutError = Schema.is(VcsProcessTimeoutError); + const checkpoints: VcsDriver.VcsCheckpointOps = { captureCheckpoint: Effect.fn("GitVcsDriver.checkpoints.captureCheckpoint")(function* (input) { const operation = "GitVcsDriver.checkpoints.captureCheckpoint"; const gitCommonDir = yield* resolveGitCommonDir(input.cwd); - const tempIndexPath = path.join( - gitCommonDir, - `t3-checkpoint-index-${NodeCrypto.randomUUID()}`, - ); - const commitEnv: NodeJS.ProcessEnv = { - ...process.env, - GIT_INDEX_FILE: tempIndexPath, - GIT_AUTHOR_NAME: "T3 Code", - GIT_AUTHOR_EMAIL: "t3code@users.noreply.github.com", - GIT_COMMITTER_NAME: "T3 Code", - GIT_COMMITTER_EMAIL: "t3code@users.noreply.github.com", - }; - const cleanupTempIndex = fileSystem - .remove(tempIndexPath, { force: true }) - .pipe(Effect.ignore); + // One attempt is fully isolated: a fresh temp index file, seeded from + // HEAD, and an idempotent update-ref — so retrying a timed-out attempt + // from scratch is always safe. + const attempt = Effect.gen(function* () { + const tempIndexPath = path.join( + gitCommonDir, + `t3-checkpoint-index-${NodeCrypto.randomUUID()}`, + ); + const commitEnv: NodeJS.ProcessEnv = { + ...process.env, + GIT_INDEX_FILE: tempIndexPath, + GIT_AUTHOR_NAME: "T3 Code", + GIT_AUTHOR_EMAIL: "t3code@users.noreply.github.com", + GIT_COMMITTER_NAME: "T3 Code", + GIT_COMMITTER_EMAIL: "t3code@users.noreply.github.com", + }; + + const cleanupTempIndex = fileSystem + .remove(tempIndexPath, { force: true }) + .pipe(Effect.ignore); + + yield* Effect.gen(function* () { + const headExists = yield* hasHeadCommit(input.cwd); + if (headExists) { + yield* execute({ + operation, + cwd: input.cwd, + args: ["read-tree", "HEAD"], + env: commitEnv, + }); + } - yield* Effect.gen(function* () { - const headExists = yield* hasHeadCommit(input.cwd); - if (headExists) { - yield* execute({ + yield* indexCheckpointPaths({ operation, cwd: input.cwd, - args: ["read-tree", "HEAD"], + gitCommonDir, env: commitEnv, + execute, + fileSystem, + path, + timeoutMs: CHECKPOINT_INDEX_TIMEOUT_MS, }); - } - yield* indexCheckpointPaths({ - operation, - cwd: input.cwd, - gitCommonDir, - env: commitEnv, - execute, - fileSystem, - path, - }); + const writeTreeResult = yield* execute({ + operation, + cwd: input.cwd, + args: ["write-tree"], + env: commitEnv, + }); + const treeOid = writeTreeResult.stdout.trim(); + if (treeOid.length === 0) { + return yield* new VcsProcessExitError({ + operation, + command: "git write-tree", + cwd: input.cwd, + exitCode: 0, + detail: "git write-tree returned an empty tree oid.", + }); + } - const writeTreeResult = yield* execute({ - operation, - cwd: input.cwd, - args: ["write-tree"], - env: commitEnv, - }); - const treeOid = writeTreeResult.stdout.trim(); - if (treeOid.length === 0) { - return yield* new VcsProcessExitError({ + const message = `t3 checkpoint ref=${input.checkpointRef}`; + const commitTreeResult = yield* execute({ operation, - command: "git write-tree", cwd: input.cwd, - exitCode: 0, - detail: "git write-tree returned an empty tree oid.", + args: ["commit-tree", treeOid, "-m", message], + env: commitEnv, }); - } + const commitOid = commitTreeResult.stdout.trim(); + if (commitOid.length === 0) { + return yield* new VcsProcessExitError({ + operation, + command: "git commit-tree", + cwd: input.cwd, + exitCode: 0, + detail: "git commit-tree returned an empty commit oid.", + }); + } - const message = `t3 checkpoint ref=${input.checkpointRef}`; - const commitTreeResult = yield* execute({ - operation, - cwd: input.cwd, - args: ["commit-tree", treeOid, "-m", message], - env: commitEnv, - }); - const commitOid = commitTreeResult.stdout.trim(); - if (commitOid.length === 0) { - return yield* new VcsProcessExitError({ + yield* execute({ operation, - command: "git commit-tree", cwd: input.cwd, - exitCode: 0, - detail: "git commit-tree returned an empty commit oid.", + args: ["update-ref", input.checkpointRef, commitOid], }); + }).pipe(Effect.ensuring(cleanupTempIndex)); + }); + + // Bounded self-heal: on a timed-out attempt, wait down the backoff + // ladder and capture again from scratch. Any other failure — and the + // final timed-out attempt — propagates unchanged. + const captureWithTimeoutSelfHeal: ( + attemptNumber: number, + ) => Effect.Effect = Effect.fn( + "GitVcsDriver.checkpoints.captureCheckpoint.timeoutSelfHeal", + )(function* (attemptNumber: number) { + if (attemptNumber >= CHECKPOINT_MAX_CAPTURE_ATTEMPTS) { + return yield* attempt; } + const backoffMs = CHECKPOINT_TIMEOUT_RETRY_BACKOFF_MS[attemptNumber - 1]!; + return yield* attempt.pipe( + Effect.catchIf( + (error) => isVcsProcessTimeoutError(error), + (error) => + Effect.gen(function* () { + yield* Effect.logWarning("checkpoint.capture.timeout-retry", { + detail: `checkpoint capture timed out in ${input.cwd}; retrying in ${backoffMs}ms (attempt ${attemptNumber} of ${CHECKPOINT_MAX_CAPTURE_ATTEMPTS})`, + cwd: input.cwd, + attemptNumber, + cause: error.message, + }); + yield* Effect.sleep(Duration.millis(backoffMs)); + return yield* captureWithTimeoutSelfHeal(attemptNumber + 1); + }), + ), + ); + }); - yield* execute({ - operation, - cwd: input.cwd, - args: ["update-ref", input.checkpointRef, commitOid], - }); - }).pipe(Effect.ensuring(cleanupTempIndex)); + return yield* captureWithTimeoutSelfHeal(1); }), hasCheckpointRef: (input) => diff --git a/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.test.ts b/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.test.ts index 4aa0b42499..9b0937dbf1 100644 --- a/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.test.ts +++ b/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.test.ts @@ -17,6 +17,7 @@ import * as VcsDriver from "./VcsDriver.ts"; import { isUnindexableHostPath } from "./t3team-GitVcsDriverCheckpointIndex.ts"; import * as VcsProcess from "./VcsProcess.ts"; import * as ServerConfig from "../config.ts"; +import { VcsProcessExitError, VcsProcessTimeoutError } from "@t3tools/contracts"; describe("isUnindexableHostPath", () => { it("classifies Windows-reserved device names case-insensitively", () => { @@ -253,3 +254,108 @@ it.effect( ); }, ); + +// Live incident: in a large repo the whole-worktree `git add` outlives the +// generic 30s VcsProcess deadline and the capture failed with a user-visible +// "Checkpoint capture failed" activity on every turn. The index step now gets +// its own deadline, and a timed-out capture attempt is retried from scratch +// (fresh temp index, idempotent update-ref) instead of surfacing the error. +const addTimeoutObservations: Array<{ readonly attempt: number; readonly timeoutMs: number }> = []; + +const DriverLayerAddTimeout = Layer.mergeAll(GitVcsDriver.vcsLayer, GitVcsDriver.layer).pipe( + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-git-ckpt-addto-" })), + Layer.provideMerge( + Layer.succeed(VcsProcess.VcsProcess, { + run: (input) => { + const args = input.args.slice(input.args[0] === "-C" ? 2 : 0); + if (args[0] === "rev-parse" && args[1] === "--git-common-dir") { + return Effect.succeed(ok(".git\n")); + } + if (args[0] === "rev-parse" && args[1] === "--verify") { + return Effect.succeed(ok("head-oid\n")); + } + if (args[0] === "add" && args.includes("-A") && args.includes("--") && args.includes(".")) { + const attempt = addTimeoutObservations.length + 1; + addTimeoutObservations.push({ attempt, timeoutMs: input.timeoutMs ?? 0 }); + // First attempt times out; the retry succeeds. + return attempt === 1 + ? Effect.fail( + new VcsProcessTimeoutError({ + operation: "GitVcsDriver.checkpoints.captureCheckpoint", + command: "git", + cwd: input.cwd, + timeoutMs: input.timeoutMs ?? 30_000, + }), + ) + : Effect.succeed(ok()); + } + if (args[0] === "read-tree") { + return Effect.succeed(ok()); + } + if (args[0] === "write-tree") { + return Effect.succeed(ok("tree-oid\n")); + } + if (args[0] === "commit-tree") { + return Effect.succeed(ok("commit-oid\n")); + } + if (args[0] === "update-ref") { + return Effect.succeed(ok()); + } + return Effect.fail( + new VcsProcessExitError({ + operation: "GitVcsDriver.checkpoints.captureCheckpoint", + command: "git", + cwd: input.cwd, + exitCode: 128, + detail: `unexpected git args: ${args.join(" ")}`, + }), + ); + }, + }), + ), + Layer.provideMerge(NodeServices.layer), +); + +// `it.live` (real clock) so the 5s backoff between capture attempts actually +// elapses; under the default TestClock the sleep would never tick. +it.live( + "self-heals a timed-out whole-worktree git add by retrying the capture with a fresh temp index", + () => { + const repoDir = NodePath.join(NodeOS.tmpdir(), `t3-ckpt-addto-test-${Date.now()}`); + + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* VcsDriver.VcsDriver; + + yield* fileSystem.makeDirectory(path.join(repoDir, ".git"), { recursive: true }); + + assert.isNotNull(driver.checkpoints); + // Would fail without the bounded retry: the first add times out, the + // capture is re-attempted from a fresh temp index and succeeds (after + // the 5s first ladder step of CHECKPOINT_TIMEOUT_RETRY_BACKOFF_MS). + yield* driver.checkpoints!.captureCheckpoint({ + cwd: repoDir, + checkpointRef: "refs/t3/checkpoints/addto-1" as never, + }); + + // Exactly one retry: the timed-out attempt plus the successful one. + assert.strictEqual(addTimeoutObservations.length, 2); + // Both index steps ran with the dedicated checkpoint deadline, not the + // generic 30s VcsProcess default. + assert.ok(addTimeoutObservations.every((obs) => obs.timeoutMs > 30_000)); + }).pipe( + Effect.provide(DriverLayerAddTimeout), + Effect.ensuring( + Effect.sync(() => { + addTimeoutObservations.length = 0; + try { + NodeFS.rmSync(repoDir, { recursive: true, force: true }); + } catch { + // best-effort cleanup + } + }), + ), + ); + }, +); diff --git a/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.ts b/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.ts index 1c4d85ca9a..9b70b4f8a4 100644 --- a/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.ts +++ b/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.ts @@ -40,6 +40,7 @@ type ExecuteGit = (input: { readonly args: readonly string[]; readonly env: NodeJS.ProcessEnv; readonly allowNonZeroExit?: boolean; + readonly timeoutMs?: number; }) => Effect.Effect< { readonly exitCode: number | null; readonly stdout: string; readonly stderr: string }, VcsError @@ -97,11 +98,20 @@ export const indexCheckpointPaths = (deps: { readonly execute: ExecuteGit; readonly fileSystem: FileSystem.FileSystem; readonly path: Path.Path; + /** Deadline for the whole-worktree `git add` / `ls-files` enumeration steps. */ + readonly timeoutMs?: number; }): Effect.Effect => Effect.gen(function* () { - const { operation, cwd, gitCommonDir, env, execute, fileSystem, path } = deps; + const { operation, cwd, gitCommonDir, env, execute, fileSystem, path, timeoutMs } = deps; const run = (args: readonly string[]) => - execute({ operation, cwd, args, env, allowNonZeroExit: true }); + execute({ + operation, + cwd, + args, + env, + allowNonZeroExit: true, + ...(timeoutMs !== undefined ? { timeoutMs } : {}), + }); // Fast path: index everything. const broadAdd = yield* run(["add", "-A", "--", "."]); From 0bb9e0a7018fe20a471a79d62903075a27796aee Mon Sep 17 00:00:00 2001 From: Phil J Date: Fri, 11 Sep 2026 09:41:09 +0200 Subject: [PATCH 2/2] harden(vcs): total wall-clock budget + ls-files deadline + retry boundary tests Follows the PR 201 adversarial review (APPROVE, non-blocking findings): - bound the whole checkpoint self-heal with a 180s total wall-clock budget so the serial checkpoint worker is never blocked for the ~14-minute worst case of 3 attempts plus backoff - thread the checkpoint deadline through the git ls-files fallback enumeration, matching the documented behavior - tests: assert the retry uses a fresh GIT_INDEX_FILE; assert non-timeout failures are not retried; assert the all-timeout boundary propagates after exactly 3 attempts --- apps/server/src/vcs/GitVcsDriver.ts | 23 +++ ...t3team-GitVcsDriverCheckpointIndex.test.ts | 158 ++++++++++++++++-- .../vcs/t3team-GitVcsDriverCheckpointIndex.ts | 14 +- 3 files changed, 179 insertions(+), 16 deletions(-) diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index f4a3908438..ed1b6cf763 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -1,6 +1,7 @@ import * as NodeCrypto from "node:crypto"; import * as Context from "effect/Context"; +import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -731,12 +732,24 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( const CHECKPOINT_TIMEOUT_RETRY_BACKOFF_MS = [5_000, 30_000] as const; /** Total capture attempts before a timed-out capture surfaces to the caller. */ const CHECKPOINT_MAX_CAPTURE_ATTEMPTS = CHECKPOINT_TIMEOUT_RETRY_BACKOFF_MS.length + 1; + /** + * Total wall-clock budget for the whole self-heal. The per-step and + * per-attempt caps above bound each attempt, but 3 attempts + backoff can + * still block the single serial checkpoint worker for many minutes. This + * gate stops scheduling further retries once the capture has already spent + * this much real time, so the worst case is bounded to this budget plus one + * in-flight attempt. Sized well above a healthy (slow) capture — the 120s + * index deadline plus the fast steps — so a legitimate slow capture still + * gets its retries. + */ + const CHECKPOINT_TOTAL_WALLCLOCK_BUDGET_MS = 180_000; const isVcsProcessTimeoutError = Schema.is(VcsProcessTimeoutError); const checkpoints: VcsDriver.VcsCheckpointOps = { captureCheckpoint: Effect.fn("GitVcsDriver.checkpoints.captureCheckpoint")(function* (input) { const operation = "GitVcsDriver.checkpoints.captureCheckpoint"; const gitCommonDir = yield* resolveGitCommonDir(input.cwd); + const captureStartMs = yield* Clock.currentTimeMillis; // One attempt is fully isolated: a fresh temp index file, seeded from // HEAD, and an idempotent update-ref — so retrying a timed-out attempt @@ -841,6 +854,16 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( (error) => isVcsProcessTimeoutError(error), (error) => Effect.gen(function* () { + const elapsedMs = (yield* Clock.currentTimeMillis) - captureStartMs; + if (elapsedMs >= CHECKPOINT_TOTAL_WALLCLOCK_BUDGET_MS) { + yield* Effect.logWarning("checkpoint.capture.timeout-budget", { + detail: `checkpoint capture exceeded its ${CHECKPOINT_TOTAL_WALLCLOCK_BUDGET_MS}ms wall-clock budget in ${input.cwd}; stopping further retries`, + cwd: input.cwd, + attemptNumber, + elapsedMs, + }); + return yield* error; + } yield* Effect.logWarning("checkpoint.capture.timeout-retry", { detail: `checkpoint capture timed out in ${input.cwd}; retrying in ${backoffMs}ms (attempt ${attemptNumber} of ${CHECKPOINT_MAX_CAPTURE_ATTEMPTS})`, cwd: input.cwd, diff --git a/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.test.ts b/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.test.ts index 9b0937dbf1..07ed0f2053 100644 --- a/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.test.ts +++ b/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.test.ts @@ -260,7 +260,31 @@ it.effect( // "Checkpoint capture failed" activity on every turn. The index step now gets // its own deadline, and a timed-out capture attempt is retried from scratch // (fresh temp index, idempotent update-ref) instead of surfacing the error. -const addTimeoutObservations: Array<{ readonly attempt: number; readonly timeoutMs: number }> = []; +const addTimeoutObservations: Array<{ + readonly attempt: number; + readonly timeoutMs: number; + readonly indexFile: string | undefined; +}> = []; + +type AddBehavior = "timeout-once" | "always-timeout" | "exit-error-once"; +let addBehavior: AddBehavior = "timeout-once"; + +const makeAddTimeout = (input: { cwd: string; readonly timeoutMs?: number }) => + new VcsProcessTimeoutError({ + operation: "GitVcsDriver.checkpoints.captureCheckpoint", + command: "git", + cwd: input.cwd, + timeoutMs: input.timeoutMs ?? 30_000, + }); + +const makeAddExitError = (input: { cwd: string }) => + new VcsProcessExitError({ + operation: "GitVcsDriver.checkpoints.captureCheckpoint", + command: "git", + cwd: input.cwd, + exitCode: 128, + detail: "fatal: unable to index 'nul'", + }); const DriverLayerAddTimeout = Layer.mergeAll(GitVcsDriver.vcsLayer, GitVcsDriver.layer).pipe( Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-git-ckpt-addto-" })), @@ -276,18 +300,19 @@ const DriverLayerAddTimeout = Layer.mergeAll(GitVcsDriver.vcsLayer, GitVcsDriver } if (args[0] === "add" && args.includes("-A") && args.includes("--") && args.includes(".")) { const attempt = addTimeoutObservations.length + 1; - addTimeoutObservations.push({ attempt, timeoutMs: input.timeoutMs ?? 0 }); - // First attempt times out; the retry succeeds. - return attempt === 1 - ? Effect.fail( - new VcsProcessTimeoutError({ - operation: "GitVcsDriver.checkpoints.captureCheckpoint", - command: "git", - cwd: input.cwd, - timeoutMs: input.timeoutMs ?? 30_000, - }), - ) - : Effect.succeed(ok()); + addTimeoutObservations.push({ + attempt, + timeoutMs: input.timeoutMs ?? 0, + indexFile: input.env?.GIT_INDEX_FILE, + }); + if (addBehavior === "always-timeout") { + return Effect.fail(makeAddTimeout(input)); + } + if (addBehavior === "exit-error-once") { + return attempt === 1 ? Effect.fail(makeAddExitError(input)) : Effect.succeed(ok()); + } + // Default ("timeout-once"): first attempt times out; the retry succeeds. + return attempt === 1 ? Effect.fail(makeAddTimeout(input)) : Effect.succeed(ok()); } if (args[0] === "read-tree") { return Effect.succeed(ok()); @@ -344,11 +369,20 @@ it.live( // Both index steps ran with the dedicated checkpoint deadline, not the // generic 30s VcsProcess default. assert.ok(addTimeoutObservations.every((obs) => obs.timeoutMs > 30_000)); + // The retry ran on a FRESH temp index, not the timed-out attempt's. + const [firstIndexFile, secondIndexFile] = [ + addTimeoutObservations[0]!.indexFile, + addTimeoutObservations[1]!.indexFile, + ]; + assert.ok(typeof firstIndexFile === "string" && firstIndexFile.includes("t3-checkpoint-index-")); + assert.ok(typeof secondIndexFile === "string" && secondIndexFile.includes("t3-checkpoint-index-")); + assert.notStrictEqual(firstIndexFile, secondIndexFile); }).pipe( Effect.provide(DriverLayerAddTimeout), Effect.ensuring( Effect.sync(() => { addTimeoutObservations.length = 0; + addBehavior = "timeout-once"; try { NodeFS.rmSync(repoDir, { recursive: true, force: true }); } catch { @@ -359,3 +393,101 @@ it.live( ); }, ); + +it.effect("does not retry a non-timeout capture failure", () => { + addBehavior = "exit-error-once"; + const repoDir = NodePath.join(NodeOS.tmpdir(), `t3-ckpt-exit-err-test-${Date.now()}`); + + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* VcsDriver.VcsDriver; + + yield* fileSystem.makeDirectory(path.join(repoDir, ".git"), { recursive: true }); + + assert.isNotNull(driver.checkpoints); + const result = yield* driver + .checkpoints!.captureCheckpoint({ + cwd: repoDir, + checkpointRef: "refs/t3/checkpoints/exit-err" as never, + }) + .pipe( + Effect.match({ + onSuccess: () => "succeeded" as const, + onFailure: (error) => error, + }), + ); + + if (result === "succeeded") { + assert.fail("expected the capture to fail with a non-timeout error"); + } + if (result._tag !== "VcsProcessExitError") { + assert.fail(`expected VcsProcessExitError, got ${result._tag}`); + } + // No retry: the timed-out-attempt ladder must not fire for non-timeout errors. + assert.strictEqual(addTimeoutObservations.length, 1); + }).pipe( + Effect.provide(DriverLayerAddTimeout), + Effect.ensuring( + Effect.sync(() => { + addTimeoutObservations.length = 0; + addBehavior = "timeout-once"; + try { + NodeFS.rmSync(repoDir, { recursive: true, force: true }); + } catch { + // best-effort cleanup + } + }), + ), + ); +}); + +// `it.live`: the boundary case sleeps the full 5s + 30s backoff ladder between +// the three attempts, so it needs the real clock (~35s wall time). +it.live("propagates the timeout after the final (third) attempt", () => { + addBehavior = "always-timeout"; + const repoDir = NodePath.join(NodeOS.tmpdir(), `t3-ckpt-all-timeout-test-${Date.now()}`); + + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* VcsDriver.VcsDriver; + + yield* fileSystem.makeDirectory(path.join(repoDir, ".git"), { recursive: true }); + + assert.isNotNull(driver.checkpoints); + const result = yield* driver + .checkpoints!.captureCheckpoint({ + cwd: repoDir, + checkpointRef: "refs/t3/checkpoints/all-timeout" as never, + }) + .pipe( + Effect.match({ + onSuccess: () => "succeeded" as const, + onFailure: (error) => error, + }), + ); + + if (result === "succeeded") { + assert.fail("expected the capture to fail after exhausting all attempts"); + } + if (result._tag !== "VcsProcessTimeoutError") { + assert.fail(`expected VcsProcessTimeoutError, got ${result._tag}`); + } + // Exactly CHECKPOINT_MAX_CAPTURE_ATTEMPTS attempts — no fourth, no off-by-one. + assert.strictEqual(addTimeoutObservations.length, 3); + }).pipe( + Effect.provide(DriverLayerAddTimeout), + Effect.ensuring( + Effect.sync(() => { + addTimeoutObservations.length = 0; + addBehavior = "timeout-once"; + try { + NodeFS.rmSync(repoDir, { recursive: true, force: true }); + } catch { + // best-effort cleanup + } + }), + ), + ); +}); diff --git a/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.ts b/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.ts index 9b70b4f8a4..9d3c3b40ad 100644 --- a/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.ts +++ b/apps/server/src/vcs/t3team-GitVcsDriverCheckpointIndex.ts @@ -75,8 +75,16 @@ const listPaths = ( cwd: string, args: readonly string[], env: NodeJS.ProcessEnv, + timeoutMs?: number, ) => - execute({ operation, cwd, args, env, allowNonZeroExit: true }).pipe( + execute({ + operation, + cwd, + args, + env, + allowNonZeroExit: true, + ...(timeoutMs !== undefined ? { timeoutMs } : {}), + }).pipe( Effect.flatMap((result) => result.exitCode === 0 ? Effect.succeed(result.stdout.split("\0").filter((part) => part.length > 0)) @@ -122,8 +130,8 @@ export const indexCheckpointPaths = (deps: { // Fallback: add an explicit NUL-separated pathspec file (argv has a // length limit; a pathspec file does not). const [tracked, untracked] = yield* Effect.all([ - listPaths(execute, operation, cwd, ["ls-files", "-z"], env), - listPaths(execute, operation, cwd, ["ls-files", "--others", "--exclude-standard", "-z"], env), + listPaths(execute, operation, cwd, ["ls-files", "-z"], env, timeoutMs), + listPaths(execute, operation, cwd, ["ls-files", "--others", "--exclude-standard", "-z"], env, timeoutMs), ]); const candidatePaths = [...untracked, ...tracked]; if (candidatePaths.length === 0) {