From 48b0ead4dc3d6a3a4668f48fa122632cc79c5c32 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:35:52 +1000 Subject: [PATCH 01/12] fix(server): preserve provider session ownership through cleanup Preserve the complete PR-owned snapshot at 6b22b7f5cee379310267c716b830db0ce484909b while removing historical upstream merge ancestry. Original PR: https://github.com/pingdotgg/t3code/pull/11492. The nine later follow-up commits are replayed separately. --- ...OrchestratorMcpToolkit.integration.test.ts | 136 +- .../ProviderSessionManager.test.ts | 7581 ++++++++++++++++- .../ProviderSessionManager.ts | 3422 ++++++-- ...estratorReplayRecovery.integration.test.ts | 2 + .../testkit/ProviderReplayHarness.ts | 58 +- .../provider-switching-and-context.md | 10 + 6 files changed, 10467 insertions(+), 742 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts index 3e67bba6e8b6..7c41a23e314d 100644 --- a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts @@ -37,6 +37,7 @@ import { import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -208,6 +209,8 @@ function makeDeterministicAdapter(input: { }); const runOrdinals = new Map(); const turnInputs = new Map(); + const turnFibers = new Map>(); + const sessionScope = yield* Effect.scope; return { instanceId: input.instanceId, @@ -279,76 +282,89 @@ function makeDeterministicAdapter(input: { }, }, ]); - const terminalGate = input.terminalGate?.(turnInput); - if (terminalGate !== undefined) { - yield* Deferred.await(terminalGate); - } else if (!input.shouldComplete(turnInput)) { - return; - } - const response = input.response(turnInput); - yield* publish([ - { - type: "provider_turn.updated", - driver: input.driver, - providerTurn: { - id: providerTurnId, - providerThreadId: turnInput.providerThread.id, - nodeId: turnInput.rootNodeId, - runAttemptId: turnInput.attemptId, - nativeTurnRef: { - driver: input.driver, - nativeId: `native-turn:${turnInput.threadId}:${turnInput.runOrdinal}`, - strength: "strong", + // Acquire-then-return like the production adapters: the call + // returns after dispatch and the gated completion runs in a + // session-scoped fiber. Holding the call open until the gate + // released would leave an admitted adapter op parked until scope + // close, where the release drain would wait on it forever. + const fiber = yield* Effect.gen(function* () { + const terminalGate = input.terminalGate?.(turnInput); + if (terminalGate !== undefined) { + yield* Deferred.await(terminalGate); + } else if (!input.shouldComplete(turnInput)) { + return; + } + const response = input.response(turnInput); + yield* publish([ + { + type: "provider_turn.updated", + driver: input.driver, + providerTurn: { + id: providerTurnId, + providerThreadId: turnInput.providerThread.id, + nodeId: turnInput.rootNodeId, + runAttemptId: turnInput.attemptId, + nativeTurnRef: { + driver: input.driver, + nativeId: `native-turn:${turnInput.threadId}:${turnInput.runOrdinal}`, + strength: "strong", + }, + ordinal: turnInput.providerTurnOrdinal, + status: "completed", + startedAt: eventTime, + completedAt: eventTime, }, - ordinal: turnInput.providerTurnOrdinal, - status: "completed", - startedAt: eventTime, - completedAt: eventTime, }, - }, - { - type: "turn_item.updated", - driver: input.driver, - turnItem: { - id: TurnItemId.make( - `turn-item:${input.instanceId}:${turnInput.threadId}:${turnInput.runOrdinal}:assistant`, - ), - threadId: turnInput.threadId, - runId: turnInput.runId, - nodeId: turnInput.rootNodeId, + { + type: "turn_item.updated", + driver: input.driver, + turnItem: { + id: TurnItemId.make( + `turn-item:${input.instanceId}:${turnInput.threadId}:${turnInput.runOrdinal}:assistant`, + ), + threadId: turnInput.threadId, + runId: turnInput.runId, + nodeId: turnInput.rootNodeId, + providerThreadId: turnInput.providerThread.id, + providerTurnId, + nativeItemRef: null, + parentItemId: null, + ordinal: turnInput.runOrdinal * 100 + 1, + status: "completed", + title: null, + startedAt: eventTime, + completedAt: eventTime, + updatedAt: eventTime, + type: "assistant_message", + messageId: MessageId.make( + `message:${input.instanceId}:${turnInput.threadId}:${turnInput.runOrdinal}:assistant`, + ), + text: response, + streaming: false, + }, + }, + { + type: "turn.terminal", + driver: input.driver, providerThreadId: turnInput.providerThread.id, providerTurnId, - nativeItemRef: null, - parentItemId: null, - ordinal: turnInput.runOrdinal * 100 + 1, + runOrdinal: turnInput.runOrdinal, status: "completed", - title: null, - startedAt: eventTime, - completedAt: eventTime, - updatedAt: eventTime, - type: "assistant_message", - messageId: MessageId.make( - `message:${input.instanceId}:${turnInput.threadId}:${turnInput.runOrdinal}:assistant`, - ), - text: response, - streaming: false, + failure: null, + threadDisposition: "reusable", }, - }, - { - type: "turn.terminal", - driver: input.driver, - providerThreadId: turnInput.providerThread.id, - providerTurnId, - runOrdinal: turnInput.runOrdinal, - status: "completed", - failure: null, - threadDisposition: "reusable", - }, - ]); + ]); + }).pipe(Effect.forkIn(sessionScope)); + turnFibers.set(providerTurnId, fiber); }), steerTurn: () => Effect.void, interruptTurn: ({ providerThread, providerTurnId }) => Effect.gen(function* () { + const fiber = turnFibers.get(providerTurnId); + if (fiber !== undefined) { + yield* Fiber.interrupt(fiber); + turnFibers.delete(providerTurnId); + } const turnInput = turnInputs.get(providerTurnId); const completedAt = yield* DateTime.now; if (turnInput !== undefined) { diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 8d2947094aaa..ff25a104d83f 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -20,12 +20,14 @@ import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; +import * as Scheduler from "effect/Scheduler"; import * as Stream from "effect/Stream"; import { TestClock } from "effect/testing"; import { HttpServer } from "effect/unstable/http"; @@ -50,11 +52,13 @@ import { ProviderAdapterEventStreamError, type ProviderAdapterV2Event, ProviderAdapterProtocolError, + ProviderAdapterTurnStartError, type ProviderAdapterV2RuntimePolicy, + type ProviderAdapterV2TurnInput, type ProviderAdapterV2SessionRuntime, type ProviderAdapterV2Shape, } from "./ProviderAdapter.ts"; -import { makeSingleLayer as makeProviderAdapterRegistryLayer } from "./ProviderAdapterRegistry.ts"; +import { makeLayer as makeProviderAdapterRegistryLayerFor } from "./ProviderAdapterRegistry.ts"; import { layer as providerEventIngestorLayer } from "./ProviderEventIngestor.ts"; import { ProviderSessionManagerV2, @@ -127,11 +131,13 @@ function makeProviderSession(input: { readonly providerSessionId: ProviderSessionId; readonly now: DateTime.Utc; readonly capabilities?: OrchestrationV2ProviderCapabilities; + readonly providerInstanceId?: ProviderInstanceId; + readonly driver?: ProviderDriverKind; }): OrchestrationV2ProviderSession { return { id: input.providerSessionId, - driver: CODEX_DRIVER, - providerInstanceId: modelSelection.instanceId, + driver: input.driver ?? CODEX_DRIVER, + providerInstanceId: input.providerInstanceId ?? modelSelection.instanceId, status: "ready", cwd: process.cwd(), model: "gpt-5.4", @@ -239,6 +245,8 @@ function unimplemented(detail: string) { function makeProviderAdapter( state: Ref.Ref, options: { + readonly instanceId?: ProviderInstanceId; + readonly driver?: ProviderDriverKind; readonly failEventStream?: boolean; readonly capabilities?: OrchestrationV2ProviderCapabilities; readonly mcpConfigs?: Ref.Ref< @@ -249,12 +257,26 @@ function makeProviderAdapter( readonly initialProviderItemIdentityVersion?: 2; }) => Effect.Effect; readonly hasPendingBackgroundWork?: Effect.Effect; - readonly hangSessionScopeClose?: boolean; + readonly hangSessionScopeClose?: Deferred.Deferred; + readonly beforeClose?: Effect.Effect; + readonly beforeInterrupt?: Effect.Effect; + readonly afterOpen?: Effect.Effect; + readonly startTurn?: ( + input: ProviderAdapterV2TurnInput, + ) => Effect.Effect; + readonly resumeThread?: ( + input: Parameters[0], + ) => ReturnType; + readonly ensureThread?: ( + input: Parameters[0], + ) => ReturnType; } = {}, ): ProviderAdapterV2Shape { + const instanceId = options.instanceId ?? ProviderInstanceId.make("codex"); + const driver = options.driver ?? CODEX_DRIVER; return { - instanceId: ProviderInstanceId.make("codex"), - driver: CODEX_DRIVER, + instanceId, + driver, getCapabilities: () => Effect.succeed(options.capabilities ?? CodexCapabilities), planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), openSession: (input) => @@ -273,6 +295,8 @@ function makeProviderAdapter( const session = makeProviderSession({ providerSessionId: input.providerSessionId, now, + providerInstanceId: instanceId, + driver, ...(options.capabilities === undefined ? {} : { capabilities: options.capabilities }), }); yield* Ref.update(state, (current) => { @@ -290,22 +314,32 @@ function makeProviderAdapter( closeCount: current.closeCount + 1, })), ); - if (options.hangSessionScopeClose === true) { + const hangSessionScopeClose = options.hangSessionScopeClose; + if (hangSessionScopeClose !== undefined) { // Registered last so it runs first on scope close, wedging the // close before the closeCount finalizer, like a provider process - // that never yields its message stream. - yield* Effect.addFinalizer(() => Effect.never); + // that never yields its message stream. The gate exists so a test + // can let teardown finish: layer shutdown joins pending releases + // and would otherwise park on this finalizer forever. + yield* Effect.addFinalizer(() => Deferred.await(hangSessionScopeClose)); + } + const beforeClose = options.beforeClose; + if (beforeClose !== undefined) { + yield* Effect.addFinalizer(() => beforeClose); + } + if (options.afterOpen !== undefined) { + yield* options.afterOpen; } return { - instanceId: ProviderInstanceId.make("codex"), - driver: CODEX_DRIVER, + instanceId, + driver, providerSessionId: input.providerSessionId, providerSession: session, events: options.failEventStream ? Stream.fail( new ProviderAdapterEventStreamError({ - driver: CODEX_DRIVER, + driver, providerSessionId: input.providerSessionId, cause: "process exited", }), @@ -314,19 +348,29 @@ function makeProviderAdapter( ...(options.hasPendingBackgroundWork === undefined ? {} : { hasPendingBackgroundWork: options.hasPendingBackgroundWork }), - ensureThread: () => unimplemented("ensureThread unused in test"), + ensureThread: (threadInput) => + options.ensureThread === undefined + ? unimplemented("ensureThread unused in test") + : options.ensureThread(threadInput), resumeThread: (threadInput) => - Ref.update(state, (current) => ({ - ...current, - resumeCount: current.resumeCount + 1, - })).pipe(Effect.as(threadInput.providerThread)), - startTurn: () => Effect.void, + options.resumeThread === undefined + ? Ref.update(state, (current) => ({ + ...current, + resumeCount: current.resumeCount + 1, + })).pipe(Effect.as(threadInput.providerThread)) + : options.resumeThread(threadInput), + startTurn: (turnInput) => + options.startTurn === undefined ? Effect.void : options.startTurn(turnInput), steerTurn: () => Effect.void, interruptTurn: () => - Ref.update(state, (current) => ({ - ...current, - interruptCount: current.interruptCount + 1, - })), + (options.beforeInterrupt ?? Effect.void).pipe( + Effect.andThen( + Ref.update(state, (current) => ({ + ...current, + interruptCount: current.interruptCount + 1, + })), + ), + ), respondToRuntimeRequest: () => Effect.void, readThreadSnapshot: () => unimplemented("readThreadSnapshot unused in test"), rollbackThread: () => unimplemented("rollbackThread unused in test"), @@ -340,6 +384,7 @@ function makeTestLayer(input: { readonly state: Ref.Ref; readonly idleTimeoutMs: number; readonly maxIdlePinMs?: number; + readonly driver?: ProviderDriverKind; readonly failEventStream?: boolean; readonly capabilities?: OrchestrationV2ProviderCapabilities; readonly mcpConfigs?: Ref.Ref< @@ -350,16 +395,34 @@ function makeTestLayer(input: { readonly initialProviderItemIdentityVersion?: 2; }) => Effect.Effect; readonly failReleaseEventWrites?: boolean; + readonly eventSinkLayer?: typeof TestEventSinkLayer; + readonly mcpRegistryLayer?: typeof TestMcpRegistryLayer; readonly hasPendingBackgroundWork?: Effect.Effect; - readonly hangSessionScopeClose?: boolean; + readonly hangSessionScopeClose?: Deferred.Deferred; + readonly beforeClose?: Effect.Effect; + readonly beforeInterrupt?: Effect.Effect; + readonly afterOpen?: Effect.Effect; + readonly startTurn?: ( + input: ProviderAdapterV2TurnInput, + ) => Effect.Effect; + readonly resumeThread?: ( + input: Parameters[0], + ) => ReturnType; + readonly ensureThread?: ( + input: Parameters[0], + ) => ReturnType; readonly serverSettingsLayer?: ReturnType; readonly projectServiceLayer?: Layer.Layer; + readonly fileSystemLayer?: Layer.Layer; + readonly extraAdapters?: ReadonlyArray; }) { - const configuredEventSinkLayer = input.failReleaseEventWrites - ? FailingReleaseEventSinkLayer - : TestEventSinkLayer; - const registryLayer = makeProviderAdapterRegistryLayer( + const mcpRegistryLayer = input.mcpRegistryLayer ?? TestMcpRegistryLayer; + const configuredEventSinkLayer = + input.eventSinkLayer ?? + (input.failReleaseEventWrites ? FailingReleaseEventSinkLayer : TestEventSinkLayer); + const adapters = [ makeProviderAdapter(input.state, { + ...(input.driver === undefined ? {} : { driver: input.driver }), failEventStream: input.failEventStream ?? false, ...(input.capabilities === undefined ? {} : { capabilities: input.capabilities }), ...(input.mcpConfigs === undefined ? {} : { mcpConfigs: input.mcpConfigs }), @@ -370,8 +433,16 @@ function makeTestLayer(input: { ...(input.hangSessionScopeClose === undefined ? {} : { hangSessionScopeClose: input.hangSessionScopeClose }), + ...(input.beforeClose === undefined ? {} : { beforeClose: input.beforeClose }), + ...(input.beforeInterrupt === undefined ? {} : { beforeInterrupt: input.beforeInterrupt }), + ...(input.afterOpen === undefined ? {} : { afterOpen: input.afterOpen }), + ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }), + ...(input.resumeThread === undefined ? {} : { resumeThread: input.resumeThread }), + ...(input.ensureThread === undefined ? {} : { ensureThread: input.ensureThread }), }), - ); + ...(input.extraAdapters ?? []), + ]; + const registryLayer = makeProviderAdapterRegistryLayerFor(adapters); const providerEventIngestorTestLayer = providerEventIngestorLayer.pipe( Layer.provide(Layer.mergeAll(configuredEventSinkLayer, idAllocatorLayer, TestStoresLayer)), ); @@ -379,7 +450,7 @@ function makeTestLayer(input: { TestStoresLayer, configuredEventSinkLayer, idAllocatorLayer, - TestMcpRegistryLayer, + mcpRegistryLayer, providerSessionManagerLayerWithOptions({ idleTimeoutMs: input.idleTimeoutMs, ...(input.maxIdlePinMs === undefined ? {} : { maxIdlePinMs: input.maxIdlePinMs }), @@ -390,10 +461,11 @@ function makeTestLayer(input: { configuredEventSinkLayer, idAllocatorLayer, providerEventIngestorTestLayer, - TestMcpRegistryLayer, + mcpRegistryLayer, TestStoresLayer, ...(input.serverSettingsLayer === undefined ? [] : [input.serverSettingsLayer]), ...(input.projectServiceLayer === undefined ? [] : [input.projectServiceLayer]), + ...(input.fileSystemLayer === undefined ? [] : [input.fileSystemLayer]), ), ), ), @@ -1179,6 +1251,8 @@ it.effect("ProviderSessionManagerV2 revokes MCP credentials when release persist assert.equal(closeError._tag, "ProviderSessionCloseError"); assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); assert.isUndefined(yield* registry.resolve(token!)); + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + assert.equal((yield* Ref.get(state)).openCount, 2); }); yield* effect.pipe( @@ -1194,6 +1268,61 @@ it.effect("ProviderSessionManagerV2 revokes MCP credentials when release persist }), ); +it.effect("ProviderSessionManagerV2 keeps MCP credential claims when release cleanup fails", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSinkV2; + const idAllocator = yield* IdAllocatorV2; + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-provider-session-manager-mcp-cleanup-failure"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + + const captured = (yield* Ref.get(mcpConfigs))[0]; + const token = captured?.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(token); + assert.isDefined(yield* registry.resolve(token!)); + + const closeError = yield* manager.close(providerSessionId).pipe(Effect.flip); + assert.equal(closeError._tag, "ProviderSessionCloseError"); + // The release stays pending: its provider process may still be alive, so + // the pending release must keep claiming the credential it configured + // rather than revoking it out from under that process. + assert.isDefined(yield* registry.resolve(token!)); + assert.isDefined(McpProviderSession.readMcpProviderSession(threadId)); + }); + + yield* effect.pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 1_000, + mcpConfigs, + beforeClose: Effect.die("cleanup defect"), + }), + ), + ); + }), +); + it.effect("ProviderSessionManagerV2 duplicate detach preserves replacement MCP credentials", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); @@ -1483,6 +1612,8 @@ it.effect( ReadonlyArray >([]); const duringOpen = yield* Ref.make>(Effect.void); + const openEntered = yield* Deferred.make(); + const openGate = yield* Deferred.make(); const effect = Effect.gen(function* () { const eventSink = yield* EventSinkV2; const idAllocator = yield* IdAllocatorV2; @@ -1513,8 +1644,21 @@ it.effect( // releases. Eager adapters (ACP, OpenCode) bake the credential into // the process during openSession, so the release must not revoke it; // rotating afterwards cannot repair those adapters. - yield* Ref.set(duringOpen, manager.close(s1).pipe(Effect.orDie)); - yield* manager.open({ threadId, providerSessionId: s2, modelSelection, runtimePolicy }); + yield* Ref.set( + duringOpen, + Deferred.succeed(openEntered, undefined).pipe(Effect.andThen(Deferred.await(openGate))), + ); + const opening = yield* manager + .open({ threadId, providerSessionId: s2, modelSelection, runtimePolicy }) + .pipe(Effect.forkChild); + yield* Deferred.await(openEntered); + const closing = yield* manager.close(s1).pipe(Effect.result, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(closing))._tag, "Failure"); + assert.equal((yield* Ref.get(state)).closeCount, 0); + yield* Deferred.succeed(openGate, undefined); + yield* Fiber.join(opening); + yield* manager.close(s1); const slot = McpProviderSession.readMcpProviderSession(threadId); assert.equal( @@ -1531,6 +1675,7 @@ it.effect( }); yield* effect.pipe( + Effect.ensuring(Deferred.succeed(openGate, undefined)), Effect.provide( makeTestLayer({ state, @@ -1639,9 +1784,10 @@ it.effect("ProviderSessionManagerV2 releases idle sessions without sweeping all }), ); -it.effect("ProviderSessionManagerV2 persists release when session scope close hangs", () => +it.effect("ProviderSessionManagerV2 reports an error when session scope close hangs", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); + const hangClose = yield* Deferred.make(); const effect = Effect.gen(function* () { const eventSink = yield* EventSinkV2; const idAllocator = yield* IdAllocatorV2; @@ -1676,16 +1822,330 @@ it.effect("ProviderSessionManagerV2 persists release when session scope close ha yield* TestClock.adjust("30 seconds"); yield* Effect.yieldNow; const projection = yield* projectionStore.getThreadProjection(threadId); - assert.equal(projection.providerSessions.at(-1)?.status, "stopped"); + assert.equal(projection.providerSessions.at(-1)?.status, "error"); + assert.include(projection.providerSessions.at(-1)?.lastError ?? "", "30 seconds"); assert.equal((yield* Ref.get(state)).closeCount, 0); }); yield* effect.pipe( - Effect.provide(makeTestLayer({ state, idleTimeoutMs: 1000, hangSessionScopeClose: true })), + Effect.ensuring(Deferred.succeed(hangClose, undefined)), + Effect.provide( + makeTestLayer({ state, idleTimeoutMs: 1000, hangSessionScopeClose: hangClose }), + ), ); }), ); +function makeThreadSessionFixture(threadId: ThreadId) { + return Effect.gen(function* () { + const idAllocator = yield* IdAllocatorV2; + const eventSink = yield* EventSinkV2; + const manager = yield* ProviderSessionManagerV2; + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now: yield* DateTime.now })], + }); + return { + allocate: idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }), + open: (providerSessionId: ProviderSessionId, attachedThreadId = threadId) => + manager.open({ + threadId: attachedThreadId, + providerSessionId, + modelSelection, + runtimePolicy, + }), + }; + }); +} + +// A scheduler that yields a fiber exactly once — when it reaches a chosen op +// boundary — and queues the continuations it would run in `tasks`, so a test +// can suspend a fiber between two adjacent runtime instructions and interrupt +// it there. That is the only way to reach windows like "after `opening.set` +// but before the unwind handler", which have no suspension for ordinary +// fork/interrupt choreography to land in. `shouldYield` must fire once: the +// resumed fiber rechecks it before evaluating the deferred op, so returning +// true repeatedly would suspend the same op forever without making progress. +function makeSteppingScheduler() { + const tasks: Array<() => void> = []; + const counts = new Map, number>(); + const targets = new Map, number>(); + // Resume tasks of fibers suspended through `spawnedTarget`, keyed by fiber so + // a `drain` cannot accidentally release them — a spawned fiber stays parked + // until `resumeSpawned` requeues its continuation. + const held = new Map, () => void>(); + // When set, a fiber with no explicit target suspends once its own op count + // reaches this value. The runtime consults `shouldYield` synchronously inside + // the yielding task, so the next scheduled task is that fiber's resume — + // capture it instead of queueing. + let spawnedTarget: number | undefined; + let capture: Fiber.Fiber | undefined; + const scheduler: Scheduler.Scheduler = { + executionMode: "sync", + shouldYield: (fiber) => { + const count = (counts.get(fiber) ?? 0) + 1; + counts.set(fiber, count); + if (targets.has(fiber)) return count === targets.get(fiber); + if (spawnedTarget === undefined || count !== spawnedTarget) return false; + // The arming is consumed by the first spawned fiber to reach the offset: + // later forks (the scope-close grandchild, settle-phase workers) must + // run free or the sweep deadlocks on its own bookkeeping. + spawnedTarget = undefined; + capture = fiber; + return true; + }, + makeDispatcher: () => ({ + scheduleTask: (task) => { + if (capture !== undefined) { + held.set(capture, task); + capture = undefined; + return; + } + tasks.push(task); + }, + flush: () => { + while (tasks.length > 0) tasks.shift()!(); + }, + }), + }; + // Runs `fiber` until it has reached `ops` op checks — at which point it is + // suspended with its resume task queued — then returns true. Returns true + // for a fiber that exits before `ops`; returns false when the fiber parks + // on work outside this dispatcher (e.g. an async fileSystem call or a + // deferred nobody completes), which caps the sweep at that boundary. + const step = (fiber: Fiber.Fiber, ops: number) => + Effect.gen(function* () { + targets.set(fiber, ops); + for (let emptyRounds = 0; (counts.get(fiber) ?? 0) < ops;) { + const task = tasks.shift(); + if (task !== undefined) { + task(); + emptyRounds = 0; + continue; + } + if (fiber.pollUnsafe() !== undefined) return true; + // The fiber's initial evaluation is dispatched through the forking + // fiber's scheduler, so give the default dispatcher turns before + // concluding it is parked. + yield* Effect.yieldNow; + if (tasks.length === 0 && ++emptyRounds > 32) return false; + } + return true; + }); + // Runs every queued continuation, yielding so default-scheduler work can + // resolve deferreds the stepped fibers are waiting on. + const drain = Effect.gen(function* () { + for (let round = 0; round < 64; round++) { + while (tasks.length > 0) tasks.shift()!(); + yield* Effect.yieldNow; + if (tasks.length === 0) return; + } + }); + // Registers `fiber` as an explicit target so `spawnedTarget` ignores it; + // `ops` beyond its lifetime means it never suspends on a target. + const aim = (fiber: Fiber.Fiber, ops: number) => { + targets.set(fiber, ops); + }; + // Suspends the first spawned (non-targeted) fiber to reach its own op `ops`; + // the arming is consumed by that hold. + const holdSpawnedAt = (ops: number) => { + spawnedTarget = ops; + }; + // Requeues the captured resume tasks of all suspended spawned fibers. + const resumeSpawned = () => { + for (const task of held.values()) tasks.push(task); + held.clear(); + }; + return { scheduler, tasks, counts, step, drain, aim, holdSpawnedAt, resumeSpawned, held }; +} + +for (const operation of ["close", "detach", "closeInstance"] as const) { + it.effect( + `ProviderSessionManagerV2 blocks replacement after ${operation} times out until cleanup completes`, + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const closeEntered = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStoreV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const threadId = ThreadId.make(`thread-release-timeout-${operation}`); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* allocate; + const replacementId = yield* allocate; + const release = + operation === "close" + ? manager.close(providerSessionId) + : operation === "closeInstance" + ? manager.closeInstance(modelSelection.instanceId) + : manager.detach({ providerSessionId, threadId }); + yield* open(providerSessionId); + const cancelledClose = yield* release.pipe(Effect.forkChild); + yield* Deferred.await(closeEntered); + yield* Fiber.interrupt(cancelledClose); + const firstClose = yield* release.pipe(Effect.result, Effect.forkChild); + assert.equal( + (yield* open(replacementId).pipe(Effect.flip))._tag, + "ProviderSessionOpenError", + ); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(firstClose))._tag, "Failure"); + assert.equal( + (yield* projectionStore.getThreadProjection(threadId)).providerSessions.at(-1)?.status, + "error", + ); + assert.equal( + (yield* open(providerSessionId).pipe(Effect.flip))._tag, + "ProviderSessionOpenError", + ); + assert.equal( + (yield* open(replacementId).pipe(Effect.flip))._tag, + "ProviderSessionOpenError", + ); + assert.equal((yield* Ref.get(state)).openCount, 1); + assert.equal((yield* Ref.get(state)).closeCount, 0); + const token = (yield* Ref.get(mcpConfigs))[0]?.authorizationHeader.replace( + /^Bearer\s+/, + "", + ); + assert.isDefined(yield* registry.resolve(token!)); + + // Retrying must wait for the original cleanup, not treat the removed entry as stopped. + const retry = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe(Effect.result, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(retry))._tag, "Failure"); + // The pending cleanup still holds the credential; the timed-out + // detach defers revocation to it instead of revoking under it. + assert.isDefined(yield* registry.resolve(token!)); + yield* Deferred.succeed(closeGate, undefined); + yield* release; + assert.equal((yield* Ref.get(state)).closeCount, 1); + assert.isUndefined(yield* registry.resolve(token!)); + const stopped = (yield* projectionStore.getThreadProjection( + threadId, + )).providerSessions.at(-1); + assert.equal(stopped?.status, "stopped"); + assert.isNull(stopped?.lastError); + yield* open(replacementId); + assert.equal((yield* Ref.get(state)).openCount, 2); + }).pipe( + Effect.ensuring(Deferred.succeed(closeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + capabilities: ExclusiveCapabilities, + mcpConfigs, + beforeClose: Deferred.succeed(closeEntered, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + ), + }), + ), + ); + }), + ); +} + +for (const cleanupOutcome of ["success", "failure"] as const) { + it.effect( + `ProviderSessionManagerV2 reuses a live peer and preserves cleanup ${cleanupOutcome} semantics`, + () => + Effect.gen(function* () { + const cleanupDefect = "private adapter output that must not reach the client"; + const state = yield* Ref.make(emptyState); + const closes = yield* Ref.make(0); + const closeEntered = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStoreV2; + const threadId = ThreadId.make(`thread-live-peer-${cleanupOutcome}`); + const otherThreadId = ThreadId.make(`thread-new-attachment-${cleanupOutcome}`); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + yield* makeThreadSessionFixture(otherThreadId); + const oldId = yield* allocate; + const liveId = yield* allocate; + const newId = yield* allocate; + yield* open(oldId); + yield* open(oldId, otherThreadId); + const liveRuntime = yield* open(liveId); + const closing = yield* manager.close(oldId).pipe(Effect.result, Effect.forkChild); + yield* Deferred.await(closeEntered); + assert.strictEqual(yield* open(liveId), liveRuntime); + assert.equal((yield* open(newId).pipe(Effect.flip))._tag, "ProviderSessionOpenError"); + assert.equal((yield* open(oldId).pipe(Effect.flip))._tag, "ProviderSessionOpenError"); + assert.equal( + (yield* open(liveId, otherThreadId).pipe(Effect.flip))._tag, + "ProviderSessionOpenError", + ); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(closing))._tag, "Failure"); + assert.strictEqual(yield* open(liveId), liveRuntime); + assert.equal((yield* Ref.get(state)).openCount, 2); + assert.equal((yield* Ref.get(state)).closeCount, 0); + yield* Deferred.succeed(closeGate, undefined); + const completed = yield* manager.close(oldId).pipe(Effect.result); + assert.equal(completed._tag, cleanupOutcome === "success" ? "Success" : "Failure"); + assert.strictEqual(yield* open(liveId), liveRuntime); + if (cleanupOutcome === "success") { + assert.strictEqual(yield* open(liveId, otherThreadId), liveRuntime); + } else { + const session = (yield* projectionStore.getThreadProjection( + threadId, + )).providerSessions.find((session) => session.id === oldId); + assert.equal(session?.status, "error"); + assert.equal(session?.lastError, "Provider session cleanup failed."); + const release = manager.release({ + providerSessionId: oldId, + reason: "manual_shutdown", + }); + const releaseError = yield* release.pipe(Effect.flip); + const cleanupCause = releaseError.cause; + if (!Cause.isCause(cleanupCause)) assert.fail("Expected the original cleanup cause"); + assert.strictEqual(Cause.squash(cleanupCause), cleanupDefect); + assert.strictEqual(yield* release.pipe(Effect.flip), releaseError); + assert.equal((yield* open(oldId).pipe(Effect.flip))._tag, "ProviderSessionOpenError"); + assert.equal( + (yield* open(liveId, otherThreadId).pipe(Effect.flip))._tag, + "ProviderSessionOpenError", + ); + assert.equal((yield* open(newId).pipe(Effect.flip))._tag, "ProviderSessionOpenError"); + } + assert.equal((yield* Ref.get(state)).openCount, 2); + }).pipe( + Effect.ensuring(Deferred.succeed(closeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + beforeClose: Ref.getAndUpdate(closes, (n) => n + 1).pipe( + Effect.flatMap((n) => + n === 0 + ? Deferred.succeed(closeEntered, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + Effect.andThen( + cleanupOutcome === "failure" ? Effect.die(cleanupDefect) : Effect.void, + ), + ) + : Effect.void, + ), + ), + }), + ), + ); + }), + ); +} + it.effect("ProviderSessionManagerV2 defers idle release while background work is pending", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); @@ -1957,28 +2417,25 @@ it.effect("ProviderSessionManagerV2 does not apply a stale idle pin to a replace yield* TestClock.adjust("1 second"); yield* Deferred.await(checkEntered); - // close removes the map entry first, then waits to interrupt the idle - // fiber (still uninterruptible). That window lets a replacement open - // under the same providerSessionId before the stale probe finishes. + // Cleanup waits for the uninterruptible probe. Replacement must wait too. const closeFiber = yield* manager.close(providerSessionId).pipe(Effect.forkDetach); for (let i = 0; i < 20; i += 1) { yield* Effect.yieldNow; } - yield* manager.open({ + const replacement = manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy, }); - assert.equal((yield* Ref.get(state)).openCount, 2); + assert.equal((yield* replacement.pipe(Effect.flip))._tag, "ProviderSessionOpenError"); + assert.equal((yield* Ref.get(state)).openCount, 1); // Stale probe reports pending work against the old runtime; the pin // stamp must no-op on the replacement (runtime / generation mismatch). yield* Deferred.succeed(checkGate, undefined); yield* Fiber.join(closeFiber); - for (let i = 0; i < 10; i += 1) { - yield* Effect.yieldNow; - } + yield* replacement; assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); assert.equal((yield* Ref.get(state)).closeCount, 1); @@ -3004,3 +3461,7041 @@ it.effect( assert.isFalse(denied?.capabilities?.has("device")); }), ); +for (const cleanupOutcome of ["success", "persistence_failure", "cleanup_failure"] as const) { + it.effect( + `ProviderSessionManagerV2 preserves shared credentials during overlapping cleanup with ${cleanupOutcome}`, + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const closeGates = [yield* Deferred.make(), yield* Deferred.make()]; + const closeEntered = [yield* Deferred.make(), yield* Deferred.make()]; + const closes = yield* Ref.make(0); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const threadId = ThreadId.make(`thread-pending-peer-${cleanupOutcome}`); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const first = yield* allocate; + const second = yield* allocate; + yield* open(first); + yield* open(second); + const configs = yield* Ref.get(mcpConfigs); + assert.equal(configs[0]?.authorizationHeader, configs[1]?.authorizationHeader); + const token = configs[0]!.authorizationHeader.replace(/^Bearer\s+/, ""); + const firstClose = yield* manager.close(first).pipe(Effect.result, Effect.forkChild); + yield* Deferred.await(closeEntered[0]!); + const secondClose = yield* manager.close(second).pipe(Effect.result, Effect.forkChild); + yield* Deferred.await(closeEntered[1]!); + // The second cleanup can finish while the first still holds the shared token. + yield* Deferred.succeed(closeGates[1]!, undefined); + assert.equal( + (yield* Fiber.join(secondClose))._tag, + cleanupOutcome === "persistence_failure" ? "Failure" : "Success", + ); + assert.equal((yield* Ref.get(state)).closeCount, 1); + assert.isDefined(yield* registry.resolve(token)); + yield* Deferred.succeed(closeGates[0]!, undefined); + assert.equal( + (yield* Fiber.join(firstClose))._tag, + cleanupOutcome === "success" ? "Success" : "Failure", + ); + assert.equal((yield* Ref.get(state)).closeCount, 2); + if (cleanupOutcome === "cleanup_failure") { + // The first cleanup failed — its release stays pending and its + // provider process may still be alive, so it must keep claiming the + // shared token rather than revoking it out from under that process. + assert.isDefined(yield* registry.resolve(token)); + } else { + assert.isUndefined(yield* registry.resolve(token)); + } + }).pipe( + Effect.ensuring( + Effect.forEach(closeGates, (gate) => Deferred.succeed(gate, undefined), { + discard: true, + }), + ), + Effect.provide( + makeTestLayer({ + state, + mcpConfigs, + failReleaseEventWrites: cleanupOutcome === "persistence_failure", + idleTimeoutMs: 3_600_000, + beforeClose: Ref.getAndUpdate(closes, (n) => n + 1).pipe( + Effect.flatMap((n) => + Deferred.succeed(closeEntered[n]!, undefined).pipe( + Effect.andThen(Deferred.await(closeGates[n]!)), + Effect.andThen( + n === 0 && cleanupOutcome === "cleanup_failure" + ? Effect.die("cleanup failed") + : Effect.void, + ), + ), + ), + ), + }), + ), + ); + }), + ); +} + +it.effect( + "ProviderSessionManagerV2 queued opens recheck the pending cleanup after taking the lock", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const openEntered = yield* Deferred.make(); + const openGate = yield* Deferred.make(); + const closeEntered = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + const opens = yield* Ref.make(0); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const threadId = ThreadId.make("edge-queued-open"); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const first = yield* allocate; + const second = yield* allocate; + const third = yield* allocate; + yield* open(first); + const secondOpen = yield* open(second).pipe(Effect.forkChild); + yield* Deferred.await(openEntered); + const queuedOpen = yield* open(third).pipe(Effect.result, Effect.forkChild); + yield* TestClock.adjust("0 seconds"); + const closing = yield* manager.close(first).pipe(Effect.result, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(closing))._tag, "Failure"); + yield* Deferred.succeed(openGate, undefined); + yield* Fiber.join(secondOpen); + assert.equal((yield* Fiber.join(queuedOpen))._tag, "Failure"); + yield* Deferred.await(closeEntered); + assert.equal((yield* Ref.get(state)).openCount, 2); + yield* Deferred.succeed(closeGate, undefined); + yield* manager.close(first); + yield* open(third); + assert.equal((yield* Ref.get(state)).openCount, 3); + }).pipe( + Effect.ensuring( + Deferred.succeed(openGate, undefined).pipe( + Effect.andThen(Deferred.succeed(closeGate, undefined)), + ), + ), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + beforeOpen: () => + Ref.getAndUpdate(opens, (n) => n + 1).pipe( + Effect.flatMap((n) => + n === 1 + ? Deferred.succeed(openEntered, undefined).pipe( + Effect.andThen(Deferred.await(openGate)), + ) + : Effect.void, + ), + ), + beforeClose: Deferred.succeed(closeEntered, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + ), + }), + ), + ); + }), +); + +for (const credentialKind of ["fresh", "reused"] as const) { + it.effect( + `ProviderSessionManagerV2 rejects a closing session attachment with a ${credentialKind} credential`, + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const preparing = yield* Deferred.make(); + const prepareGate = yield* Deferred.make(); + const pauseSettings = yield* Ref.make(false); + const issuedCredentials = yield* Ref.make< + ReadonlyArray + >([]); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (input) => + delegate + .issue(input) + .pipe( + Effect.tap((issued) => + Ref.update(issuedCredentials, (configs) => [...configs, issued.config]), + ), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + const settingsLayer = Layer.effect( + ServerSettings.ServerSettingsService, + Effect.gen(function* () { + const delegate = yield* ServerSettings.ServerSettingsService; + return ServerSettings.ServerSettingsService.of({ + ...delegate, + getSettings: Ref.get(pauseSettings).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(prepareGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.getSettings), + ), + }); + }), + ).pipe(Layer.provide(ServerSettings.layerTest({ enableAgentBrowserAccess: true }))); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const firstThread = ThreadId.make(`thread-attach-first-${credentialKind}`); + const attachingThread = ThreadId.make(`thread-attach-second-${credentialKind}`); + const first = yield* makeThreadSessionFixture(firstThread); + const peer = yield* makeThreadSessionFixture(attachingThread); + const providerSessionId = yield* first.allocate; + yield* first.open(providerSessionId); + const peerId = yield* peer.allocate; + if (credentialKind === "reused") yield* peer.open(peerId); + const peerCredential = McpProviderSession.readMcpProviderSession(attachingThread); + yield* Ref.set(pauseSettings, true); + const attaching = yield* peer + .open(providerSessionId) + .pipe(Effect.result, Effect.forkChild); + yield* Deferred.await(preparing); + const closing = yield* manager + .close(providerSessionId) + .pipe(Effect.result, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(closing))._tag, "Failure"); + yield* Deferred.succeed(prepareGate, undefined); + assert.equal((yield* Fiber.join(attaching))._tag, "Failure"); + yield* manager.close(providerSessionId); + assert.equal((yield* Ref.get(state)).closeCount, 1); + if (credentialKind === "reused") { + assert.isDefined(peerCredential); + const token = peerCredential!.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(yield* registry.resolve(token)); + yield* manager.close(peerId); + assert.isUndefined(yield* registry.resolve(token)); + } + const issued = (yield* Ref.get(issuedCredentials)).find( + (config) => config.threadId === attachingThread, + ); + assert.isDefined(issued); + assert.isUndefined( + yield* registry.resolve(issued!.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.isUndefined(McpProviderSession.readMcpProviderSession(attachingThread)); + }).pipe( + Effect.ensuring(Deferred.succeed(prepareGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + serverSettingsLayer: settingsLayer, + mcpRegistryLayer, + }), + ), + ); + }), + ); +} + +it.effect("ProviderSessionManagerV2 detach joins a cleanup that started mid-detach", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const interruptEntered = yield* Deferred.make(); + const interruptGate = yield* Deferred.make(); + const closeEntered = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + const detachReturned = yield* Deferred.make(); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const eventSink = yield* EventSinkV2; + const idAllocator = yield* IdAllocatorV2; + const now = yield* DateTime.now; + const firstThreadId = ThreadId.make("thread-detach-race-first"); + const secondThreadId = ThreadId.make("thread-detach-race-second"); + const first = yield* makeThreadSessionFixture(firstThreadId); + const second = yield* makeThreadSessionFixture(secondThreadId); + const providerSessionId = yield* first.allocate; + const providerThread = makeProviderThread({ + idAllocator, + threadId: firstThreadId, + providerSessionId, + now, + }); + const runId = idAllocator.derive.run({ threadId: firstThreadId, ordinal: 1 }); + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId: firstThreadId }), + type: "provider-thread.updated", + threadId: firstThreadId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: providerThread, + }, + { + id: yield* idAllocator.allocate.event({ threadId: firstThreadId }), + type: "provider-turn.updated", + threadId: firstThreadId, + runId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: { + id: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-detach-race", + }), + providerThreadId: providerThread.id, + nodeId: idAllocator.derive.rootNode({ runId }), + runAttemptId: null, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }, + ], + }); + yield* first.open(providerSessionId); + yield* second.open(providerSessionId); + + // Detach parks inside interruptTurn on the shared session; close wins + // the race to `releasing` while detach is still in its async window. + const detaching = yield* manager.detach({ providerSessionId, threadId: firstThreadId }).pipe( + Effect.tap(() => Deferred.succeed(detachReturned, undefined)), + Effect.result, + Effect.forkChild, + ); + yield* Deferred.await(interruptEntered); + const closing = yield* manager.close(providerSessionId).pipe(Effect.result, Effect.forkChild); + yield* Deferred.await(closeEntered); + + yield* Deferred.succeed(interruptGate, undefined); + for (let i = 0; i < 10; i += 1) { + yield* Effect.yieldNow; + } + assert.equal((yield* Ref.get(state)).interruptCount, 1); + // The losing detach joined the pending cleanup instead of reporting + // success over still-running work. + assert.isFalse(yield* Deferred.isDone(detachReturned)); + + // The join has the same timeout as any cleanup waiter: while the + // finalizer stays gated, detach fails rather than returning early. + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(detaching))._tag, "Failure"); + assert.equal((yield* Fiber.join(closing))._tag, "Failure"); + + yield* Deferred.succeed(closeGate, undefined); + yield* manager.close(providerSessionId); + assert.equal((yield* Ref.get(state)).closeCount, 1); + }).pipe( + Effect.ensuring( + Deferred.succeed(interruptGate, undefined).pipe( + Effect.andThen(Deferred.succeed(closeGate, undefined)), + ), + ), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + beforeInterrupt: Deferred.succeed(interruptEntered, undefined).pipe( + Effect.andThen(Deferred.await(interruptGate)), + ), + beforeClose: Deferred.succeed(closeEntered, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 blocks runtime thread attachments while a peer cleanup is pending", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const closeEntered = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const threadId = ThreadId.make("thread-runtime-attach-blocked"); + const peerThreadId = ThreadId.make("thread-runtime-attach-peer"); + const first = yield* makeThreadSessionFixture(threadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const firstId = yield* first.allocate; + const peerId = yield* peer.allocate; + yield* first.open(firstId); + const peerRuntime = yield* peer.open(peerId); + const closing = yield* manager.close(firstId).pipe(Effect.result, Effect.forkChild); + yield* Deferred.await(closeEntered); + + // Attaching the closing session's thread through a live peer runtime + // must fail like a blocked `open`, not slip past `observeActivity`. + const error = yield* peerRuntime + .ensureThread({ threadId, modelSelection, runtimePolicy }) + .pipe(Effect.flip); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + assert.include( + error._tag === "ProviderAdapterProtocolError" ? error.detail : "", + "not finished cleanup", + ); + + yield* Deferred.succeed(closeGate, undefined); + assert.equal((yield* Fiber.join(closing))._tag, "Success"); + }).pipe( + Effect.ensuring(Deferred.succeed(closeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + beforeClose: Deferred.succeed(closeEntered, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps a busy peer busy when a blocked startTurn is rejected", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const closeEntered = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const projectionStore = yield* ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-busy-peer-blocked"); + const peerThreadId = ThreadId.make("thread-busy-peer"); + const first = yield* makeThreadSessionFixture(threadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const firstId = yield* first.allocate; + const peerId = yield* peer.allocate; + yield* first.open(firstId); + const peerRuntime = yield* peer.open(peerId); + yield* peerRuntime.events.pipe(Stream.runDrain, Effect.forkScoped); + const peerProviderThread = makeProviderThread({ + idAllocator, + threadId: peerThreadId, + providerSessionId: peerId, + now, + }); + const peerAppThread = (yield* projectionStore.getThreadProjection(peerThreadId)).thread; + const peerRunId = idAllocator.derive.run({ threadId: peerThreadId, ordinal: 1 }); + + // The peer turn never terminates, so the session must stay busy and + // ineligible for idle release. + yield* peerRuntime.startTurn({ + appThread: peerAppThread, + threadId: peerThreadId, + runId: peerRunId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId: peerRunId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId: peerRunId }), + providerThread: peerProviderThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId: peerThreadId, ordinal: 1 }), + text: "keep me busy", + attachments: [], + }, + modelSelection, + runtimePolicy, + }); + + const closing = yield* manager.close(firstId).pipe(Effect.result, Effect.forkChild); + yield* Deferred.await(closeEntered); + + const blockedRunId = idAllocator.derive.run({ threadId, ordinal: 1 }); + const error = yield* peerRuntime + .startTurn({ + appThread: peerAppThread, + threadId, + runId: blockedRunId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId: blockedRunId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId: blockedRunId }), + providerThread: makeProviderThread({ + idAllocator, + threadId, + providerSessionId: firstId, + now, + }), + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "blocked", + attachments: [], + }, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.flip); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + + yield* Deferred.succeed(closeGate, undefined); + assert.equal((yield* Fiber.join(closing))._tag, "Success"); + assert.equal((yield* Ref.get(state)).closeCount, 1); + + // A rejected attachment must not run the busy/idle pipeline: if it + // decremented the peer's busy count, the idle release scheduled past + // this point would close the peer mid-turn. + yield* TestClock.adjust("2 seconds"); + yield* Effect.yieldNow; + assert.isTrue(Option.isSome(yield* manager.get(peerId))); + assert.equal((yield* Ref.get(state)).closeCount, 1); + }).pipe( + Effect.ensuring(Deferred.succeed(closeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 1_000, + beforeClose: Deferred.succeed(closeEntered, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + ), + }), + ), + ); + }), +); +it.effect("ProviderSessionManagerV2 rejects runtime thread attachments on a released session", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const threadId = ThreadId.make("thread-released-attach"); + const otherThreadId = ThreadId.make("thread-released-attach-other"); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* allocate; + const runtime = yield* open(providerSessionId); + yield* manager.close(providerSessionId); + assert.equal((yield* Ref.get(state)).closeCount, 1); + + // The session entry is gone and cleanup finished: a stale runtime + // must not hand work to the released provider process. + const error = yield* runtime + .ensureThread({ threadId: otherThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.flip); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + assert.include( + error._tag === "ProviderAdapterProtocolError" ? error.detail : "", + "no longer running", + ); + }).pipe(Effect.provide(makeTestLayer({ state, idleTimeoutMs: 3_600_000 }))); + }), +); + +it.effect( + "ProviderSessionManagerV2 rejects a runtime attachment whose session closed mid-prepare", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const preparing = yield* Deferred.make(); + const prepareGate = yield* Deferred.make(); + const pauseSettings = yield* Ref.make(false); + const issuedCredentials = yield* Ref.make< + ReadonlyArray + >([]); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (input) => + delegate + .issue(input) + .pipe( + Effect.tap((issued) => + Ref.update(issuedCredentials, (configs) => [...configs, issued.config]), + ), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + const settingsLayer = Layer.effect( + ServerSettings.ServerSettingsService, + Effect.gen(function* () { + const delegate = yield* ServerSettings.ServerSettingsService; + return ServerSettings.ServerSettingsService.of({ + ...delegate, + getSettings: Ref.get(pauseSettings).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(prepareGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.getSettings), + ), + }); + }), + ).pipe(Layer.provide(ServerSettings.layerTest({ enableAgentBrowserAccess: true }))); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const threadId = ThreadId.make("thread-runtime-prepare-close-owner"); + const attachingThreadId = ThreadId.make("thread-runtime-prepare-close"); + const first = yield* makeThreadSessionFixture(threadId); + yield* makeThreadSessionFixture(attachingThreadId); + const providerSessionId = yield* first.allocate; + const runtime = yield* first.open(providerSessionId); + yield* Ref.set(pauseSettings, true); + // The attach records, then suspends inside prepareMcpSession. The + // adapter's ensureThread is unimplemented and dies if reached, so a + // clean ProviderAdapterProtocolError proves it was never called. + const attaching = yield* runtime + .ensureThread({ threadId: attachingThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(preparing); + yield* manager.close(providerSessionId); + assert.equal((yield* Ref.get(state)).closeCount, 1); + yield* Deferred.succeed(prepareGate, undefined); + const error = yield* Fiber.join(attaching); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + assert.include( + error._tag === "ProviderAdapterProtocolError" ? error.detail : "", + "no longer running", + ); + const issued = (yield* Ref.get(issuedCredentials)).find( + (config) => config.threadId === attachingThreadId, + ); + assert.isDefined(issued); + assert.isUndefined( + yield* registry.resolve(issued!.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.isUndefined(McpProviderSession.readMcpProviderSession(attachingThreadId)); + }).pipe( + Effect.ensuring(Deferred.succeed(prepareGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + serverSettingsLayer: settingsLayer, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 waits for an in-flight runtime ensureThread before closing the session scope", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const adapterEntered = yield* Deferred.make(); + const adapterRelease = yield* Deferred.make(); + const sessionIdSlot = yield* Ref.make(undefined); + const allocatorSlot = yield* Ref.make(undefined); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + yield* Ref.set(allocatorSlot, yield* IdAllocatorV2); + const threadId = ThreadId.make("thread-ensure-drain-owner"); + const attachingThreadId = ThreadId.make("thread-ensure-drain"); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + yield* makeThreadSessionFixture(attachingThreadId); + const providerSessionId = yield* allocate; + yield* Ref.set(sessionIdSlot, providerSessionId); + const runtime = yield* open(providerSessionId); + // The adapter call is parked mid-flight after passing the ownership + // recheck — the window where a resource-creating operation (Cursor's + // awaited runner.open) has not yet installed its result. + const ensuring = yield* runtime + .ensureThread({ threadId: attachingThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.forkChild); + yield* Deferred.await(adapterEntered); + const closing = yield* manager.close(providerSessionId).pipe(Effect.forkChild); + // The release must drain the admitted adapter operation before the + // session scope closes: no resource may be installed after the + // finalizer that would have closed it. Give the close fiber enough + // turns to reach its park point — under the fix it stays parked on + // the attach lock the in-flight ensureThread holds. + for (let round = 0; round < 64 && closing.pollUnsafe() === undefined; round++) { + yield* Effect.yieldNow; + } + const closedEarly = closing.pollUnsafe(); + const closesWhileParked = (yield* Ref.get(state)).closeCount; + yield* Deferred.succeed(adapterRelease, undefined); + const ensuredExit = yield* Fiber.await(ensuring); + const closingExit = yield* Fiber.join(closing); + assert.isUndefined(closedEarly); + assert.equal(closesWhileParked, 0); + assert.isTrue(Exit.isSuccess(ensuredExit)); + assert.equal(closingExit, undefined); + assert.equal((yield* Ref.get(state)).closeCount, 1); + }).pipe( + Effect.ensuring(Deferred.succeed(adapterRelease, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + ensureThread: (threadInput) => + Effect.gen(function* () { + yield* Deferred.succeed(adapterEntered, undefined); + yield* Deferred.await(adapterRelease); + return makeProviderThread({ + idAllocator: (yield* Ref.get(allocatorSlot))!, + threadId: threadInput.threadId, + providerSessionId: (yield* Ref.get(sessionIdSlot))!, + now: yield* DateTime.now, + }); + }), + }), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 bounds the adapter-op drain for a long-lived turn call", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const turnEntered = yield* Deferred.make(); + const turnGate = yield* Deferred.make(); + const closes = yield* Ref.make(0); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const projectionStore = yield* ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-drain-bound"); + const first = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* first.allocate; + const firstRuntime = yield* first.open(providerSessionId); + yield* firstRuntime.events.pipe(Stream.runDrain, Effect.forkScoped); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + const appThread = (yield* projectionStore.getThreadProjection(threadId)).thread; + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + // The admitted turn op parks inside the adapter call — the OpenCode + // shape where the whole operation (session.summarize) runs inside the + // call rather than being forked into the session scope. The fixture + // reports the opencode driver so the in-band drain bound applies. + const turn = yield* firstRuntime + .startTurn({ + appThread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "long lived", + attachments: [], + }, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(turnEntered); + const closing = yield* manager.close(providerSessionId).pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 16; i += 1) yield* Effect.yieldNow; + // The drain waits on the admitted op, but only up to its bound: a + // long-lived turn call must not wedge the release. + assert.isUndefined(closing.pollUnsafe()); + yield* TestClock.adjust("19 seconds"); + for (let i = 0; i < 8; i += 1) yield* Effect.yieldNow; + assert.isUndefined(closing.pollUnsafe()); + yield* TestClock.adjust("2 seconds"); + for (let i = 0; i < 16; i += 1) yield* Effect.yieldNow; + const closeExit = closing.pollUnsafe(); + assert.isDefined(closeExit); + assert.isTrue( + closeExit !== undefined && Exit.isSuccess(closeExit) && Exit.isSuccess(closeExit.value), + ); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + assert.equal(yield* Ref.get(closes), 1); + // The abandoned op is still parked — its uninterruptible acquisition + // outlives the scope close and settles honestly when released. + assert.isUndefined(turn.pollUnsafe()); + yield* Deferred.succeed(turnGate, undefined); + yield* Fiber.await(turn); + }).pipe( + Effect.ensuring(Deferred.succeed(turnGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + driver: ProviderDriverKind.make("opencode"), + beforeClose: Ref.getAndUpdate(closes, (n) => n + 1), + startTurn: () => + Deferred.succeed(turnEntered, undefined).pipe(Effect.andThen(Deferred.await(turnGate))), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 does not serialize a replacement behind an abandoned in-band call's attach lock", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const turnEntered = yield* Deferred.make(); + const turnGate = yield* Deferred.make(); + const secondTurnEntered = yield* Deferred.make(); + const closes = yield* Ref.make(0); + const startTurnCalls = yield* Ref.make(0); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const projectionStore = yield* ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-drain-lock"); + const first = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* first.allocate; + const firstRuntime = yield* first.open(providerSessionId); + yield* firstRuntime.events.pipe(Stream.runDrain, Effect.forkScoped); + const appThread = (yield* projectionStore.getThreadProjection(threadId)).thread; + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + // The first turn call parks inside the adapter (in-band OpenCode + // shape) still holding its runtime's attach lock. + const turn = yield* firstRuntime + .startTurn({ + appThread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread: makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }), + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "long lived", + attachments: [], + }, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(turnEntered); + const closing = yield* manager.close(providerSessionId).pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("25 seconds"); + for (let i = 0; i < 16; i += 1) yield* Effect.yieldNow; + const closeExit = closing.pollUnsafe(); + assert.isDefined(closeExit); + assert.isTrue( + closeExit !== undefined && Exit.isSuccess(closeExit) && Exit.isSuccess(closeExit.value), + ); + // The abandoned call still holds its lock, but a same-id reopen + // installs a new runtime — the replacement's turn on the same + // thread must not wait behind a lock the dead runtime may never + // release. + const replacementRuntime = yield* first.open(providerSessionId); + yield* replacementRuntime.events.pipe(Stream.runDrain, Effect.forkScoped); + const secondRunId = idAllocator.derive.run({ threadId, ordinal: 2 }); + const secondTurn = yield* replacementRuntime + .startTurn({ + appThread, + threadId, + runId: secondRunId, + runOrdinal: 2, + providerTurnOrdinal: 2, + attemptId: idAllocator.derive.runAttempt({ + runId: secondRunId, + attemptOrdinal: 1, + }), + rootNodeId: idAllocator.derive.rootNode({ runId: secondRunId }), + providerThread: makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }), + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 2 }), + text: "replacement turn", + attachments: [], + }, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.forkChild); + for (let i = 0; i < 24; i += 1) yield* Effect.yieldNow; + yield* Deferred.await(secondTurnEntered); + yield* Fiber.await(secondTurn); + // The original abandoned call is still parked and settles honestly + // once released. + assert.isUndefined(turn.pollUnsafe()); + yield* Deferred.succeed(turnGate, undefined); + yield* Fiber.await(turn); + }).pipe( + Effect.ensuring(Deferred.succeed(turnGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + driver: ProviderDriverKind.make("opencode"), + beforeClose: Ref.getAndUpdate(closes, (n) => n + 1), + startTurn: () => + Ref.getAndUpdate(startTurnCalls, (n) => n + 1).pipe( + Effect.flatMap((n) => + n === 0 + ? Deferred.succeed(turnEntered, undefined).pipe( + Effect.andThen(Deferred.await(turnGate)), + ) + : Deferred.succeed(secondTurnEntered, undefined), + ), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 does not abandon an acquire-then-return turn call at the drain bound", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const turnEntered = yield* Deferred.make(); + const turnGate = yield* Deferred.make(); + const closes = yield* Ref.make(0); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const projectionStore = yield* ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-drain-acquire"); + const first = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* first.allocate; + const firstRuntime = yield* first.open(providerSessionId); + yield* firstRuntime.events.pipe(Stream.runDrain, Effect.forkScoped); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + const appThread = (yield* projectionStore.getThreadProjection(threadId)).thread; + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + // Same parked-call shape as the bounded test, but on a driver whose + // turn call is acquire-then-return (Cursor's runner.open window): + // abandoning it would let the acquisition install a resource after + // the scope close has finished checking for one. + const turn = yield* firstRuntime + .startTurn({ + appThread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "acquiring", + attachments: [], + }, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(turnEntered); + const closing = yield* manager.close(providerSessionId).pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 16; i += 1) yield* Effect.yieldNow; + assert.isUndefined(closing.pollUnsafe()); + // Past the in-band drain bound: an acquire-then-return call is never + // abandoned, so the release is still holding the scope open. + yield* TestClock.adjust("25 seconds"); + for (let i = 0; i < 16; i += 1) yield* Effect.yieldNow; + assert.isUndefined(closing.pollUnsafe()); + assert.equal(yield* Ref.get(closes), 0); + // The caller's own wait is bounded — close reports the cleanup + // timeout — but the detached worker still owns the session: the + // scope close has not run and a same-id replacement is refused. + yield* TestClock.adjust("40 seconds"); + for (let i = 0; i < 16; i += 1) yield* Effect.yieldNow; + const closeExit = closing.pollUnsafe(); + assert.isDefined(closeExit); + assert.isTrue( + closeExit !== undefined && Exit.isSuccess(closeExit) && Exit.isFailure(closeExit.value), + ); + assert.equal(yield* Ref.get(closes), 0); + assert.isTrue(Exit.isFailure(yield* first.open(providerSessionId).pipe(Effect.exit))); + // Once the acquisition settles, the drain completes and cleanup + // finishes honestly after the caller's bounded wait — the scope + // close runs and a same-id replacement can open. + yield* Deferred.succeed(turnGate, undefined); + yield* Fiber.await(turn); + for (let i = 0; i < 24; i += 1) yield* Effect.yieldNow; + assert.equal(yield* Ref.get(closes), 1); + const replacement = yield* first.open(providerSessionId); + yield* replacement.events.pipe(Stream.runDrain, Effect.forkScoped); + }).pipe( + Effect.ensuring(Deferred.succeed(turnGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + beforeClose: Ref.getAndUpdate(closes, (n) => n + 1), + startTurn: () => + Deferred.succeed(turnEntered, undefined).pipe( + Effect.andThen(Deferred.await(turnGate)), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 rejects a runtime attachment when its session is replaced mid-prepare", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const preparing = yield* Deferred.make(); + const prepareGate = yield* Deferred.make(); + const pauseSettings = yield* Ref.make(false); + const issuedCredentials = yield* Ref.make< + ReadonlyArray + >([]); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (input) => + delegate + .issue(input) + .pipe( + Effect.tap((issued) => + Ref.update(issuedCredentials, (configs) => [...configs, issued.config]), + ), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + const settingsLayer = Layer.effect( + ServerSettings.ServerSettingsService, + Effect.gen(function* () { + const delegate = yield* ServerSettings.ServerSettingsService; + return ServerSettings.ServerSettingsService.of({ + ...delegate, + // Single-shot pause: the reopening open() must still read settings. + getSettings: Ref.getAndSet(pauseSettings, false).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(prepareGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.getSettings), + ), + }); + }), + ).pipe(Layer.provide(ServerSettings.layerTest({ enableAgentBrowserAccess: true }))); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const threadId = ThreadId.make("thread-runtime-replace-owner"); + const attachingThreadId = ThreadId.make("thread-runtime-replace-attach"); + const first = yield* makeThreadSessionFixture(threadId); + yield* makeThreadSessionFixture(attachingThreadId); + const providerSessionId = yield* first.allocate; + const runtime = yield* first.open(providerSessionId); + yield* Ref.set(pauseSettings, true); + const attaching = yield* runtime + .ensureThread({ threadId: attachingThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(preparing); + yield* manager.close(providerSessionId); + // Multi-thread providers derive a deterministic shared session id, so + // a reopen legitimately reuses it. The replacement must not claim the + // suspended attachment. + yield* first.open(providerSessionId); + yield* Deferred.succeed(prepareGate, undefined); + const error = yield* Fiber.join(attaching); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + assert.include( + error._tag === "ProviderAdapterProtocolError" ? error.detail : "", + "no longer running", + ); + const issued = (yield* Ref.get(issuedCredentials)).find( + (config) => config.threadId === attachingThreadId, + ); + assert.isDefined(issued); + assert.isUndefined( + yield* registry.resolve(issued!.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + }).pipe( + Effect.ensuring(Deferred.succeed(prepareGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + serverSettingsLayer: settingsLayer, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 revokes an abandoned reused credential when the last holder closes", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const resolving = yield* Deferred.make(); + const resolveGate = yield* Deferred.make(); + const pauseResolve = yield* Ref.make(false); + const issuedCredentials = yield* Ref.make< + ReadonlyArray + >([]); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (input) => + delegate + .issue(input) + .pipe( + Effect.tap((issued) => + Ref.update(issuedCredentials, (configs) => [...configs, issued.config]), + ), + ), + resolve: (token) => + Ref.get(pauseResolve).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(resolving, undefined).pipe( + Effect.andThen(Deferred.await(resolveGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.resolve(token)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const ownerThreadId = ThreadId.make("thread-abandoned-reused-owner"); + const attachingThreadId = ThreadId.make("thread-abandoned-reused-peer"); + const owner = yield* makeThreadSessionFixture(ownerThreadId); + const peer = yield* makeThreadSessionFixture(attachingThreadId); + const peerId = yield* peer.allocate; + yield* peer.open(peerId); + const peerCredential = McpProviderSession.readMcpProviderSession(attachingThreadId); + assert.isDefined(peerCredential); + const ownerId = yield* owner.allocate; + const runtime = yield* owner.open(ownerId); + yield* Ref.set(pauseResolve, true); + // The attach reuses the peer's credential, reserves it, then suspends + // on resolve. Closing both sessions leaves the credential owned only + // by that reservation; abandoning the attach must revoke it. + const attaching = yield* runtime + .ensureThread({ threadId: attachingThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(resolving); + // The closes' credential sweeps park on the prepare lock the gated + // attach still holds; fork them so the gate can be released. + const peerClosing = yield* manager.close(peerId).pipe(Effect.forkChild); + const ownerClosing = yield* manager.close(ownerId).pipe(Effect.forkChild); + yield* Effect.forEach(Array.from({ length: 8 }), () => Effect.yieldNow, { + discard: true, + }); + assert.equal((yield* Ref.get(state)).closeCount, 2); + yield* Deferred.succeed(resolveGate, undefined); + const error = yield* Fiber.join(attaching); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + yield* Fiber.join(peerClosing); + yield* Fiber.join(ownerClosing); + const token = peerCredential!.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isUndefined(yield* registry.resolve(token)); + assert.isUndefined(McpProviderSession.readMcpProviderSession(attachingThreadId)); + }).pipe( + Effect.ensuring(Deferred.succeed(resolveGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 revokes a reused credential when resolution is interrupted after the last holder closed", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const resolving = yield* Deferred.make(); + const resolveGate = yield* Deferred.make(); + const pauseResolve = yield* Ref.make(false); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + resolve: (token) => + Ref.get(pauseResolve).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(resolving, undefined).pipe( + Effect.andThen(Deferred.await(resolveGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.resolve(token)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const ownerThreadId = ThreadId.make("thread-failed-resolve-owner"); + const attachingThreadId = ThreadId.make("thread-failed-resolve-peer"); + const owner = yield* makeThreadSessionFixture(ownerThreadId); + const peer = yield* makeThreadSessionFixture(attachingThreadId); + const peerId = yield* peer.allocate; + yield* peer.open(peerId); + const peerCredential = McpProviderSession.readMcpProviderSession(attachingThreadId); + assert.isDefined(peerCredential); + const ownerId = yield* owner.allocate; + const runtime = yield* owner.open(ownerId); + yield* Ref.set(pauseResolve, true); + // The attach reserves the peer's credential, then suspends inside the + // resolve. Closing the credential holder leaves the reservation plus + // the attaching session's provisional thread attachment as the only + // claims; interrupting the attach must unwind the provisional + // attachment and not leave the credential valid and unclaimed once + // the attaching session closes too. + const attaching = yield* runtime + .ensureThread({ threadId: attachingThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(resolving); + // The close's credential sweep parks on the prepare lock the gated + // attach still holds; fork it so the interrupt can unwind the attach. + const peerClosing = yield* manager.close(peerId).pipe(Effect.forkChild); + yield* Fiber.interrupt(attaching); + const exit = yield* Fiber.await(attaching); + const ownerClosing = yield* manager.close(ownerId).pipe(Effect.forkChild); + yield* Fiber.join(peerClosing); + yield* Fiber.join(ownerClosing); + assert.equal((yield* Ref.get(state)).closeCount, 2); + assert.equal(exit._tag, "Failure"); + yield* Ref.set(pauseResolve, false); + const token = peerCredential!.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isUndefined(yield* registry.resolve(token)); + assert.isUndefined(McpProviderSession.readMcpProviderSession(attachingThreadId)); + }).pipe( + Effect.ensuring(Deferred.succeed(resolveGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 rejects a stale runtime whose same-id session was replaced", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const ownerThreadId = ThreadId.make("thread-stale-runtime-owner"); + const attachingThreadId = ThreadId.make("thread-stale-runtime-attach"); + const owner = yield* makeThreadSessionFixture(ownerThreadId); + const attaching = yield* makeThreadSessionFixture(attachingThreadId); + const providerSessionId = yield* owner.allocate; + const runtimeA = yield* owner.open(providerSessionId); + yield* attaching.open(providerSessionId); + yield* manager.close(providerSessionId); + // Multi-thread providers derive a deterministic shared session id, so + // the reopen legitimately reuses it and re-attaches the same thread. + yield* owner.open(providerSessionId); + yield* attaching.open(providerSessionId); + // A's stale handle must not reach the dead adapter through B's + // already-attached thread record. + const error = yield* runtimeA + .ensureThread({ threadId: attachingThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.flip); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + assert.include( + error._tag === "ProviderAdapterProtocolError" ? error.detail : "", + "no longer running", + ); + // Non-attach entry points reject the stale runtime too instead of + // invoking the closed adapter. + const idAllocator = yield* IdAllocatorV2; + const interruptError = yield* runtimeA + .interruptTurn({ + providerThread: makeProviderThread({ + idAllocator, + threadId: attachingThreadId, + providerSessionId, + now: yield* DateTime.now, + }), + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "stale-turn", + }), + }) + .pipe(Effect.flip); + assert.equal(interruptError._tag, "ProviderAdapterProtocolError"); + assert.equal((yield* Ref.get(state)).interruptCount, 0); + const providerThread = makeProviderThread({ + idAllocator, + threadId: attachingThreadId, + providerSessionId, + now: yield* DateTime.now, + }); + // The spread-through entry points (snapshot, rollback, null-app-thread + // resume) reject the stale handle too — a clean protocol error proves + // the closed adapter was never invoked. + const snapshotError = yield* runtimeA + .readThreadSnapshot({ providerThread }) + .pipe(Effect.flip); + assert.equal(snapshotError._tag, "ProviderAdapterProtocolError"); + const orphanResumeError = yield* runtimeA + .resumeThread({ providerThread: { ...providerThread, appThreadId: null } }) + .pipe(Effect.flip); + assert.equal(orphanResumeError._tag, "ProviderAdapterProtocolError"); + assert.equal((yield* Ref.get(state)).resumeCount, 0); + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + assert.equal((yield* Ref.get(state)).closeCount, 1); + }).pipe(Effect.provide(makeTestLayer({ state, idleTimeoutMs: 3_600_000 }))); + }), +); + +it.effect( + "ProviderSessionManagerV2 rejects a runtime attachment whose session closed mid-event-write", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const writing = yield* Deferred.make(); + const writeGate = yield* Deferred.make(); + const closeReached = yield* Deferred.make(); + const pauseWrite = yield* Ref.make(false); + const issuedCredentials = yield* Ref.make< + ReadonlyArray + >([]); + const ownerThreadId = ThreadId.make("thread-event-write-owner"); + const attachingThreadId = ThreadId.make("thread-event-write-attach"); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (input) => + delegate + .issue(input) + .pipe( + Effect.tap((issued) => + Ref.update(issuedCredentials, (configs) => [...configs, issued.config]), + ), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + const writtenEventTypes = yield* Ref.make< + ReadonlyArray<{ readonly threadId: string; readonly type: string }> + >([]); + const eventSinkLayer = Layer.effect( + EventSinkV2, + Effect.gen(function* () { + const delegate = yield* EventSinkV2; + return EventSinkV2.of({ + ...delegate, + write: (input) => + Ref.get(pauseWrite).pipe( + Effect.flatMap((pause) => + pause && + input.events.some( + (event) => + event.type === "provider-session.attached" && + event.threadId === attachingThreadId, + ) + ? Deferred.succeed(writing, undefined).pipe( + Effect.andThen(Deferred.await(writeGate)), + ) + : Effect.void, + ), + Effect.andThen( + Ref.update(writtenEventTypes, (types) => [ + ...types, + ...input.events.map((event) => ({ + threadId: event.threadId, + type: event.type, + })), + ]), + ), + Effect.andThen(delegate.write(input)), + ), + }); + }), + ).pipe(Layer.provide(TestEventSinkLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const owner = yield* makeThreadSessionFixture(ownerThreadId); + yield* makeThreadSessionFixture(attachingThreadId); + const providerSessionId = yield* owner.allocate; + const runtime = yield* owner.open(providerSessionId); + yield* Ref.set(pauseWrite, true); + // The attach records the credential, then suspends persisting the + // attachment event. The adapter's ensureThread is unimplemented and + // dies if reached, so a clean error proves it was never called. + const attaching = yield* runtime + .ensureThread({ threadId: attachingThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(writing); + // beforeClose runs inside scope cleanup, after the release claimed the + // session: awaiting it here guarantees the post-write recheck observes + // the pending release. The suspended attach write also holds the + // release-status lock, so the close's terminal event report queues + // behind it until the gate opens. + const closing = yield* manager.close(providerSessionId).pipe(Effect.forkChild); + yield* Deferred.await(closeReached); + yield* Deferred.succeed(writeGate, undefined); + const error = yield* Fiber.join(attaching); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + assert.match( + error._tag === "ProviderAdapterProtocolError" ? error.detail : "", + /not finished cleanup|no longer running/, + ); + yield* Fiber.join(closing); + assert.equal((yield* Ref.get(state)).closeCount, 1); + // The gated attach event still lands strictly before the release's + // terminal status write. + const eventTypes = yield* Ref.get(writtenEventTypes); + const attachedIndex = eventTypes.findIndex( + (event) => + event.threadId === attachingThreadId && event.type === "provider-session.attached", + ); + const releasedIndex = eventTypes.findIndex( + (event) => event.type === "provider-session.updated", + ); + assert.isAtLeast(attachedIndex, 0); + assert.isAtLeast(releasedIndex, 0); + assert.isBelow(attachedIndex, releasedIndex); + const issued = (yield* Ref.get(issuedCredentials)).find( + (config) => config.threadId === attachingThreadId, + ); + assert.isDefined(issued); + assert.isUndefined( + yield* registry.resolve(issued!.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.isUndefined(McpProviderSession.readMcpProviderSession(attachingThreadId)); + }).pipe( + Effect.ensuring(Deferred.succeed(writeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpRegistryLayer, + eventSinkLayer, + beforeClose: Deferred.succeed(closeReached, undefined), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 rejects a runtime attachment whose attach event write fails after close", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const writing = yield* Deferred.make(); + const writeGate = yield* Deferred.make(); + const closeReached = yield* Deferred.make(); + const pauseWrite = yield* Ref.make(false); + const ownerThreadId = ThreadId.make("thread-event-write-fail-owner"); + const attachingThreadId = ThreadId.make("thread-event-write-fail-attach"); + const eventSinkLayer = Layer.effect( + EventSinkV2, + Effect.gen(function* () { + const delegate = yield* EventSinkV2; + return EventSinkV2.of({ + ...delegate, + write: (input) => + Ref.get(pauseWrite).pipe( + Effect.flatMap((pause) => + pause && + input.events.some( + (event) => + event.type === "provider-session.attached" && + event.threadId === attachingThreadId, + ) + ? Deferred.succeed(writing, undefined).pipe( + Effect.andThen(Deferred.await(writeGate)), + Effect.andThen( + Effect.fail(new EventSinkWriteError({ eventCount: input.events.length })), + ), + ) + : Effect.void, + ), + Effect.andThen(delegate.write(input)), + ), + }); + }), + ).pipe(Layer.provide(TestEventSinkLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const owner = yield* makeThreadSessionFixture(ownerThreadId); + yield* makeThreadSessionFixture(attachingThreadId); + const providerSessionId = yield* owner.allocate; + const runtime = yield* owner.open(providerSessionId); + yield* Ref.set(pauseWrite, true); + // The attach suspends inside the event write holding the + // release-status lock; once the gate opens the write fails while the + // close is already pending. The failed write must still surface the + // ownership loss instead of invoking the closing adapter (the + // adapter's ensureThread is unimplemented and dies if reached). + const attaching = yield* runtime + .ensureThread({ threadId: attachingThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(writing); + const closing = yield* manager.close(providerSessionId).pipe(Effect.forkChild); + yield* Deferred.await(closeReached); + yield* Deferred.succeed(writeGate, undefined); + const error = yield* Fiber.join(attaching); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + assert.match( + error._tag === "ProviderAdapterProtocolError" ? error.detail : "", + /not finished cleanup|no longer running/, + ); + yield* Fiber.join(closing); + assert.equal((yield* Ref.get(state)).closeCount, 1); + assert.isUndefined(McpProviderSession.readMcpProviderSession(attachingThreadId)); + }).pipe( + Effect.ensuring(Deferred.succeed(writeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + eventSinkLayer, + beforeClose: Deferred.succeed(closeReached, undefined), + }), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 fails open when the session is claimed mid-attach-write", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const writing = yield* Deferred.make(); + const writeGate = yield* Deferred.make(); + const pauseWrite = yield* Ref.make(false); + const threadId = ThreadId.make("thread-open-mid-write-close"); + const writtenEventTypes = yield* Ref.make< + ReadonlyArray<{ readonly threadId: string; readonly type: string }> + >([]); + const eventSinkLayer = Layer.effect( + EventSinkV2, + Effect.gen(function* () { + const delegate = yield* EventSinkV2; + return EventSinkV2.of({ + ...delegate, + write: (input) => + Ref.get(pauseWrite).pipe( + Effect.flatMap((pause) => + pause && + input.events.some( + (event) => + event.type === "provider-session.attached" && event.threadId === threadId, + ) + ? Deferred.succeed(writing, undefined).pipe( + Effect.andThen(Deferred.await(writeGate)), + ) + : Effect.void, + ), + Effect.andThen( + Ref.update(writtenEventTypes, (types) => [ + ...types, + ...input.events.map((event) => ({ + threadId: event.threadId, + type: event.type, + })), + ]), + ), + Effect.andThen(delegate.write(input)), + ), + }); + }), + ).pipe(Layer.provide(TestEventSinkLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const { allocate } = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* allocate; + yield* Ref.set(pauseWrite, true); + const opening = yield* manager + .open({ threadId, providerSessionId, modelSelection, runtimePolicy }) + .pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(writing); + // The suspended write holds the sessionOpen and release-status locks; + // the close claims the session and then parks waiting for them. + const closing = yield* manager.close(providerSessionId).pipe(Effect.result, Effect.forkChild); + for (let i = 0; i < 10; i += 1) { + yield* Effect.yieldNow; + } + // Keeping the write gated past the timeout must still let the close + // caller fail: the error-status report is ordered behind the attach + // write through the release-status lock but does not block the + // caller's bounded wait. + yield* TestClock.adjust("30 seconds"); + const closeExit = yield* Fiber.join(closing); + assert.equal(closeExit._tag, "Failure"); + yield* Deferred.succeed(writeGate, undefined); + const error = yield* Fiber.join(opening); + assert.equal(error._tag, "ProviderSessionOpenError"); + // The claimed cleanup still owns teardown; a retried close joins it. + yield* manager.close(providerSessionId); + assert.equal((yield* Ref.get(state)).closeCount, 1); + // The attach event still lands strictly before the release's terminal + // status write — never after it. + const eventTypes = yield* Ref.get(writtenEventTypes); + const attachedIndex = eventTypes.findIndex( + (event) => event.threadId === threadId && event.type === "provider-session.attached", + ); + const releasedIndex = eventTypes.findIndex( + (event) => event.type === "provider-session.updated", + ); + assert.isAtLeast(attachedIndex, 0); + assert.isAtLeast(releasedIndex, 0); + assert.isBelow(attachedIndex, releasedIndex); + }).pipe( + Effect.ensuring(Deferred.succeed(writeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + eventSinkLayer, + }), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 unwinds an open interrupted inside openSession", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const opening = yield* Deferred.make(); + const openGate = yield* Deferred.make(); + const opens = yield* Ref.make(0); + yield* Effect.gen(function* () { + const threadId = ThreadId.make("thread-interrupted-open"); + const fixture = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* fixture.allocate; + const openingFiber = yield* fixture + .open(providerSessionId) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(opening); + // The open minted a credential and the adapter registered its scope + // finalizer, but no entry exists in `sessions` yet — interruption must + // close the scope, drop the reservation, and revoke the unclaimed + // credential. + yield* Fiber.interrupt(openingFiber); + const exit = yield* Fiber.await(openingFiber); + assert.equal(exit._tag, "Failure"); + assert.equal((yield* Ref.get(state)).closeCount, 1); + assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); + // No phantom ownership: a fresh open of the same session id succeeds. + yield* fixture.open(providerSessionId); + assert.equal((yield* Ref.get(state)).openCount, 2); + }).pipe( + Effect.ensuring(Deferred.succeed(openGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + afterOpen: Ref.getAndUpdate(opens, (n) => n + 1).pipe( + Effect.flatMap((n) => + n === 0 + ? Deferred.succeed(opening, undefined).pipe( + Effect.andThen(Deferred.await(openGate)), + ) + : Effect.void, + ), + ), + }), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 serializes a concurrent attach behind an in-flight one", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const writing = yield* Deferred.make(); + const writeGate = yield* Deferred.make(); + const pauseWrite = yield* Ref.make(true); + const secondDone = yield* Ref.make(false); + const ownerThreadId = ThreadId.make("thread-attach-serial-owner"); + const attachingThreadId = ThreadId.make("thread-attach-serial-peer"); + const attachedWrites = yield* Ref.make(0); + const eventSinkLayer = Layer.effect( + EventSinkV2, + Effect.gen(function* () { + const delegate = yield* EventSinkV2; + return EventSinkV2.of({ + ...delegate, + write: (input) => + Ref.get(pauseWrite).pipe( + Effect.flatMap((pause) => + pause && + input.events.some( + (event) => + event.type === "provider-session.attached" && + event.threadId === attachingThreadId, + ) + ? Deferred.succeed(writing, undefined).pipe( + Effect.andThen(Deferred.await(writeGate)), + ) + : Effect.void, + ), + Effect.andThen( + Ref.update(attachedWrites, (n) => + input.events.some( + (event) => + event.type === "provider-session.attached" && + event.threadId === attachingThreadId, + ) + ? n + 1 + : n, + ), + ), + Effect.andThen(delegate.write(input)), + ), + }); + }), + ).pipe(Layer.provide(TestEventSinkLayer)); + yield* Effect.gen(function* () { + const idAllocator = yield* IdAllocatorV2; + const owner = yield* makeThreadSessionFixture(ownerThreadId); + yield* makeThreadSessionFixture(attachingThreadId); + const providerSessionId = yield* owner.allocate; + const runtime = yield* owner.open(providerSessionId); + const providerThread = makeProviderThread({ + idAllocator, + threadId: attachingThreadId, + providerSessionId, + now: yield* DateTime.now, + }); + // First attach suspends persisting the attachment event while holding + // the per-(session, thread) attach lock. + const first = yield* runtime + .resumeThread({ providerThread, threadId: attachingThreadId }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(writing); + // A concurrent attach of the same thread must wait for the in-flight + // one to become durable rather than riding its provisional + // attachment: an attach that later fails unwinds the bookkeeping and + // credential claim the second caller was relying on. + const second = yield* runtime + .resumeThread({ providerThread, threadId: attachingThreadId }) + .pipe(Effect.ensuring(Ref.set(secondDone, true)), Effect.exit, Effect.forkChild); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* Effect.yieldNow; + assert.isFalse(yield* Ref.get(secondDone)); + yield* Ref.set(pauseWrite, false); + yield* Deferred.succeed(writeGate, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(first))); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(second))); + // The second caller joined the durable attachment: exactly one attach + // event was persisted, and the adapter resume ran once — the second + // call sees the loaded provider thread and short-circuits. + assert.equal(yield* Ref.get(attachedWrites), 1); + assert.equal((yield* Ref.get(state)).resumeCount, 1); + }).pipe( + Effect.ensuring(Deferred.succeed(writeGate, undefined)), + Effect.provide(makeTestLayer({ state, idleTimeoutMs: 3_600_000, eventSinkLayer })), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 releases a session whose open was interrupted after registration", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const writing = yield* Deferred.make(); + const writeGate = yield* Deferred.make(); + const pauseWrite = yield* Ref.make(true); + const threadId = ThreadId.make("thread-interrupted-registered-open"); + const eventSinkLayer = Layer.effect( + EventSinkV2, + Effect.gen(function* () { + const delegate = yield* EventSinkV2; + return EventSinkV2.of({ + ...delegate, + write: (input) => + Ref.get(pauseWrite).pipe( + Effect.flatMap((pause) => + pause && + input.events.some( + (event) => + event.type === "provider-session.attached" && event.threadId === threadId, + ) + ? Deferred.succeed(writing, undefined).pipe( + Effect.andThen(Deferred.await(writeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.write(input)), + ), + }); + }), + ).pipe(Layer.provide(TestEventSinkLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const fixture = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* fixture.allocate; + const opening = yield* fixture.open(providerSessionId).pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(writing); + // The entry is registered but the attach-event write is suspended; + // interrupting here must hand the half-initialized session to the + // release path rather than leaving it live without an event pump. + yield* Fiber.interrupt(opening); + const exit = yield* Fiber.await(opening); + assert.equal(exit._tag, "Failure"); + yield* Ref.set(pauseWrite, false); + yield* Deferred.succeed(writeGate, undefined); + // Do not call close here: the interrupted open's own unwind must hand + // the session to release, and an explicit close would perform the + // missing cleanup itself. Poll the credential sweep — the last step of + // cleanup — as the completion barrier instead. + for ( + let i = 0; + i < 24 && McpProviderSession.readMcpProviderSession(threadId) !== undefined; + i += 1 + ) { + yield* Effect.yieldNow; + } + assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + assert.equal((yield* Ref.get(state)).closeCount, 1); + }).pipe( + Effect.ensuring(Deferred.succeed(writeGate, undefined)), + Effect.provide(makeTestLayer({ state, idleTimeoutMs: 3_600_000, eventSinkLayer })), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 rejects a runtime attachment whose event write fails while live", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const writing = yield* Deferred.make(); + const writeGate = yield* Deferred.make(); + const pauseWrite = yield* Ref.make(true); + const ownerThreadId = ThreadId.make("thread-write-fail-live-owner"); + const attachingThreadId = ThreadId.make("thread-write-fail-live-attach"); + const eventSinkLayer = Layer.effect( + EventSinkV2, + Effect.gen(function* () { + const delegate = yield* EventSinkV2; + return EventSinkV2.of({ + ...delegate, + write: (input) => + Ref.get(pauseWrite).pipe( + Effect.flatMap((pause) => + pause && + input.events.some( + (event) => + event.type === "provider-session.attached" && + event.threadId === attachingThreadId, + ) + ? Deferred.succeed(writing, undefined).pipe( + Effect.andThen(Deferred.await(writeGate)), + Effect.andThen( + Effect.fail(new EventSinkWriteError({ eventCount: input.events.length })), + ), + ) + : Effect.void, + ), + Effect.andThen(delegate.write(input)), + ), + }); + }), + ).pipe(Layer.provide(TestEventSinkLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const owner = yield* makeThreadSessionFixture(ownerThreadId); + yield* makeThreadSessionFixture(attachingThreadId); + const providerSessionId = yield* owner.allocate; + const runtime = yield* owner.open(providerSessionId); + // The attach suspends on the failing event write while the session + // stays live: the caller must be rejected rather than reaching the + // adapter for an untracked attachment. The adapter's ensureThread is + // unimplemented, so a clean protocol error proves it was not called. + const attaching = yield* runtime + .ensureThread({ threadId: attachingThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(writing); + yield* Deferred.succeed(writeGate, undefined); + const error = yield* Fiber.join(attaching); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + assert.include( + error._tag === "ProviderAdapterProtocolError" ? error.detail : "", + "could not attach", + ); + // The session stays live and the failed attach's credential is + // revoked — the provisional claim was unwound with nothing else + // holding it. + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + assert.isUndefined(McpProviderSession.readMcpProviderSession(attachingThreadId)); + }).pipe( + Effect.ensuring(Deferred.succeed(writeGate, undefined)), + Effect.provide(makeTestLayer({ state, idleTimeoutMs: 3_600_000, eventSinkLayer })), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 rejects a queued attachment whose thread was claimed by a pending release", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const writing = yield* Deferred.make(); + const writeGate = yield* Deferred.make(); + const closeReached = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + const pauseWrite = yield* Ref.make(false); + const oldThreadId = ThreadId.make("thread-queued-attach-old"); + const ownerThreadId = ThreadId.make("thread-queued-attach-owner"); + const attachingThreadId = oldThreadId; + const eventSinkLayer = Layer.effect( + EventSinkV2, + Effect.gen(function* () { + const delegate = yield* EventSinkV2; + return EventSinkV2.of({ + ...delegate, + write: (input) => + Ref.get(pauseWrite).pipe( + Effect.flatMap((pause) => + pause && + input.events.some( + (event) => + event.type === "provider-session.attached" && + event.threadId === attachingThreadId, + ) + ? Deferred.succeed(writing, undefined).pipe( + Effect.andThen(Deferred.await(writeGate)), + Effect.andThen( + Effect.fail(new EventSinkWriteError({ eventCount: input.events.length })), + ), + ) + : Effect.void, + ), + Effect.andThen(delegate.write(input)), + ), + }); + }), + ).pipe(Layer.provide(TestEventSinkLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const oldSession = yield* makeThreadSessionFixture(oldThreadId); + const owner = yield* makeThreadSessionFixture(ownerThreadId); + const oldSessionId = yield* oldSession.allocate; + yield* oldSession.open(oldSessionId); + const sharedSessionId = yield* owner.allocate; + const runtime = yield* owner.open(sharedSessionId); + // The first attach of the shared thread suspends on the failing event + // write while holding the per-(session, thread) attach lock. + yield* Ref.set(pauseWrite, true); + const first = yield* runtime + .ensureThread({ threadId: attachingThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(writing); + // The queued caller passed the pending-release gate before its lock + // wait; the thread's old session then starts releasing and stays + // pending behind the close gate. + const second = yield* runtime + .ensureThread({ threadId: attachingThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.flip, Effect.forkChild); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + const closing = yield* manager.close(oldSessionId).pipe(Effect.forkChild); + yield* Deferred.await(closeReached); + yield* Deferred.succeed(writeGate, undefined); + // The first attach fails its write; the queued caller must re-check + // the pending release inside the atomic attach instead of adding the + // thread anyway. + assert.equal((yield* Fiber.join(first))._tag, "Failure"); + const error = yield* Fiber.join(second); + assert.equal(error._tag, "ProviderAdapterProtocolError"); + assert.include( + error._tag === "ProviderAdapterProtocolError" ? error.detail : "", + "not finished cleanup", + ); + yield* Deferred.succeed(closeGate, undefined); + yield* Fiber.join(closing); + }).pipe( + Effect.ensuring( + Deferred.succeed(writeGate, undefined).pipe( + Effect.andThen(Deferred.succeed(closeGate, undefined)), + ), + ), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + eventSinkLayer, + beforeClose: Deferred.succeed(closeReached, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps a live session's claimed credential valid when reattach rotates it", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const browserAccess = yield* Ref.make(true); + const issuedCredentials = yield* Ref.make< + ReadonlyArray + >([]); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (input) => + delegate + .issue(input) + .pipe( + Effect.tap((issued) => + Ref.update(issuedCredentials, (configs) => [...configs, issued.config]), + ), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + const settingsLayer = Layer.effect( + ServerSettings.ServerSettingsService, + Effect.gen(function* () { + const delegate = yield* ServerSettings.ServerSettingsService; + return ServerSettings.ServerSettingsService.of({ + ...delegate, + getSettings: Ref.get(browserAccess).pipe( + Effect.flatMap((browser) => + Effect.map(delegate.getSettings, (settings) => ({ + ...settings, + enableAgentBrowserAccess: browser, + })), + ), + ), + }); + }), + ).pipe(Layer.provide(ServerSettings.layerTest({ enableAgentBrowserAccess: true }))); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const sharedThreadId = ThreadId.make("thread-rotate-claimed-shared"); + const ownerThreadId = ThreadId.make("thread-rotate-claimed-owner"); + const shared = yield* makeThreadSessionFixture(sharedThreadId); + const owner = yield* makeThreadSessionFixture(ownerThreadId); + const firstSessionId = yield* shared.allocate; + yield* shared.open(firstSessionId); + const rotated = McpProviderSession.readMcpProviderSession(sharedThreadId); + assert.isDefined(rotated); + const rotatedToken = rotated!.authorizationHeader.replace(/^Bearer\s+/, ""); + const sharedSessionId = yield* owner.allocate; + const runtime = yield* owner.open(sharedSessionId); + // Flip browser access off so the shared thread's credential no longer + // matches the required capabilities: reattaching rotates it, but the + // first session's process still holds the old token. + yield* Ref.set(browserAccess, false); + // The adapter's ensureThread is unimplemented; the credential rotation + // has already completed by the time it dies. + yield* runtime + .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.ignore); + const replacement = McpProviderSession.readMcpProviderSession(sharedThreadId); + assert.isDefined(replacement); + assert.notEqual(replacement!.providerSessionId, rotated!.providerSessionId); + // The rotated-out credential must stay valid while a live session + // still records it — its process keeps calling with it. + assert.isDefined(yield* registry.resolve(rotatedToken)); + yield* manager.close(firstSessionId); + // The first session was the rotated credential's last recorded + // holder; the second session recorded the replacement, so the release + // revokes it rather than leaving a valid unclaimed token. + assert.isUndefined(yield* registry.resolve(rotatedToken)); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpRegistryLayer, + serverSettingsLayer: settingsLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 orders a suspended session update before the release terminal write", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const writing = yield* Deferred.make(); + const writeGate = yield* Deferred.make(); + const pauseWrite = yield* Ref.make(false); + const threadId = ThreadId.make("thread-update-ordering"); + const eventSinkLayer = Layer.effect( + EventSinkV2, + Effect.gen(function* () { + const delegate = yield* EventSinkV2; + return EventSinkV2.of({ + ...delegate, + write: (input) => + Ref.get(pauseWrite).pipe( + Effect.flatMap((pause) => + pause && input.events.some((event) => event.type === "provider-session.updated") + ? Deferred.succeed(writing, undefined).pipe( + Effect.andThen(Deferred.await(writeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.write(input)), + ), + }); + }), + ).pipe(Layer.provide(TestEventSinkLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStoreV2; + const session = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* session.allocate; + const runtime = yield* session.open(providerSessionId); + yield* Ref.set(pauseWrite, true); + // The pump picks up the provider's status update and suspends inside + // its event write while holding releaseStatus; the close then queues + // its terminal write behind it instead of being overwritten. + const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(queue); + yield* Queue.offer(queue!, { + type: "provider_session.updated", + driver: CODEX_DRIVER, + providerSession: runtime.providerSession, + }); + yield* Deferred.await(writing); + const closing = yield* manager.close(providerSessionId).pipe(Effect.forkChild); + yield* Ref.set(pauseWrite, false); + yield* Deferred.succeed(writeGate, undefined); + yield* Fiber.join(closing); + const sessionStatus = (yield* projectionStore.getThreadProjection( + threadId, + )).providerSessions.at(-1)?.status; + assert.equal(sessionStatus, "stopped"); + }).pipe( + Effect.ensuring(Deferred.succeed(writeGate, undefined)), + Effect.provide(makeTestLayer({ state, idleTimeoutMs: 3_600_000, eventSinkLayer })), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 never lets a suspended detach strip a same-id replacement's attachment", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const interruptEntered = yield* Deferred.make(); + const interruptGate = yield* Deferred.make(); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const eventSink = yield* EventSinkV2; + const projectionStore = yield* ProjectionStoreV2; + const idAllocator = yield* IdAllocatorV2; + const now = yield* DateTime.now; + const firstThreadId = ThreadId.make("thread-detach-replace-first"); + const secondThreadId = ThreadId.make("thread-detach-replace-second"); + const first = yield* makeThreadSessionFixture(firstThreadId); + const second = yield* makeThreadSessionFixture(secondThreadId); + const providerSessionId = yield* first.allocate; + const providerThread = makeProviderThread({ + idAllocator, + threadId: firstThreadId, + providerSessionId, + now, + }); + const runId = idAllocator.derive.run({ threadId: firstThreadId, ordinal: 1 }); + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId: firstThreadId }), + type: "provider-thread.updated", + threadId: firstThreadId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: providerThread, + }, + { + id: yield* idAllocator.allocate.event({ threadId: firstThreadId }), + type: "provider-turn.updated", + threadId: firstThreadId, + runId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: { + id: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-detach-replace", + }), + providerThreadId: providerThread.id, + nodeId: idAllocator.derive.rootNode({ runId }), + runAttemptId: null, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }, + ], + }); + yield* first.open(providerSessionId); + yield* second.open(providerSessionId); + // The detach captures session A's entry, then parks inside + // interruptTurn on the seeded running turn. A closes completely and a + // same-id replacement re-attaches the thread while it is suspended. + const detaching = yield* manager + .detach({ providerSessionId, threadId: firstThreadId }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(interruptEntered); + yield* manager.close(providerSessionId); + yield* manager.open({ + threadId: firstThreadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + yield* Deferred.succeed(interruptGate, undefined); + assert.equal((yield* Fiber.join(detaching))._tag, "Success"); + // The replacement still owns the thread: closing it must write the + // thread's terminal session event instead of silently skipping it. + yield* manager.close(providerSessionId); + const sessionStatus = (yield* projectionStore.getThreadProjection( + firstThreadId, + )).providerSessions.at(-1)?.status; + assert.equal(sessionStatus, "stopped"); + }).pipe( + Effect.ensuring(Deferred.succeed(interruptGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + beforeInterrupt: Deferred.succeed(interruptEntered, undefined).pipe( + Effect.andThen(Deferred.await(interruptGate)), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 terminal detach keeps a credential a peer session still claims", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const sharedThreadId = ThreadId.make("thread-terminal-detach-shared"); + const ownerThreadId = ThreadId.make("thread-terminal-detach-owner"); + const shared = yield* makeThreadSessionFixture(sharedThreadId); + const owner = yield* makeThreadSessionFixture(ownerThreadId); + const firstSessionId = yield* shared.allocate; + yield* shared.open(firstSessionId); + const credential = McpProviderSession.readMcpProviderSession(sharedThreadId); + assert.isDefined(credential); + const token = credential!.authorizationHeader.replace(/^Bearer\s+/, ""); + const sharedSessionId = yield* owner.allocate; + const runtime = yield* owner.open(sharedSessionId); + // The peer session reuses the same credential for the shared thread. + yield* runtime + .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.ignore); + assert.equal( + McpProviderSession.readMcpProviderSession(sharedThreadId)?.providerSessionId, + credential!.providerSessionId, + ); + // Terminally detaching the thread from the first session must not + // revoke the credential the peer still records. + yield* manager.detach({ + providerSessionId: firstSessionId, + threadId: sharedThreadId, + revokeMcpCredential: true, + }); + assert.isDefined(yield* registry.resolve(token)); + assert.isUndefined(McpProviderSession.readMcpProviderSession(sharedThreadId)); + // The peer's own cleanup remains the credential's final holder. + yield* manager.close(sharedSessionId); + assert.isUndefined(yield* registry.resolve(token)); + }).pipe(Effect.provide(makeTestLayer({ state, idleTimeoutMs: 3_600_000 }))); + }), +); + +it.effect( + "ProviderSessionManagerV2 rechecks live claims for each credential a release sweeps", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const resolving = yield* Deferred.make(); + const resolveGate = yield* Deferred.make(); + const revoking = yield* Deferred.make(); + const revokeGate = yield* Deferred.make(); + const pauseResolve = yield* Ref.make(false); + const revokeTarget = yield* Ref.make>(Option.none()); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + resolve: (token) => + Ref.get(pauseResolve).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(resolving, undefined).pipe( + Effect.andThen(Deferred.await(resolveGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.resolve(token)), + ), + revokeProviderSession: (providerSessionId) => + Ref.get(revokeTarget).pipe( + Effect.flatMap((target) => + Option.isSome(target) && target.value === providerSessionId + ? Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(revokeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.revokeProviderSession(providerSessionId)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const ownerThreadId = ThreadId.make("thread-release-sweep-owner"); + const sharedThreadId = ThreadId.make("thread-release-sweep-shared"); + const peerThreadId = ThreadId.make("thread-release-sweep-peer"); + const owner = yield* makeThreadSessionFixture(ownerThreadId); + yield* makeThreadSessionFixture(sharedThreadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const ownerSessionId = yield* owner.allocate; + const runtimeA = yield* owner.open(ownerSessionId); + // Session A ends up holding two credentials. The adapter's + // ensureThread is unimplemented; the credential record lands before + // the adapter call dies. + yield* runtimeA + .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.ignore); + const ownerCredential = McpProviderSession.readMcpProviderSession(ownerThreadId); + const sharedCredential = McpProviderSession.readMcpProviderSession(sharedThreadId); + assert.isDefined(ownerCredential); + assert.isDefined(sharedCredential); + const sharedToken = sharedCredential!.authorizationHeader.replace(/^Bearer\s+/, ""); + const peerSessionId = yield* peer.allocate; + const runtimeB = yield* peer.open(peerSessionId); + yield* Ref.set(pauseResolve, true); + // B attaches the shared thread, reserves its credential, then suspends + // in resolve before its claim is recorded. + const attaching = yield* runtimeB + .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(resolving); + yield* Ref.set(revokeTarget, Option.some(ownerCredential!.providerSessionId)); + const closing = yield* manager.close(ownerSessionId).pipe(Effect.forkChild); + // The sweep parked inside the first credential's revocation has + // already taken any up-front snapshot of live sessions. + yield* Deferred.await(revoking); + // B's attach resumes and records its claim while the sweep is parked; + // by the time it finishes the reservation is gone, so only the live + // session record protects the credential. + yield* Deferred.succeed(resolveGate, undefined); + assert.equal((yield* Fiber.join(attaching))._tag, "Failure"); + yield* Deferred.succeed(revokeGate, undefined); + yield* Fiber.join(closing); + assert.isDefined(yield* registry.resolve(sharedToken)); + // B's own release remains the credential's final holder. + yield* manager.close(peerSessionId); + assert.isUndefined(yield* registry.resolve(sharedToken)); + }).pipe( + Effect.ensuring( + Deferred.succeed(resolveGate, undefined).pipe( + Effect.andThen(Deferred.succeed(revokeGate, undefined)), + ), + ), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 serializes detach behind an in-flight attach of the same thread", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const issuing = yield* Deferred.make(); + const issueGate = yield* Deferred.make(); + const pauseIssue = yield* Ref.make(false); + const detachDone = yield* Deferred.make(); + const issuedCredentials = yield* Ref.make< + ReadonlyArray + >([]); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (input) => + Ref.get(pauseIssue).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(issuing, undefined).pipe( + Effect.andThen(Deferred.await(issueGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.issue(input)), + Effect.tap((issued) => + Ref.update(issuedCredentials, (configs) => [...configs, issued.config]), + ), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const ownerThreadId = ThreadId.make("thread-detach-attach-owner"); + const sharedThreadId = ThreadId.make("thread-detach-attach-shared"); + const owner = yield* makeThreadSessionFixture(ownerThreadId); + yield* makeThreadSessionFixture(sharedThreadId); + const providerSessionId = yield* owner.allocate; + const runtime = yield* owner.open(providerSessionId); + yield* Ref.set(pauseIssue, true); + // The shared thread's attach suspends inside credential issuance with + // its provisional attachment already recorded in the entry. + const attaching = yield* runtime + .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(issuing); + yield* manager + .detach({ providerSessionId, threadId: sharedThreadId, revokeMcpCredential: true }) + .pipe(Effect.ensuring(Deferred.succeed(detachDone, undefined)), Effect.forkChild); + // The detach must stay parked behind the attach's lock instead of + // removing the provisional attachment out from under it. + yield* Effect.forEach(Array.from({ length: 8 }), () => Effect.yieldNow, { + discard: true, + }); + assert.isFalse(yield* Deferred.isDone(detachDone)); + yield* Deferred.succeed(issueGate, undefined); + // The adapter's ensureThread is unimplemented and dies once the attach + // bookkeeping completes; the detach then runs and wins. + assert.equal((yield* Fiber.join(attaching))._tag, "Failure"); + yield* Deferred.await(detachDone); + // The detach ran after the attach fully landed: the freshly minted + // credential was recorded, then pruned and revoked by the terminal + // detach instead of surviving as an untracked claim. + const issued = (yield* Ref.get(issuedCredentials)).find( + (config) => config.threadId === sharedThreadId, + ); + assert.isDefined(issued); + assert.isUndefined( + yield* registry.resolve(issued!.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + assert.isUndefined(McpProviderSession.readMcpProviderSession(sharedThreadId)); + }).pipe( + Effect.ensuring(Deferred.succeed(issueGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 serializes credential revocation against a new attachment's reservation", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const revoking = yield* Deferred.make(); + const revokeGate = yield* Deferred.make(); + const attachDone = yield* Deferred.make(); + const revokeTarget = yield* Ref.make>(Option.none()); + const issuedCredentials = yield* Ref.make< + ReadonlyArray + >([]); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (input) => + delegate + .issue(input) + .pipe( + Effect.tap((issued) => + Ref.update(issuedCredentials, (configs) => [...configs, issued.config]), + ), + ), + revokeProviderSession: (providerSessionId) => + Ref.get(revokeTarget).pipe( + Effect.flatMap((target) => + Option.isSome(target) && target.value === providerSessionId + ? Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(revokeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.revokeProviderSession(providerSessionId)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const ownerThreadId = ThreadId.make("thread-revoke-race-owner"); + const sharedThreadId = ThreadId.make("thread-revoke-race-shared"); + const peerThreadId = ThreadId.make("thread-revoke-race-peer"); + const owner = yield* makeThreadSessionFixture(ownerThreadId); + yield* makeThreadSessionFixture(sharedThreadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const ownerSessionId = yield* owner.allocate; + const runtimeA = yield* owner.open(ownerSessionId); + yield* runtimeA + .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.ignore); + const sharedCredential = McpProviderSession.readMcpProviderSession(sharedThreadId); + assert.isDefined(sharedCredential); + const sharedToken = sharedCredential!.authorizationHeader.replace(/^Bearer\s+/, ""); + const peerSessionId = yield* peer.allocate; + const runtimeB = yield* peer.open(peerSessionId); + yield* Ref.set(revokeTarget, Option.some(sharedCredential!.providerSessionId)); + // Session A stays live for the owner thread; the terminal detach's + // claim check finds the shared credential unclaimed, then suspends + // inside the registry delete. + const detaching = yield* manager + .detach({ + providerSessionId: ownerSessionId, + threadId: sharedThreadId, + revokeMcpCredential: true, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(revoking); + // B's attach of the same thread must wait for the in-flight + // revocation instead of reserving and resolving the about-to-be + // deleted credential. + const attaching = yield* runtimeB + .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) + .pipe( + Effect.exit, + Effect.ensuring(Deferred.succeed(attachDone, undefined)), + Effect.forkChild, + ); + yield* Effect.forEach(Array.from({ length: 8 }), () => Effect.yieldNow, { + discard: true, + }); + assert.isFalse(yield* Deferred.isDone(attachDone)); + yield* Deferred.succeed(revokeGate, undefined); + yield* Fiber.join(detaching); + // The adapter's ensureThread is unimplemented; the credential outcome + // is decided before the adapter call dies. + assert.equal((yield* Fiber.join(attaching))._tag, "Failure"); + assert.isUndefined(yield* registry.resolve(sharedToken)); + // B could not ride the doomed credential: it was issued a fresh one, + // recorded against B, which resolves. + const reissued = (yield* Ref.get(issuedCredentials)).find( + (config) => + config.threadId === sharedThreadId && + config.providerSessionId !== sharedCredential!.providerSessionId, + ); + assert.isDefined(reissued); + assert.isDefined( + yield* registry.resolve(reissued!.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + }).pipe( + Effect.ensuring(Deferred.succeed(revokeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 revokes a superseded credential when its sweeper attach is interrupted", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const browserAccess = yield* Ref.make(true); + const revoking = yield* Deferred.make(); + const revokeGate = yield* Deferred.make(); + const revokeTarget = yield* Ref.make>(Option.none()); + const issuedCredentials = yield* Ref.make< + ReadonlyArray + >([]); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (input) => + delegate + .issue(input) + .pipe( + Effect.tap((issued) => + Ref.update(issuedCredentials, (configs) => [...configs, issued.config]), + ), + ), + revokeProviderSession: (providerSessionId) => + Ref.get(revokeTarget).pipe( + Effect.flatMap((target) => + Option.isSome(target) && target.value === providerSessionId + ? Deferred.isDone(revoking).pipe( + Effect.flatMap((alreadySignaled) => + alreadySignaled + ? Effect.void + : Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(revokeGate)), + ), + ), + ) + : Effect.void, + ), + Effect.andThen(delegate.revokeProviderSession(providerSessionId)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + const settingsLayer = Layer.effect( + ServerSettings.ServerSettingsService, + Effect.gen(function* () { + const delegate = yield* ServerSettings.ServerSettingsService; + return ServerSettings.ServerSettingsService.of({ + ...delegate, + getSettings: Ref.get(browserAccess).pipe( + Effect.flatMap((browser) => + Effect.map(delegate.getSettings, (settings) => ({ + ...settings, + enableAgentBrowserAccess: browser, + })), + ), + ), + }); + }), + ).pipe(Layer.provide(ServerSettings.layerTest({ enableAgentBrowserAccess: true }))); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const ownerThreadId = ThreadId.make("thread-superseded-owner"); + const sharedThreadId = ThreadId.make("thread-superseded-shared"); + const owner = yield* makeThreadSessionFixture(ownerThreadId); + yield* makeThreadSessionFixture(sharedThreadId); + const providerSessionId = yield* owner.allocate; + const runtime = yield* owner.open(providerSessionId); + yield* runtime + .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.ignore); + const retained = McpProviderSession.readMcpProviderSession(sharedThreadId); + assert.isDefined(retained); + const retainedToken = retained!.authorizationHeader.replace(/^Bearer\s+/, ""); + // A plain detach keeps the shared session live for the owner thread + // and retains the credential record for reuse. + yield* manager.detach({ providerSessionId, threadId: sharedThreadId }); + // Rotating capabilities makes the reattach mint a replacement + // credential and supersede the retained one. + yield* Ref.set(browserAccess, false); + yield* Ref.set(revokeTarget, Option.some(retained!.providerSessionId)); + // The reattach records the replacement, then suspends inside the + // superseded credential's registry delete. + const attaching = yield* runtime + .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(revoking); + yield* Fiber.interrupt(attaching); + assert.equal((yield* Fiber.await(attaching))._tag, "Failure"); + // The superseded credential is out of every record and the config + // slot; the unwind is its last chance to be revoked. + assert.isUndefined(yield* registry.resolve(retainedToken)); + const replacement = (yield* Ref.get(issuedCredentials)).find( + (config) => + config.threadId === sharedThreadId && + config.providerSessionId !== retained!.providerSessionId, + ); + assert.isDefined(replacement); + assert.isUndefined( + yield* registry.resolve(replacement!.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + }).pipe( + Effect.ensuring(Deferred.succeed(revokeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpRegistryLayer, + serverSettingsLayer: settingsLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps the bounded release wait when a terminal detach retries during pending cleanup", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const resolving = yield* Deferred.make(); + const resolveGate = yield* Deferred.make(); + const pauseResolve = yield* Ref.make(false); + const detachDone = yield* Deferred.make(); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + resolve: (token) => + Ref.get(pauseResolve).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(resolving, undefined).pipe( + Effect.andThen(Deferred.await(resolveGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.resolve(token)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const sharedThreadId = ThreadId.make("thread-detach-retry-shared"); + const peerThreadId = ThreadId.make("thread-detach-retry-peer"); + const shared = yield* makeThreadSessionFixture(sharedThreadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const providerSessionId = yield* shared.allocate; + yield* shared.open(providerSessionId); + const peerSessionId = yield* peer.allocate; + const runtimeB = yield* peer.open(peerSessionId); + yield* Ref.set(pauseResolve, true); + // A peer attachment of the shared thread holds the prepare lock while + // its credential resolution is stalled. + yield* runtimeB + .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(resolving); + // Closing the shared session parks its credential sweep on the same + // lock; the close's own join times out while cleanup continues. + const closing = yield* manager.close(providerSessionId).pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(closing))._tag, "Failure"); + // A terminal detach retry must join the still-bounded release wait, + // not block unboundedly on the stalled prepare lock first. + const detaching = yield* manager + .detach({ providerSessionId, threadId: sharedThreadId, revokeMcpCredential: true }) + .pipe( + Effect.exit, + Effect.ensuring(Deferred.succeed(detachDone, undefined)), + Effect.forkChild, + ); + yield* TestClock.adjust("30 seconds"); + yield* Effect.forEach(Array.from({ length: 8 }), () => Effect.yieldNow, { + discard: true, + }); + assert.isTrue(yield* Deferred.isDone(detachDone)); + assert.equal((yield* Fiber.join(detaching))._tag, "Failure"); + }).pipe( + Effect.ensuring(Deferred.succeed(resolveGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps a pruned credential's cleanup alive when its terminal detach is interrupted", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const issuing = yield* Deferred.make(); + const issueGate = yield* Deferred.make(); + const revoked = yield* Deferred.make(); + const pauseIssue = yield* Ref.make(false); + const failResolve = yield* Ref.make(false); + const revokeTarget = yield* Ref.make>(Option.none()); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + resolve: (token) => + Ref.get(failResolve).pipe( + Effect.flatMap((fail) => + fail ? Effect.succeed(undefined) : delegate.resolve(token), + ), + ), + issue: (input) => + Ref.get(pauseIssue).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(issuing, undefined).pipe( + Effect.andThen(Deferred.await(issueGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.issue(input)), + ), + revokeProviderSession: (providerSessionId) => + delegate + .revokeProviderSession(providerSessionId) + .pipe( + Effect.andThen( + Ref.get(revokeTarget).pipe( + Effect.flatMap((target) => + Option.isSome(target) && target.value === providerSessionId + ? Deferred.succeed(revoked, undefined) + : Effect.void, + ), + ), + ), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const sharedThreadId = ThreadId.make("thread-detach-interrupt-shared"); + const peerThreadId = ThreadId.make("thread-detach-interrupt-peer"); + const shared = yield* makeThreadSessionFixture(sharedThreadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const providerSessionId = yield* shared.allocate; + yield* shared.open(providerSessionId); + const credential = McpProviderSession.readMcpProviderSession(sharedThreadId); + assert.isDefined(credential); + const token = credential!.authorizationHeader.replace(/^Bearer\s+/, ""); + yield* Ref.set(revokeTarget, Option.some(credential!.providerSessionId)); + const peerSessionId = yield* peer.allocate; + const runtimeB = yield* peer.open(peerSessionId); + yield* Ref.set(failResolve, true); + yield* Ref.set(pauseIssue, true); + // A peer attachment misses the recorded credential on resolve, skips + // revoking it while the first session still records it, then parks + // inside `issue` holding the shared thread's prepare lock. + const attaching = yield* runtimeB + .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(issuing); + // The terminal detach prunes the credential record, then its + // revocation parks behind the peer's prepare lock. Interrupting the + // detach must not strand the captured credential id: the detached + // sweep still owns its cleanup. + const detaching = yield* manager + .detach({ providerSessionId, threadId: sharedThreadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Effect.forEach(Array.from({ length: 12 }), () => Effect.yieldNow, { + discard: true, + }); + yield* Fiber.interrupt(detaching); + assert.equal((yield* Fiber.await(detaching))._tag, "Failure"); + yield* Deferred.succeed(issueGate, undefined); + yield* Fiber.await(attaching); + yield* Deferred.await(revoked); + yield* Ref.set(failResolve, false); + assert.isUndefined(yield* registry.resolve(token)); + }).pipe( + Effect.ensuring(Deferred.succeed(issueGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps the bounded release wait when a detach discovers pending cleanup mid-flight", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const interruptEntered = yield* Deferred.make(); + const interruptGate = yield* Deferred.make(); + const resolving = yield* Deferred.make(); + const resolveGate = yield* Deferred.make(); + const pauseResolve = yield* Ref.make(false); + const detachDone = yield* Deferred.make(); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + resolve: (token) => + Ref.get(pauseResolve).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(resolving, undefined).pipe( + Effect.andThen(Deferred.await(resolveGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.resolve(token)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const eventSink = yield* EventSinkV2; + const idAllocator = yield* IdAllocatorV2; + const now = yield* DateTime.now; + const firstThreadId = ThreadId.make("thread-detach-latejoin-first"); + const peerThreadId = ThreadId.make("thread-detach-latejoin-peer"); + const first = yield* makeThreadSessionFixture(firstThreadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const providerSessionId = yield* first.allocate; + const providerThread = makeProviderThread({ + idAllocator, + threadId: firstThreadId, + providerSessionId, + now, + }); + const runId = idAllocator.derive.run({ threadId: firstThreadId, ordinal: 1 }); + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId: firstThreadId }), + type: "provider-thread.updated", + threadId: firstThreadId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: providerThread, + }, + { + id: yield* idAllocator.allocate.event({ threadId: firstThreadId }), + type: "provider-turn.updated", + threadId: firstThreadId, + runId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: { + id: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-detach-latejoin", + }), + providerThreadId: providerThread.id, + nodeId: idAllocator.derive.rootNode({ runId }), + runAttemptId: null, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }, + ], + }); + yield* first.open(providerSessionId); + const peerSessionId = yield* peer.allocate; + const runtimeB = yield* peer.open(peerSessionId); + yield* Ref.set(pauseResolve, true); + // A peer attachment parks on the stalled resolution holding the + // shared thread's prepare lock. + yield* runtimeB + .ensureThread({ threadId: firstThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(resolving); + // The detach passes the pending-release check, then parks inside + // interruptTurn on the seeded running turn. + const detaching = yield* manager + .detach({ + providerSessionId, + threadId: firstThreadId, + revokeMcpCredential: true, + }) + .pipe( + Effect.exit, + Effect.ensuring(Deferred.succeed(detachDone, undefined)), + Effect.forkChild, + ); + yield* Deferred.await(interruptEntered); + // The close claims the session while the detach is suspended; its own + // join times out while the credential sweep stays parked on the lock. + const closing = yield* manager.close(providerSessionId).pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(closing))._tag, "Failure"); + // The resumed detach discovers the pending release after its atomic + // decision; it must join the bounded wait instead of parking on the + // stalled prepare lock first. + yield* Deferred.succeed(interruptGate, undefined); + yield* TestClock.adjust("30 seconds"); + yield* Effect.forEach(Array.from({ length: 8 }), () => Effect.yieldNow, { + discard: true, + }); + assert.isTrue(yield* Deferred.isDone(detachDone)); + assert.equal((yield* Fiber.join(detaching))._tag, "Failure"); + }).pipe( + Effect.ensuring( + Deferred.succeed(interruptGate, undefined).pipe( + Effect.andThen(Deferred.succeed(resolveGate, undefined)), + ), + ), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpRegistryLayer, + beforeInterrupt: Deferred.succeed(interruptEntered, undefined).pipe( + Effect.andThen(Deferred.await(interruptGate)), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps a failed open's credential claim until its scope finishes closing", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const openReached = yield* Deferred.make(); + const openGate = yield* Deferred.make(); + const finalizerEntered = yield* Deferred.make(); + const finalizerGate = yield* Deferred.make(); + const opens = yield* Ref.make(0); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const threadId = ThreadId.make("thread-open-reservation-scope"); + const fixture = yield* makeThreadSessionFixture(threadId); + const firstSessionId = yield* fixture.allocate; + yield* fixture.open(firstSessionId); + const credential = McpProviderSession.readMcpProviderSession(threadId); + assert.isDefined(credential); + const token = credential!.authorizationHeader.replace(/^Bearer\s+/, ""); + // A second open for the same thread reuses the live session's + // credential, then is interrupted after its provider process started + // but before the entry registers. + const secondSessionId = yield* fixture.allocate; + const opening = yield* manager + .open({ + threadId, + providerSessionId: secondSessionId, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(openReached); + // The unwind stalls inside the provider scope's close finalizer; the + // reservation must outlive it or a peer's terminal detach can revoke + // the credential out from under the still-closing process. + yield* Fiber.interrupt(opening).pipe(Effect.forkChild); + yield* Deferred.await(finalizerEntered); + yield* manager.detach({ + providerSessionId: firstSessionId, + threadId, + revokeMcpCredential: true, + }); + assert.isDefined(yield* registry.resolve(token)); + // Once the scope close settles the reservation drops and the + // claim-aware sweep revokes the now-unclaimed credential. + yield* Deferred.succeed(finalizerGate, undefined); + assert.equal((yield* Fiber.await(opening))._tag, "Failure"); + yield* Effect.forEach(Array.from({ length: 12 }), () => Effect.yieldNow, { + discard: true, + }); + assert.isUndefined(yield* registry.resolve(token)); + }).pipe( + Effect.ensuring( + Deferred.succeed(openGate, undefined).pipe( + Effect.andThen(Deferred.succeed(finalizerGate, undefined)), + ), + ), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + beforeClose: Deferred.succeed(finalizerEntered, undefined).pipe( + Effect.andThen(Deferred.await(finalizerGate)), + ), + afterOpen: Ref.getAndUpdate(opens, (n) => n + 1).pipe( + Effect.flatMap((n) => + n === 1 + ? Deferred.succeed(openReached, undefined).pipe( + Effect.andThen(Deferred.await(openGate)), + ) + : Effect.void, + ), + ), + }), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 marks pending runtime work failed when cleanup times out", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const hangClose = yield* Deferred.make(); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSinkV2; + const idAllocator = yield* IdAllocatorV2; + const manager = yield* ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: "provider-session-manager-timeout-requests", + projectId: yield* idAllocator.allocate.project({ + fixtureName: "provider-session-manager-timeout-requests", + }), + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const pendingRequest = yield* makePendingRuntimeRequestEvents({ + idAllocator, + threadId, + providerSessionId, + providerThread, + now, + }); + yield* eventSink.write({ events: pendingRequest.events }); + yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + + const closing = yield* manager.close(providerSessionId).pipe(Effect.result, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(closing))._tag, "Failure"); + yield* Effect.yieldNow; + + const projection = yield* projectionStore.getThreadProjection(threadId); + const request = projection.runtimeRequests.find( + (candidate) => candidate.id === pendingRequest.requestId, + ); + const requestNode = projection.nodes.find((node) => node.id === pendingRequest.nodeId); + const requestTurnItem = projection.turnItems.find( + (item) => item.type === "approval_request" && item.requestId === pendingRequest.requestId, + ); + // Cleanup is still pending at the timeout, so the release is a runtime + // failure — not a normal close that would mark the work cancelled. + assert.equal(request?.status, "expired"); + assert.equal(request?.responseCapability.type, "not_resumable"); + assert.equal(requestNode?.status, "failed"); + assert.equal(requestTurnItem?.status, "failed"); + }); + + yield* effect.pipe( + Effect.ensuring(Deferred.succeed(hangClose, undefined)), + Effect.provide( + makeTestLayer({ state, idleTimeoutMs: 1000, hangSessionScopeClose: hangClose }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 does not release a same-id replacement from a stale detach", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const revoking = yield* Deferred.make(); + const revokeGate = yield* Deferred.make(); + const pauseRevoke = yield* Ref.make(false); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + revokeProviderSession: (providerSessionId) => + Ref.getAndSet(pauseRevoke, false).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(revokeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.revokeProviderSession(providerSessionId)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const threadId = ThreadId.make("thread-stale-detach-replacement"); + const replacementThreadId = ThreadId.make("thread-stale-detach-replacement-peer"); + const fixture = yield* makeThreadSessionFixture(threadId); + const replacementFixture = yield* makeThreadSessionFixture(replacementThreadId); + const providerSessionId = yield* fixture.allocate; + yield* fixture.open(providerSessionId); + + yield* Ref.set(pauseRevoke, true); + // The terminal detach prunes the thread's credential record and hands + // it to the forked release's sweep, then its detached revocation parks + // on the gated registry call while holding mcpPrepareLock — so the + // release (and anything joining it) settles only once the gate opens. + const detaching = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(revoking); + + // Close joins the pending release the detach forked; it stays parked + // behind the gated revocation until the lock frees. + const closing = yield* manager.close(providerSessionId).pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 4; i += 1) { + yield* Effect.yieldNow; + } + yield* Deferred.succeed(revokeGate, undefined); + assert.equal((yield* Fiber.join(closing))._tag, "Success"); + + // A same-id replacement for another thread opens once the pending + // release cleared; the stale detach's join observes the old cleanup. + yield* replacementFixture.open(providerSessionId); + assert.equal((yield* Ref.get(state)).openCount, 2); + + assert.equal((yield* Fiber.await(detaching))._tag, "Success"); + + // The emptied session's auto-release must not claim the replacement: + // its runtime identity differs from the entry the detach removed. + yield* Effect.forEach(Array.from({ length: 12 }), () => Effect.yieldNow, { + discard: true, + }); + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + assert.equal((yield* Ref.get(state)).closeCount, 1); + }).pipe( + Effect.ensuring(Deferred.succeed(revokeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + capabilities: ExclusiveCapabilities, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 does not join a replacement session's cleanup from a stale detach", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const interruptEntered = yield* Deferred.make(); + const interruptGate = yield* Deferred.make(); + const closeEntered = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + const gateCloses = yield* Ref.make(false); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const eventSink = yield* EventSinkV2; + const idAllocator = yield* IdAllocatorV2; + const now = yield* DateTime.now; + const firstThreadId = ThreadId.make("thread-stale-join-first"); + const secondThreadId = ThreadId.make("thread-stale-join-second"); + const replacementThreadId = ThreadId.make("thread-stale-join-replacement"); + const first = yield* makeThreadSessionFixture(firstThreadId); + const second = yield* makeThreadSessionFixture(secondThreadId); + const replacement = yield* makeThreadSessionFixture(replacementThreadId); + const providerSessionId = yield* first.allocate; + const providerThread = makeProviderThread({ + idAllocator, + threadId: firstThreadId, + providerSessionId, + now, + }); + const runId = idAllocator.derive.run({ threadId: firstThreadId, ordinal: 1 }); + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId: firstThreadId }), + type: "provider-thread.updated", + threadId: firstThreadId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: providerThread, + }, + { + id: yield* idAllocator.allocate.event({ threadId: firstThreadId }), + type: "provider-turn.updated", + threadId: firstThreadId, + runId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: { + id: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-stale-join", + }), + providerThreadId: providerThread.id, + nodeId: idAllocator.derive.rootNode({ runId }), + runAttemptId: null, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }, + ], + }); + yield* first.open(providerSessionId); + yield* second.open(providerSessionId); + + // Detach parks inside interruptTurn on session A. While it is parked: + // A's own close completes, a same-id replacement B opens for another + // thread, and B begins a gated cleanup of its own. + const detachReturned = yield* Deferred.make(); + const detaching = yield* manager + .detach({ providerSessionId, threadId: firstThreadId }) + .pipe( + Effect.tap(() => Deferred.succeed(detachReturned, undefined)), + Effect.exit, + Effect.forkChild, + ); + yield* Deferred.await(interruptEntered); + yield* manager.close(providerSessionId); + yield* replacement.open(providerSessionId); + yield* Ref.set(gateCloses, true); + const closingB = yield* manager + .close(providerSessionId) + .pipe(Effect.result, Effect.forkChild); + yield* Deferred.await(closeEntered); + + // The detach's target is already gone; joining B's pending cleanup + // would park this call on another session's shutdown. + yield* Deferred.succeed(interruptGate, undefined); + for (let i = 0; i < 12; i += 1) { + yield* Effect.yieldNow; + } + assert.isTrue(yield* Deferred.isDone(detachReturned)); + assert.equal((yield* Fiber.await(detaching))._tag, "Success"); + + yield* Deferred.succeed(closeGate, undefined); + yield* Fiber.join(closingB); + assert.equal((yield* Ref.get(state)).closeCount, 2); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + }).pipe( + Effect.ensuring( + Deferred.succeed(interruptGate, undefined).pipe( + Effect.andThen(Deferred.succeed(closeGate, undefined)), + ), + ), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + beforeInterrupt: Deferred.succeed(interruptEntered, undefined).pipe( + Effect.andThen(Deferred.await(interruptGate)), + ), + beforeClose: Ref.get(gateCloses).pipe( + Effect.flatMap((gate) => + gate + ? Deferred.succeed(closeEntered, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + ) + : Effect.void, + ), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 does not let a stale turn failure idle a same-id replacement", + () => + Effect.gen(function* () { + const discriminated = yield* Ref.make(false); + // Sweep scheduler suspension points on the stale turn fiber: the + // discriminating window is anywhere the fiber is suspended before its + // failure bookkeeping (markIdle) runs — resuming it after the + // replacement opened makes the stale mark land on the replacement's + // entry, where the runtime-identity guard must reject it. Points + // inside the admitted adapter op keep `done` unfired, so the release + // drain correctly waits on them instead. + for (let ops = 30; ops < 110; ops += 1) { + const state = yield* Ref.make(emptyState); + const aTurnEntered = yield* Deferred.make(); + const bTurnBusy = yield* Deferred.make(); + const bTurnGate = yield* Deferred.make(); + const bClosed = yield* Deferred.make(); + const closes = yield* Ref.make(0); + const firstThreadId = ThreadId.make(`thread-stale-idle-first-${ops}`); + const replacementThreadId = ThreadId.make(`thread-stale-idle-replacement-${ops}`); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const stepper = makeSteppingScheduler(); + const idAllocator = yield* IdAllocatorV2; + const projectionStore = yield* ProjectionStoreV2; + const now = yield* DateTime.now; + const first = yield* makeThreadSessionFixture(firstThreadId); + const replacement = yield* makeThreadSessionFixture(replacementThreadId); + const providerSessionId = yield* first.allocate; + const firstRuntime = yield* first.open(providerSessionId); + yield* firstRuntime.events.pipe(Stream.runDrain, Effect.forkScoped); + const firstProviderThread = makeProviderThread({ + idAllocator, + threadId: firstThreadId, + providerSessionId, + now, + }); + const firstAppThread = (yield* projectionStore.getThreadProjection(firstThreadId)).thread; + const firstRunId = idAllocator.derive.run({ threadId: firstThreadId, ordinal: 1 }); + + // A's turn runs on the stepping scheduler; the adapter operation + // fails as soon as it is invoked so every suspension point after it + // still has its failure bookkeeping outstanding. + const aTurn = yield* firstRuntime + .startTurn({ + appThread: firstAppThread, + threadId: firstThreadId, + runId: firstRunId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId: firstRunId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId: firstRunId }), + providerThread: firstProviderThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ + threadId: firstThreadId, + ordinal: 1, + }), + text: "park me", + attachments: [], + }, + modelSelection, + runtimePolicy, + }) + .pipe( + Effect.exit, + Effect.forkDetach, + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + ); + const stepped = yield* stepper.step(aTurn, ops); + if (!stepped) { + yield* stepper.drain; + yield* Fiber.await(aTurn); + return; + } + + // A closes while its turn is still in flight. When the turn is + // parked inside its admitted adapter op the drain waits on it; when + // it is parked before admission or after the op settled, the close + // completes and the stale bookkeeping is left outstanding. + const closing = yield* manager + .close(providerSessionId) + .pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 16; i += 1) yield* Effect.yieldNow; + const closePolled = closing.pollUnsafe(); + const closeSucceeded = + closePolled !== undefined && + Exit.isSuccess(closePolled) && + Exit.isSuccess(closePolled.value); + if (!closeSucceeded) { + yield* stepper.drain; + yield* Fiber.await(closing); + yield* Fiber.await(aTurn); + return; + } + + // The release finished while the stale turn's bookkeeping is still + // suspended: open the same-id replacement and park its own turn + // busy inside the adapter. + const replacementRuntime = yield* replacement.open(providerSessionId); + yield* replacementRuntime.events.pipe(Stream.runDrain, Effect.forkScoped); + const replacementProviderThread = makeProviderThread({ + idAllocator, + threadId: replacementThreadId, + providerSessionId, + now, + }); + const replacementAppThread = (yield* projectionStore.getThreadProjection( + replacementThreadId, + )).thread; + const replacementRunId = idAllocator.derive.run({ + threadId: replacementThreadId, + ordinal: 1, + }); + yield* replacementRuntime + .startTurn({ + appThread: replacementAppThread, + threadId: replacementThreadId, + runId: replacementRunId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ + runId: replacementRunId, + attemptOrdinal: 1, + }), + rootNodeId: idAllocator.derive.rootNode({ runId: replacementRunId }), + providerThread: replacementProviderThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ + threadId: replacementThreadId, + ordinal: 1, + }), + text: "keep me busy", + attachments: [], + }, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(bTurnBusy); + + // Now let the stale fiber finish: its markIdle lands on the + // replacement's entry and must be rejected by the runtime-identity + // guard instead of decrementing the replacement's busy count. + const aStillParked = aTurn.pollUnsafe() === undefined; + yield* stepper.drain; + const aExit = yield* Fiber.await(aTurn); + assert.isTrue( + Exit.isSuccess(aExit) && Exit.isFailure(aExit.value), + `stale turn should end with its own failure (op ${ops})`, + ); + if (aStillParked) { + yield* Ref.set(discriminated, true); + } + + // A stale markIdle that decremented the replacement's busy count + // leaves it releasable while its own turn is still parked: any + // activity on the replacement installs an idle probe that then + // releases it within one idle window. + yield* replacementRuntime.interruptTurn({ + providerThread: replacementProviderThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: `replacement-turn-${ops}`, + }), + }); + yield* TestClock.adjust("2 seconds"); + for (let i = 0; i < 12; i += 1) { + yield* Effect.yieldNow; + } + assert.isFalse(yield* Deferred.isDone(bClosed)); + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + assert.equal((yield* Ref.get(state)).closeCount, 1); + }).pipe( + Effect.ensuring(Deferred.succeed(bTurnGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 1_000, + beforeClose: Ref.getAndUpdate(closes, (n) => n + 1).pipe( + Effect.flatMap((n) => + n + 1 === 2 ? Deferred.succeed(bClosed, undefined) : Effect.void, + ), + ), + startTurn: (input) => + input.threadId === firstThreadId + ? Deferred.succeed(aTurnEntered, undefined).pipe( + Effect.andThen( + Effect.fail( + new ProviderAdapterTurnStartError({ + driver: CODEX_DRIVER, + threadId: input.threadId, + providerThreadId: input.providerThread.id, + runId: input.runId, + }), + ), + ), + ) + : Deferred.succeed(bTurnBusy, undefined).pipe( + Effect.andThen(Deferred.await(bTurnGate)), + ), + }), + ), + ); + } + assert.isTrue( + yield* Ref.get(discriminated), + "sweep never observed the stale turn parked through the replacement open", + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 does not let a late resume completion mark a replacement's thread loaded", + () => + Effect.gen(function* () { + const discriminated = yield* Ref.make(false); + // Sweep scheduler suspension points: the discriminating window is the + // boundary where the stale runtime's adapter op has settled (so the + // release drain does not wait on it) but its loaded-thread bookkeeping + // has not yet run — the mark then lands against the replacement's + // entry, where the runtime-identity guard rejects it. + for (let ops = 60; ops < 95; ops += 1) { + const state = yield* Ref.make(emptyState); + const resumeCalls = yield* Ref.make(0); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const stepper = makeSteppingScheduler(); + const idAllocator = yield* IdAllocatorV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread-late-resume-replacement-${ops}`); + const first = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* first.allocate; + const firstRuntime = yield* first.open(providerSessionId); + yield* firstRuntime.events.pipe(Stream.runDrain, Effect.forkScoped); + const resumeProviderThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + const aResume = yield* firstRuntime + .resumeThread({ providerThread: resumeProviderThread, threadId }) + .pipe( + Effect.exit, + Effect.forkDetach, + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + ); + const stepped = yield* stepper.step(aResume, ops); + if (!stepped) { + yield* stepper.drain; + yield* Fiber.await(aResume); + return; + } + const closing = yield* manager + .close(providerSessionId) + .pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 16; i += 1) yield* Effect.yieldNow; + const closePolled = closing.pollUnsafe(); + const closeSucceeded = + closePolled !== undefined && + Exit.isSuccess(closePolled) && + Exit.isSuccess(closePolled.value); + if (!closeSucceeded) { + // The suspended resume is still inside its admitted operation — + // the release drain is correctly waiting on it. + yield* stepper.drain; + yield* Fiber.await(closing); + yield* Fiber.await(aResume); + return; + } + // The op settled and the release finished while the resume's mark + // is still suspended: open the replacement, then let the stale + // bookkeeping land against it. + const replacementRuntime = yield* first.open(providerSessionId); + yield* replacementRuntime.events.pipe(Stream.runDrain, Effect.forkScoped); + const resumeStillParked = aResume.pollUnsafe() === undefined; + yield* stepper.drain; + assert.isDefined( + aResume.pollUnsafe(), + `resume fiber still suspended after drain (op ${ops})`, + ); + // The stale resume ends honestly either way: admitted before the + // claim it completes with a mark the replacement's entry rejects, + // or the admission fence refuses it with a protocol error once the + // entry is gone. What must not happen is its bookkeeping landing in + // the replacement's loaded-thread cache. + const resumeExit = yield* Fiber.await(aResume); + // The discriminating iteration is the one where the stale resume + // was still suspended when the drain ran — so its mark lands after + // the replacement opened — and its adapter op had actually + // completed. A refused admission (inner Failure) never reaches the + // mark, and a resume that finished during `step` marked the dead + // entry; neither exercises the runtime-identity guard. + if (resumeStillParked && Exit.isSuccess(resumeExit) && Exit.isSuccess(resumeExit.value)) { + yield* Ref.set(discriminated, true); + } + const beforeReplacement = yield* Ref.get(state); + yield* replacementRuntime.resumeThread({ + providerThread: resumeProviderThread, + threadId, + }); + assert.equal( + (yield* Ref.get(state)).resumeCount, + beforeReplacement.resumeCount + 1, + `stale resume marked the replacement's thread loaded (op ${ops})`, + ); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + resumeThread: (input) => + Ref.getAndUpdate(resumeCalls, (n) => n + 1).pipe( + Effect.andThen( + Ref.update(state, (current) => ({ + ...current, + resumeCount: current.resumeCount + 1, + })), + ), + Effect.as(input.providerThread), + ), + }), + ), + ); + } + assert.isTrue( + yield* Ref.get(discriminated), + "sweep found no boundary where the release finished before the mark ran", + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 does not clear a replacement session's MCP config from a stale detach", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const interruptEntered = yield* Deferred.make(); + const interruptGate = yield* Deferred.make(); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const eventSink = yield* EventSinkV2; + const idAllocator = yield* IdAllocatorV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-stale-detach-config"); + const peerThreadId = ThreadId.make("thread-stale-detach-config-peer"); + const first = yield* makeThreadSessionFixture(threadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const providerSessionId = yield* first.allocate; + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId }), + type: "provider-thread.updated", + threadId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: providerThread, + }, + { + id: yield* idAllocator.allocate.event({ threadId }), + type: "provider-turn.updated", + threadId, + runId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: { + id: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-stale-config", + }), + providerThreadId: providerThread.id, + nodeId: idAllocator.derive.rootNode({ runId }), + runAttemptId: null, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }, + ], + }); + yield* first.open(providerSessionId); + yield* peer.open(providerSessionId); + + // The terminal detach parks inside interruptTurn holding no locks. + // While it is parked the session closes and a same-id replacement + // re-opens the same thread, configuring a fresh credential. + const detaching = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(interruptEntered); + yield* manager.close(providerSessionId); + yield* first.open(providerSessionId); + const replacementConfig = McpProviderSession.readMcpProviderSession(threadId); + assert.isDefined(replacementConfig); + + yield* Deferred.succeed(interruptGate, undefined); + assert.equal((yield* Fiber.await(detaching))._tag, "Success"); + + // The stale detach's credential sweep must not clear the slot the + // replacement configured for the thread. + assert.equal( + McpProviderSession.readMcpProviderSession(threadId)?.providerSessionId, + replacementConfig!.providerSessionId, + ); + assert.isDefined( + yield* registry.resolve(replacementConfig!.authorizationHeader.replace(/^Bearer\s+/, "")), + ); + }).pipe( + Effect.ensuring(Deferred.succeed(interruptGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + beforeInterrupt: Deferred.succeed(interruptEntered, undefined).pipe( + Effect.andThen(Deferred.await(interruptGate)), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 does not re-arm a same-id replacement's idle release from a stale detach", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const revoking = yield* Deferred.make(); + const revokeGate = yield* Deferred.make(); + const pauseRevoke = yield* Ref.make(true); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + revokeProviderSession: (providerSessionId) => + Ref.getAndSet(pauseRevoke, false).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(revokeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.revokeProviderSession(providerSessionId)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const firstThreadId = ThreadId.make("thread-stale-sched-first"); + const secondThreadId = ThreadId.make("thread-stale-sched-second"); + const replacementThreadId = ThreadId.make("thread-stale-sched-replacement"); + const first = yield* makeThreadSessionFixture(firstThreadId); + const second = yield* makeThreadSessionFixture(secondThreadId); + const replacement = yield* makeThreadSessionFixture(replacementThreadId); + const providerSessionId = yield* first.allocate; + yield* first.open(providerSessionId); + yield* second.open(providerSessionId); + + // Detaching one thread from the shared session leaves the session + // live for the second thread, so the detach tail reschedules idle + // release. Park it inside the gated credential revocation. + const detaching = yield* manager + .detach({ providerSessionId, threadId: firstThreadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(revoking); + + // The session closes and a same-id replacement opens for another + // thread while the stale detach is still parked. + yield* manager.close(providerSessionId); + yield* replacement.open(providerSessionId); + + // Advance the clock so the replacement's own idle probe is due before + // the stale detach resumes; a re-armed timer would push it past this. + yield* TestClock.adjust("500 millis"); + yield* Deferred.succeed(revokeGate, undefined); + assert.equal((yield* Fiber.await(detaching))._tag, "Success"); + yield* TestClock.adjust("600 millis"); + for (let i = 0; i < 12; i += 1) { + yield* Effect.yieldNow; + } + + // The replacement's own idle schedule released it at its original + // deadline; a stale re-arm would leave it live until 1500ms. + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + assert.equal((yield* Ref.get(state)).closeCount, 2); + }).pipe( + Effect.ensuring(Deferred.succeed(revokeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 1_000, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 does not let an orphaned idle timer release a same-id replacement", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const firstProbe = yield* Ref.make(true); + const probeCalls = yield* Ref.make(0); + const probeEntered = yield* Deferred.make(); + const probeGate = yield* Deferred.make(); + const effect = Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const threadId = ThreadId.make("thread-orphaned-idle-timer"); + const fixture = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* fixture.allocate; + const runtimeA = yield* fixture.open(providerSessionId); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now: yield* DateTime.now, + }); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "orphan-idle-turn", + }); + + // A's idle timer fires and parks inside an uninterruptible probe. + yield* TestClock.adjust("1 second"); + yield* Deferred.await(probeEntered); + + // Two schedulers both block interrupting that probe fiber. When the + // gate opens the pending interrupts kill it before it can release A, + // so both schedulers resume against a live entry: one installs its + // timer (generation 2, now tracked) and the other's install is + // displaced. Without loser-cancellation the displaced timer survives + // as an orphaned callback; the close below only cancels the tracked + // one. + const s1 = yield* runtimeA + .interruptTurn({ providerThread, providerTurnId }) + .pipe(Effect.exit, Effect.forkChild); + const s2 = yield* runtimeA + .interruptTurn({ providerThread, providerTurnId }) + .pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 20; i += 1) { + yield* Effect.yieldNow; + } + yield* Deferred.succeed(probeGate, undefined); + yield* Fiber.join(s1); + yield* Fiber.join(s2); + // A is still live: the interrupted probe never reached releaseEntry. + assert.equal((yield* Ref.get(state)).closeCount, 0); + + yield* manager.close(providerSessionId); + assert.equal((yield* Ref.get(state)).closeCount, 1); + + // Open the same-id replacement half a window before the orphan's + // deadline so its own idle timer (due at T0+2.5s) stays behind it, + // then bump its generation to 2 to match the orphan's. Observe through + // closeCount, not manager.get: get touches activity and would re-arm + // the timer being measured. + yield* TestClock.adjust("500 millis"); + const runtimeB = yield* fixture.open(providerSessionId); + yield* runtimeB.interruptTurn({ providerThread, providerTurnId }).pipe(Effect.ignore); + + // Orphan deadline. Pinned to A's runtime it must bail before probing + // the replacement; unguarded it reads B's entry at generation 2 and + // releases it ahead of B's own timer. + yield* TestClock.adjust("500 millis"); + for (let i = 0; i < 12; i += 1) { + yield* Effect.yieldNow; + } + assert.equal(yield* Ref.get(probeCalls), 1); + assert.equal((yield* Ref.get(state)).closeCount, 1); + + // The replacement's own idle release still fires on schedule. + yield* TestClock.adjust("500 millis"); + for (let i = 0; i < 12; i += 1) { + yield* Effect.yieldNow; + } + assert.equal(yield* Ref.get(probeCalls), 2); + assert.equal((yield* Ref.get(state)).closeCount, 2); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + }); + + yield* effect.pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 1_000, + hasPendingBackgroundWork: Effect.uninterruptible( + Effect.gen(function* () { + yield* Ref.update(probeCalls, (calls) => calls + 1); + if (yield* Ref.getAndSet(firstProbe, false)) { + yield* Deferred.succeed(probeEntered, undefined); + yield* Deferred.await(probeGate); + } + return false; + }), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps idle cleanup armed when a rescheduler is interrupted", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const firstProbe = yield* Ref.make(true); + const probeEntered = yield* Deferred.make(); + const probeGate = yield* Deferred.make(); + const effect = Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const threadId = ThreadId.make("thread-interrupted-rescheduler"); + const fixture = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* fixture.allocate; + yield* fixture.open(providerSessionId); + + // The session's idle timer fires and parks inside the uninterruptible + // pending-work probe. + yield* TestClock.adjust("1 second"); + yield* Deferred.await(probeEntered); + + // get() touches activity, so its rescheduler blocks interrupting that + // probe fiber. Interrupting the lookup must not leave the entry + // pointing at a dead timer with no replacement installed. + const getting = yield* manager.get(providerSessionId).pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 20; i += 1) { + yield* Effect.yieldNow; + } + const interrupting = yield* Fiber.interrupt(getting).pipe(Effect.forkChild); + for (let i = 0; i < 10; i += 1) { + yield* Effect.yieldNow; + } + yield* Deferred.succeed(probeGate, undefined); + yield* Fiber.join(interrupting); + yield* Fiber.await(getting); + + // The rescheduler must have installed a replacement timer before it + // could be interrupted away: the session still idles out. + yield* TestClock.adjust("1 second"); + for (let i = 0; i < 12; i += 1) { + yield* Effect.yieldNow; + } + assert.equal((yield* Ref.get(state)).closeCount, 1); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + }); + + yield* effect.pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 1_000, + hasPendingBackgroundWork: Effect.uninterruptible( + Effect.gen(function* () { + if (yield* Ref.getAndSet(firstProbe, false)) { + yield* Deferred.succeed(probeEntered, undefined); + yield* Deferred.await(probeGate); + } + return false; + }), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 releases an exclusive session when its terminal detach is interrupted", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const revoking = yield* Deferred.make(); + const revokeGate = yield* Deferred.make(); + const pauseRevoke = yield* Ref.make(false); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + revokeProviderSession: (providerSessionId) => + Ref.getAndSet(pauseRevoke, false).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(revokeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.revokeProviderSession(providerSessionId)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const threadId = ThreadId.make("thread-interrupted-terminal-detach"); + const fixture = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* fixture.allocate; + yield* fixture.open(providerSessionId); + + yield* Ref.set(pauseRevoke, true); + // The terminal detach removes the last attachment and prunes the + // credential record, then parks joining the detached revocation. + const detaching = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(revoking); + + // Interrupting the revocation wait must not strand the emptied + // exclusive session: the release handoff already happened, so the + // forked cleanup completes without this caller. + yield* Fiber.interrupt(detaching); + yield* Deferred.succeed(revokeGate, undefined); + for (let i = 0; i < 12; i += 1) { + yield* Effect.yieldNow; + } + assert.equal((yield* Ref.get(state)).closeCount, 1); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + + // A retry finds nothing left to release: the handoff already ran. + yield* manager.detach({ providerSessionId, threadId }); + assert.equal((yield* Ref.get(state)).closeCount, 1); + }).pipe( + Effect.ensuring(Deferred.succeed(revokeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + capabilities: ExclusiveCapabilities, + mcpRegistryLayer, + }), + ), + ); + }), +); + +for (const operation of ["close", "detach"] as const) { + it.effect( + `ProviderSessionManagerV2 ${operation} during provider startup joins the in-flight open's cleanup`, + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const duringOpen = yield* Ref.make>(Effect.void); + const openEntered = yield* Deferred.make(); + const openGate = yield* Deferred.make(); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const threadId = ThreadId.make(`thread-open-startup-${operation}`); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* allocate; + yield* Ref.set( + duringOpen, + Deferred.succeed(openEntered, undefined).pipe(Effect.andThen(Deferred.await(openGate))), + ); + const opening = yield* open(providerSessionId).pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(openEntered); + + // The provider process is spawning with a held credential + // reservation but no live entry: the release must mark the startup + // and join its unwind, not report success over owned resources. + const releasing = yield* ( + operation === "close" + ? manager.close(providerSessionId) + : manager.detach({ providerSessionId, threadId }) + ).pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 4; i += 1) { + yield* Effect.yieldNow; + } + yield* Deferred.succeed(openGate, undefined); + + const releaseExit = yield* Fiber.join(releasing); + const openExit = yield* Fiber.join(opening); + assert.isTrue(Exit.isSuccess(releaseExit)); + assert.isTrue(Exit.isFailure(openExit)); + + // The spawned provider's scope was closed by the startup unwind and + // its reserved credential swept — nothing is left live or unowned. + assert.equal((yield* Ref.get(state)).closeCount, 1); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + const issued = (yield* Ref.get(mcpConfigs)).at(-1); + const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(token); + assert.isUndefined(yield* registry.resolve(token!)); + assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); + + // With the startup settled, a replacement open proceeds. + yield* open(providerSessionId); + assert.equal((yield* Ref.get(state)).openCount, 2); + yield* manager.close(providerSessionId); + }).pipe( + Effect.ensuring(Deferred.succeed(openGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + capabilities: ExclusiveCapabilities, + mcpConfigs, + beforeOpen: () => + Ref.get(duringOpen).pipe( + Effect.flatten, + Effect.tap(() => Ref.set(duringOpen, Effect.void)), + ), + }), + ), + ); + }), + ); +} + +it.effect( + "ProviderSessionManagerV2 keeps failed startup cleanup visible and blocks replacement", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const poisonOpen = yield* Ref.make(true); + const poisonClose = yield* Ref.make(true); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const threadId = ThreadId.make("thread-open-startup-failed-cleanup"); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* allocate; + const replacementId = yield* allocate; + + // openSession fails after registering its scope finalizers, and the + // scope close itself defects: the startup cleanup cannot complete, so + // the provider process and its credential may still be owned. + const openExit = yield* open(providerSessionId).pipe(Effect.exit); + assert.isTrue(Exit.isFailure(openExit)); + assert.equal((yield* Ref.get(state)).closeCount, 1); + + // The failed startup record blocks replacement by session id and by + // thread, and a close retry surfaces the recorded cleanup failure. + assert.equal( + (yield* open(providerSessionId).pipe(Effect.flip))._tag, + "ProviderSessionOpenError", + ); + assert.equal( + (yield* open(replacementId).pipe(Effect.flip))._tag, + "ProviderSessionOpenError", + ); + assert.equal( + (yield* manager.close(providerSessionId).pipe(Effect.flip))._tag, + "ProviderSessionCloseError", + ); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + + // The unclaimed credential was still swept even though the scope close + // failed: the record stays only for the provider resources. + const issued = (yield* Ref.get(mcpConfigs)).at(-1); + const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(token); + assert.isUndefined(yield* registry.resolve(token!)); + assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + capabilities: ExclusiveCapabilities, + mcpConfigs, + afterOpen: Ref.getAndSet(poisonOpen, false).pipe( + Effect.flatMap((armed) => + armed ? Effect.die("provider spawn blew up") : Effect.void, + ), + ), + beforeClose: Ref.getAndSet(poisonClose, false).pipe( + Effect.flatMap((armed) => + armed ? Effect.die("provider process did not exit") : Effect.void, + ), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 terminal detach bounds the credential revocation wait when the session stays live", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const pauseSettings = yield* Ref.make(false); + const preparing = yield* Deferred.make(); + const prepareGate = yield* Deferred.make(); + const settingsLayer = Layer.effect( + ServerSettings.ServerSettingsService, + Effect.gen(function* () { + const delegate = yield* ServerSettings.ServerSettingsService; + return ServerSettings.ServerSettingsService.of({ + ...delegate, + getSettings: Ref.get(pauseSettings).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(prepareGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.getSettings), + ), + }); + }), + ).pipe(Layer.provide(ServerSettings.layerTest({ enableAgentBrowserAccess: true }))); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const idAllocator = yield* IdAllocatorV2; + const threadId = ThreadId.make("thread-detach-revoke-bounded"); + const peerThreadId = ThreadId.make("thread-detach-revoke-bounded-peer"); + const first = yield* makeThreadSessionFixture(threadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const providerSessionId = yield* first.allocate; + // Multi-thread session with two attachments: detaching one leaves the + // session live, so no release owns the pruned credential's sweep. + yield* first.open(providerSessionId); + yield* peer.open(providerSessionId, peerThreadId); + const issued = (yield* Ref.get(mcpConfigs)).find((config) => config?.threadId === threadId); + const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(token); + + // A peer open for the same thread parks inside prepareMcpSession + // holding mcpPrepareLock[threadId] — the lock the detach's revocation + // sweep must acquire. + yield* Ref.set(pauseSettings, true); + const peerId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const peerOpening = yield* manager + .open({ threadId, providerSessionId: peerId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(preparing); + + const detaching = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 8; i += 1) { + yield* Effect.yieldNow; + } + // The revocation is parked behind the stalled peer prepare; the detach + // must report the unfinished cleanup within the 30-second bound + // instead of waiting on the peer lock indefinitely. + yield* TestClock.adjust("30 seconds"); + for (let i = 0; i < 8; i += 1) { + yield* Effect.yieldNow; + } + assert.equal((yield* Fiber.join(detaching))._tag, "Failure"); + + // The session stays live for the remaining attachment — only the + // revocation was bounded, not the detach's entry update. + assert.equal((yield* Ref.get(state)).closeCount, 0); + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + + // Late success: once the peer releases the lock, the detached sweep + // clears the thread's binding while the credential survives for its + // remaining holder. + yield* Deferred.succeed(prepareGate, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(peerOpening))); + for (let i = 0; i < 12; i += 1) { + yield* Effect.yieldNow; + } + assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); + assert.isDefined(yield* registry.resolve(token!)); + }).pipe( + Effect.ensuring(Deferred.succeed(prepareGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + serverSettingsLayer: settingsLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 bounds an exclusive terminal detach when the forked release stalls", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const pauseSettings = yield* Ref.make(false); + const preparing = yield* Deferred.make(); + const prepareGate = yield* Deferred.make(); + const settingsLayer = Layer.effect( + ServerSettings.ServerSettingsService, + Effect.gen(function* () { + const delegate = yield* ServerSettings.ServerSettingsService; + return ServerSettings.ServerSettingsService.of({ + ...delegate, + getSettings: Ref.get(pauseSettings).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(prepareGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.getSettings), + ), + }); + }), + ).pipe(Layer.provide(ServerSettings.layerTest({ enableAgentBrowserAccess: true }))); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const threadId = ThreadId.make("thread-detach-exclusive-revoke-bounded"); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* allocate; + yield* open(providerSessionId); + + // A peer open for the same thread parks inside prepareMcpSession + // holding threadLifecycle[threadId] and mcpPrepareLock[threadId] — + // the release's open drain and its credential sweep both wait behind + // it, so the release can never reach its scope close. + yield* Ref.set(pauseSettings, true); + const peerId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* manager + .open({ threadId, providerSessionId: peerId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(preparing); + + // The exclusive session's last detach forks the release and joins it; + // the release is parked behind the stalled peer. The detach must + // report the unfinished cleanup within the 30-second bound — the + // release keeps running and keeps ownership. + const detaching = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 8; i += 1) { + yield* Effect.yieldNow; + } + yield* TestClock.adjust("30 seconds"); + for (let i = 0; i < 8; i += 1) { + yield* Effect.yieldNow; + } + assert.equal((yield* Fiber.join(detaching))._tag, "Failure"); + }).pipe( + Effect.ensuring(Deferred.succeed(prepareGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + capabilities: ExclusiveCapabilities, + mcpConfigs, + serverSettingsLayer: settingsLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps the detached thread's ownership when close races the last detach", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const hangClose = yield* Deferred.make(); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const stepper = makeSteppingScheduler(); + // The window sits between the detach's attachment mutation and the + // release claim its forked worker performs. Stepping the outer detach + // fiber cannot reach it — the mutation and the fork share one op + // boundary and the worker's claim then runs during the same task + // drain. The discriminating pause is inside the fibers the detach + // spawns: hold the first spawned fiber at each of its own op offsets + // (offset 19 parks the detach's timeout worker after the mutation and + // before the release claim; later offsets hold the scope-close + // worker), let a racing close claim first, then check the pending + // record still covers this thread. + for (let childOps = 0; childOps < 48; childOps++) { + const threadId = ThreadId.make(`thread-detach-close-claim-${childOps}`); + const fixture = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* fixture.allocate; + const replacementId = yield* fixture.allocate; + yield* fixture.open(providerSessionId); + + const detaching = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe( + Effect.exit, + Effect.forkDetach, + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + ); + stepper.aim(detaching, Number.MAX_SAFE_INTEGER); + stepper.holdSpawnedAt(childOps); + yield* stepper.drain; + + // Interleave a close while the spawned release worker is held: it + // claims the emptied entry or joins the pending release — either + // way the parked scope close keeps the record alive for the gate + // check below. + const closing = yield* manager + .close(providerSessionId) + .pipe(Effect.exit, Effect.forkChild); + // Ambient turns for the close's claim — a synchronous map write a + // few scheduler turns in — before the held worker resumes. + for (let i = 0; i < 24; i += 1) { + yield* Effect.yieldNow; + } + stepper.resumeSpawned(); + yield* stepper.drain; + for (let i = 0; i < 24; i += 1) { + yield* Effect.yieldNow; + } + + const replacing = yield* manager + .open({ + providerSessionId: replacementId, + threadId, + modelSelection, + runtimePolicy, + }) + .pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 12; i += 1) { + yield* Effect.yieldNow; + } + const openExit = yield* Fiber.join(replacing); + assert.equal( + openExit._tag, + "Failure", + `close interleaved while the release worker was held at op ${childOps} let a same-thread replacement open during cleanup`, + ); + + yield* TestClock.adjust("30 seconds"); + stepper.resumeSpawned(); + yield* stepper.drain; + yield* Fiber.join(detaching); + yield* Fiber.join(closing); + } + }).pipe( + Effect.ensuring(Deferred.succeed(hangClose, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + capabilities: ExclusiveCapabilities, + hangSessionScopeClose: hangClose, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 fences a startup that finishes opening while shutdown is busy closing another session", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const openEntered = yield* Deferred.make(); + const openGate = yield* Deferred.make(); + const closeEntered = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + // beforeOpen runs for every session: park only the second open (B's) + // so A can register normally. beforeClose is likewise shared: park the + // first close (A's) so B's fenced unwind can still complete. + const opens = yield* Ref.make(0); + const closes = yield* Ref.make(0); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const a = yield* makeThreadSessionFixture(ThreadId.make("thread-shutdown-startup-race-a")); + const b = yield* makeThreadSessionFixture(ThreadId.make("thread-shutdown-startup-race-b")); + const aId = yield* a.allocate; + yield* a.open(aId); + const bId = yield* b.allocate; + const bOpening = yield* b.open(bId).pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(openEntered); + const stopping = yield* manager.shutdown.pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(closeEntered); + // Shutdown marked B's startup record before releasing A. Letting B's + // open resume now must not register a live session behind the + // shutdown's back. + yield* Deferred.succeed(openGate, undefined); + assert.equal((yield* Fiber.join(bOpening))._tag, "Failure"); + yield* Deferred.succeed(closeGate, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(stopping))); + assert.isTrue(Option.isNone(yield* manager.get(bId))); + // A's scope and B's fenced startup scope were both closed. + assert.equal((yield* Ref.get(state)).closeCount, 2); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + beforeOpen: () => + Ref.modify(opens, (n) => [n === 1, n + 1] as const).pipe( + Effect.andThen((second) => + second + ? Deferred.succeed(openEntered, undefined).pipe( + Effect.andThen(Deferred.await(openGate)), + ) + : Effect.void, + ), + ), + beforeClose: Ref.modify(closes, (n) => [n === 0, n + 1] as const).pipe( + Effect.andThen((first) => + first + ? Deferred.succeed(closeEntered, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + ) + : Effect.void, + ), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 fences a startup that finishes opening while closeInstance is busy closing another session", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const openEntered = yield* Deferred.make(); + const openGate = yield* Deferred.make(); + const closeEntered = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + const opens = yield* Ref.make(0); + const closes = yield* Ref.make(0); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const a = yield* makeThreadSessionFixture( + ThreadId.make("thread-closeinstance-startup-race-a"), + ); + const b = yield* makeThreadSessionFixture( + ThreadId.make("thread-closeinstance-startup-race-b"), + ); + const aId = yield* a.allocate; + yield* a.open(aId); + const bId = yield* b.allocate; + const bOpening = yield* b.open(bId).pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(openEntered); + const closing = yield* manager + .closeInstance(modelSelection.instanceId) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(closeEntered); + yield* Deferred.succeed(openGate, undefined); + assert.equal((yield* Fiber.join(bOpening))._tag, "Failure"); + yield* Deferred.succeed(closeGate, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(closing))); + assert.isTrue(Option.isNone(yield* manager.get(bId))); + assert.equal((yield* Ref.get(state)).closeCount, 2); + // The fence is transient: once the instance close settles, a fresh + // open for it proceeds. + const c = yield* makeThreadSessionFixture( + ThreadId.make("thread-closeinstance-startup-race-c"), + ); + const cId = yield* c.allocate; + yield* c.open(cId); + assert.isTrue(Option.isSome(yield* manager.get(cId))); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + beforeOpen: () => + Ref.modify(opens, (n) => [n === 1, n + 1] as const).pipe( + Effect.andThen((second) => + second + ? Deferred.succeed(openEntered, undefined).pipe( + Effect.andThen(Deferred.await(openGate)), + ) + : Effect.void, + ), + ), + beforeClose: Ref.modify(closes, (n) => [n === 0, n + 1] as const).pipe( + Effect.andThen((first) => + first + ? Deferred.succeed(closeEntered, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + ) + : Effect.void, + ), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 blocks a same-thread attach while a failed startup is still unwinding", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const unwinding = yield* Deferred.make(); + const unwindGate = yield* Deferred.make(); + const opens = yield* Ref.make(0); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const threadId = ThreadId.make("thread-failed-startup-unwind"); + const peerThreadId = ThreadId.make("thread-failed-startup-unwind-peer"); + const failing = yield* makeThreadSessionFixture(threadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const failedId = yield* failing.allocate; + // The first openSession defects; its scope close parks at + // beforeClose, so the startup keeps owning provider resources for + // the thread until the unwind finishes. + const opening = yield* failing.open(failedId).pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(unwinding); + const peerId = yield* peer.allocate; + const peerRuntime = yield* peer.open(peerId); + // The runtime attach path does not wait on the doomed open's + // lifecycle lock; it must be rejected by the startup mark alone. + const attachExit = yield* peerRuntime + .ensureThread({ threadId, modelSelection, runtimePolicy }) + .pipe(Effect.exit); + assert.isTrue(Exit.isFailure(attachExit)); + if (Exit.isFailure(attachExit)) { + const failure = Cause.findErrorOption(attachExit.cause); + assert.isTrue(Option.isSome(failure)); + if (Option.isSome(failure)) { + assert.include( + failure.value._tag === "ProviderAdapterProtocolError" ? failure.value.detail : "", + "cleanup", + ); + } + } + yield* Deferred.succeed(unwindGate, undefined); + assert.equal((yield* Fiber.join(opening))._tag, "Failure"); + // Once the unwind completed, the thread is free to attach again. + yield* peerRuntime + .ensureThread({ threadId, modelSelection, runtimePolicy }) + .pipe(Effect.ignore); + assert.isTrue(Option.isSome(yield* manager.get(peerId))); + }).pipe( + Effect.ensuring(Deferred.succeed(unwindGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + // afterOpen runs after the adapter registered its finalizers, so + // the defecting session still owns a scope that must be closed. + afterOpen: Ref.modify(opens, (n) => [n === 0, n + 1] as const).pipe( + Effect.andThen((first) => + first ? Effect.die(new Error("startup defect")) : Effect.void, + ), + ), + beforeClose: Deferred.succeed(unwinding, undefined).pipe( + Effect.andThen(Deferred.await(unwindGate)), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 rejects an attach that queued behind a doomed startup's mark", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const openEntered = yield* Deferred.make(); + const openGate = yield* Deferred.make(); + const preparing = yield* Deferred.make(); + const prepareGate = yield* Deferred.make(); + const pauseSettings = yield* Ref.make(false); + const opens = yield* Ref.make(0); + const settingsLayer = Layer.effect( + ServerSettings.ServerSettingsService, + Effect.gen(function* () { + const delegate = yield* ServerSettings.ServerSettingsService; + return ServerSettings.ServerSettingsService.of({ + ...delegate, + getSettings: Ref.get(pauseSettings).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(prepareGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.getSettings), + ), + }); + }), + ).pipe(Layer.provide(ServerSettings.layerTest({ enableAgentBrowserAccess: true }))); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const startupThreadId = ThreadId.make("thread-queued-attach-startup"); + const peerThreadId = ThreadId.make("thread-queued-attach-peer"); + const startup = yield* makeThreadSessionFixture(startupThreadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const startupId = yield* startup.allocate; + // The startup parks inside openSession with its record tracked but + // unmarked — runtime attaches for its thread still pass the guard. + const startupOpening = yield* startup.open(startupId).pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(openEntered); + const peerId = yield* peer.allocate; + const peerRuntime = yield* peer.open(peerId); + yield* Ref.set(pauseSettings, true); + // Attach #1 passes the guard before the close lands, then parks in + // credential preparation holding the session+thread attach lock. + const firstAttach = yield* peerRuntime + .ensureThread({ threadId: startupThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(preparing); + // Attach #2 queues behind the lock while the guard still passes. + const secondAttach = yield* peerRuntime + .ensureThread({ threadId: startupThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 4; i += 1) { + yield* Effect.yieldNow; + } + // Close marks the startup and joins its unwind; the unwind never + // finishes because the open stays parked, so the mark outlives the + // close's own bounded wait. + const closing = yield* manager.close(startupId).pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(closing))._tag, "Failure"); + // Letting attach #1 finish preparation must not resurrect the thread: + // the atomic recheck sees the doomed startup's mark. + yield* Deferred.succeed(prepareGate, undefined); + const firstExit = yield* Fiber.join(firstAttach); + const secondExit = yield* Fiber.join(secondAttach); + // Both attaches must be rejected by the doomed startup's mark, not by + // the unimplemented adapter call that follows a successful attach. + for (const attachExit of [firstExit, secondExit]) { + assert.isTrue(Exit.isFailure(attachExit)); + if (Exit.isFailure(attachExit)) { + const failure = Cause.findErrorOption(attachExit.cause); + assert.isTrue(Option.isSome(failure)); + if (Option.isSome(failure)) { + assert.include( + failure.value._tag === "ProviderAdapterProtocolError" ? failure.value.detail : "", + "cleanup", + ); + } + } + } + yield* Deferred.succeed(openGate, undefined); + assert.equal((yield* Fiber.join(startupOpening))._tag, "Failure"); + assert.isTrue(Option.isSome(yield* manager.get(peerId))); + }).pipe( + Effect.ensuring( + Deferred.succeed(prepareGate, undefined).pipe( + Effect.andThen(Deferred.succeed(openGate, undefined)), + ), + ), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + serverSettingsLayer: settingsLayer, + beforeOpen: () => + Ref.modify(opens, (n) => [n === 0, n + 1] as const).pipe( + Effect.andThen((first) => + first + ? Deferred.succeed(openEntered, undefined).pipe( + Effect.andThen(Deferred.await(openGate)), + ) + : Effect.void, + ), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 joins a pending credential revocation on terminal-detach retry", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const revoking = yield* Deferred.make(); + const revokeGate = yield* Deferred.make(); + const pauseRevoke = yield* Ref.make(false); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + revokeProviderSession: (providerSessionId) => + Ref.getAndSet(pauseRevoke, false).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(revokeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.revokeProviderSession(providerSessionId)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const threadId = ThreadId.make("thread-detach-revoke-retry"); + const peerThreadId = ThreadId.make("thread-detach-revoke-retry-peer"); + const first = yield* makeThreadSessionFixture(threadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const providerSessionId = yield* first.allocate; + // Shared session: detaching one thread leaves the session live for + // the other, so no release record owns the revocation — the tracked + // sweep is the only cleanup a retry can join. + yield* first.open(providerSessionId); + yield* peer.open(providerSessionId, peerThreadId); + const issued = (yield* Ref.get(mcpConfigs)).find((config) => config?.threadId === threadId); + const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(token); + + yield* Ref.set(pauseRevoke, true); + const detaching = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(revoking); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(detaching))._tag, "Failure"); + + // The retry must not report success over the still-parked sweep, and + // must not fork a second one. + const retry = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 8; i += 1) { + yield* Effect.yieldNow; + } + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(retry))._tag, "Failure"); + + // Late success: the tracked sweep completes and a later retry joins + // its recorded completion — the credential and the thread's binding + // are both gone. + yield* Deferred.succeed(revokeGate, undefined); + yield* manager.detach({ providerSessionId, threadId, revokeMcpCredential: true }); + assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); + assert.isUndefined(yield* registry.resolve(token!)); + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + }).pipe( + Effect.ensuring(Deferred.succeed(revokeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 queues a retry's pruned credential behind the pending revocation", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const revoking = yield* Deferred.make(); + const revokeGate = yield* Deferred.make(); + const pauseRevoke = yield* Ref.make(false); + const failResolve = yield* Ref.make(false); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + resolve: (token) => + Ref.get(failResolve).pipe( + Effect.flatMap((fail) => + fail ? Effect.succeed(undefined) : delegate.resolve(token), + ), + ), + revokeProviderSession: (providerSessionId) => + Ref.get(pauseRevoke).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(revokeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.revokeProviderSession(providerSessionId)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const threadId = ThreadId.make("thread-detach-revoke-chained"); + const peerThreadId = ThreadId.make("thread-detach-revoke-chained-peer"); + const session = yield* makeThreadSessionFixture(threadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + // Three sessions on one thread; a failing resolve forces a fresh + // credential per open so each session records a distinct credential + // and the configured binding ends on the third. + const sessionA = yield* session.allocate; + yield* session.open(sessionA); + yield* Ref.set(failResolve, true); + const sessionB = yield* session.allocate; + yield* session.open(sessionB); + yield* peer.open(sessionB, peerThreadId); + const sessionC = yield* session.allocate; + yield* session.open(sessionC); + yield* Ref.set(failResolve, false); + const configs = yield* Ref.get(mcpConfigs); + const tokenOf = (index: number) => + configs[index]?.authorizationHeader.replace(/^Bearer\s+/, ""); + const tokenA = tokenOf(0); + const tokenB = tokenOf(1); + const tokenC = tokenOf(2); + assert.isDefined(tokenA); + assert.isDefined(tokenB); + assert.isDefined(tokenC); + + // Detach A terminally; its tracked sweep captures {cA, cC} and parks + // inside the gated revocation. + yield* Ref.set(pauseRevoke, true); + const detachA = yield* manager + .detach({ providerSessionId: sessionA, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(revoking); + // Detach B's attachment while the sweep is parked: B stays live for + // its peer thread, so no release record owns cB — only this caller's + // captured set does. Joining the pending sweep must not discard it. + const detachB = yield* manager + .detach({ providerSessionId: sessionB, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 8; i += 1) { + yield* Effect.yieldNow; + } + + yield* Deferred.succeed(revokeGate, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(detachA))); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(detachB))); + for (let i = 0; i < 8; i += 1) { + yield* Effect.yieldNow; + } + // The chained sweep released B's pruned credential; C's claim on the + // configured credential survives. + assert.isUndefined(yield* registry.resolve(tokenA!)); + assert.isUndefined(yield* registry.resolve(tokenB!)); + assert.isDefined(yield* registry.resolve(tokenC!)); + assert.isTrue(Option.isSome(yield* manager.get(sessionB))); + }).pipe( + Effect.ensuring(Deferred.succeed(revokeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps a terminal-detach retry waiting on the pending revocation past a session release", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const revoking = yield* Deferred.make(); + const revokeGate = yield* Deferred.make(); + const pauseRevoke = yield* Ref.make(false); + const closing = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + revokeProviderSession: (providerSessionId) => + Ref.getAndSet(pauseRevoke, false).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(revokeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.revokeProviderSession(providerSessionId)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const threadId = ThreadId.make("thread-detach-revoke-release-join"); + const peerThreadId = ThreadId.make("thread-detach-revoke-release-join-peer"); + const first = yield* makeThreadSessionFixture(threadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const providerSessionId = yield* first.allocate; + yield* first.open(providerSessionId); + yield* peer.open(providerSessionId, peerThreadId); + const issued = (yield* Ref.get(mcpConfigs)).find((config) => config?.threadId === threadId); + const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(token); + + // The first terminal detach prunes the thread's credential and parks + // inside the tracked revocation sweep. + yield* Ref.set(pauseRevoke, true); + const detaching = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(revoking); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(detaching))._tag, "Failure"); + + // A close claims the session; its record never contained the pruned + // credential, so the parked sweep remains its only owner. + const closingSession = yield* manager + .close(providerSessionId) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(closing); + const retry = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 8; i += 1) { + yield* Effect.yieldNow; + } + + // The release finishes while the revocation is still parked; the + // retry must not report success over cleanup it does not own. + yield* Deferred.succeed(closeGate, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(closingSession))); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(retry))._tag, "Failure"); + + // Once the revocation actually drains, the retry's own bounded wait + // completes and the binding is gone. + yield* Deferred.succeed(revokeGate, undefined); + yield* manager.detach({ providerSessionId, threadId, revokeMcpCredential: true }); + assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); + assert.isUndefined(yield* registry.resolve(token!)); + }).pipe( + Effect.ensuring( + Deferred.succeed(revokeGate, undefined).pipe( + Effect.andThen(Deferred.succeed(closeGate, undefined)), + ), + ), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + mcpRegistryLayer, + beforeClose: Deferred.succeed(closing, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 closeInstance does not close another instance's same-id replacement", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const openEntered = yield* Deferred.make(); + const openGate = yield* Deferred.make(); + const closing = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + const closeCalls = yield* Ref.make(0); + const openCalls = yield* Ref.make(0); + const startupThreadId = ThreadId.make("thread-closeinstance-sameid-startup"); + const liveThreadId = ThreadId.make("thread-closeinstance-sameid-live"); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const live = yield* makeThreadSessionFixture(liveThreadId); + const liveId = yield* live.allocate; + yield* live.open(liveId); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: startupThreadId, + }); + // The startup parks inside openSession with its record tracked. + const startupOpening = yield* manager + .open({ threadId: startupThreadId, providerSessionId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(openEntered); + + const closingInstance = yield* manager + .closeInstance(modelSelection.instanceId) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(closing); + + // The fenced startup unwinds and drops its opening record while the + // instance close stays parked on the live session's scope. + yield* Deferred.succeed(openGate, undefined); + assert.equal((yield* Fiber.join(startupOpening))._tag, "Failure"); + // A different instance may reuse the same session id — the captured + // record is gone by now, so only the settled deferred still refers to + // the startup's own cleanup. + yield* manager.open({ + threadId: startupThreadId, + providerSessionId, + modelSelection: { + ...modelSelection, + instanceId: ProviderInstanceId.make("codex_other"), + }, + runtimePolicy, + }); + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + + yield* Deferred.succeed(closeGate, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(closingInstance))); + // Joining the captured record's settled deferred must not claim the + // replacement that registered after the record was removed. + assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); + }).pipe( + Effect.ensuring( + Deferred.succeed(openGate, undefined).pipe( + Effect.andThen(Deferred.succeed(closeGate, undefined)), + ), + ), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + // The live session opens first; the second open is the startup + // that must park while closeInstance captures its record. + beforeOpen: () => + Ref.modify(openCalls, (n) => [n === 1, n + 1] as const).pipe( + Effect.andThen((startup) => + startup + ? Deferred.succeed(openEntered, undefined).pipe( + Effect.andThen(Deferred.await(openGate)), + ) + : Effect.void, + ), + ), + // Only the live session's close parks: the startup's scope had no + // finalizers yet and the replacement's scope closes after the + // gate is already open. + beforeClose: Ref.modify(closeCalls, (n) => [n === 0, n + 1] as const).pipe( + Effect.andThen((first) => + first + ? Deferred.succeed(closing, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + ) + : Effect.void, + ), + ), + extraAdapters: [ + makeProviderAdapter(state, { + instanceId: ProviderInstanceId.make("codex_other"), + }), + ], + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 keeps a mid-flight terminal-detach retry waiting on the pending revocation", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const revoking = yield* Deferred.make(); + const revokeGate = yield* Deferred.make(); + const pauseRevoke = yield* Ref.make(false); + const interruptEntered = yield* Deferred.make(); + const interruptGate = yield* Deferred.make(); + const interruptCalls = yield* Ref.make(0); + const closing = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + const closeCalls = yield* Ref.make(0); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + revokeProviderSession: (providerSessionId) => + Ref.getAndSet(pauseRevoke, false).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(revokeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.revokeProviderSession(providerSessionId)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const eventSink = yield* EventSinkV2; + const idAllocator = yield* IdAllocatorV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-detach-revoke-midflight"); + const peerThreadId = ThreadId.make("thread-detach-revoke-midflight-peer"); + const first = yield* makeThreadSessionFixture(threadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const providerSessionId = yield* first.allocate; + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId }), + type: "provider-thread.updated", + threadId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: providerThread, + }, + { + id: yield* idAllocator.allocate.event({ threadId }), + type: "provider-turn.updated", + threadId, + runId, + driver: CODEX_DRIVER, + occurredAt: now, + payload: { + id: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-detach-revoke-midflight", + }), + providerThreadId: providerThread.id, + nodeId: idAllocator.derive.rootNode({ runId }), + runAttemptId: null, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }, + ], + }); + yield* first.open(providerSessionId); + yield* peer.open(providerSessionId, peerThreadId); + const issued = (yield* Ref.get(mcpConfigs)).find((config) => config?.threadId === threadId); + const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(token); + + // The first terminal detach prunes the credential and parks inside + // the tracked revocation sweep. + yield* Ref.set(pauseRevoke, true); + const detaching = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(revoking); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(detaching))._tag, "Failure"); + + // The retry reads the live entry, then parks inside interruptTurn — + // a close claims the session while it is suspended, and the release + // record it joins never contained the pruned credential. + const retry = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(interruptEntered); + const closingSession = yield* manager + .close(providerSessionId) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(closing); + + // The release finishes while the revocation is still parked; the + // retry must not report success over cleanup it does not own. + yield* Deferred.succeed(interruptGate, undefined); + yield* Deferred.succeed(closeGate, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(closingSession))); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(retry))._tag, "Failure"); + + // Once the revocation actually drains, a later retry completes and + // the binding is gone. + yield* Deferred.succeed(revokeGate, undefined); + yield* manager.detach({ providerSessionId, threadId, revokeMcpCredential: true }); + assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); + assert.isUndefined(yield* registry.resolve(token!)); + }).pipe( + Effect.ensuring( + Deferred.succeed(interruptGate, undefined).pipe( + Effect.andThen(Deferred.succeed(closeGate, undefined)), + Effect.andThen(Deferred.succeed(revokeGate, undefined)), + ), + ), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + mcpRegistryLayer, + // The first detach interrupts the seeded running turn unpaused; + // the retry parks inside its own interrupt. + beforeInterrupt: Ref.modify(interruptCalls, (n) => [n > 0, n + 1] as const).pipe( + Effect.andThen((seen) => + seen + ? Deferred.succeed(interruptEntered, undefined).pipe( + Effect.andThen(Deferred.await(interruptGate)), + ) + : Effect.void, + ), + ), + beforeClose: Ref.modify(closeCalls, (n) => [n === 0, n + 1] as const).pipe( + Effect.andThen((firstCall) => + firstCall + ? Deferred.succeed(closing, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + ) + : Effect.void, + ), + ), + }), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 shutdown joins a session release already in progress", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const closing = yield* Deferred.make(); + const closeGate = yield* Deferred.make(); + const closeCalls = yield* Ref.make(0); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const threadId = ThreadId.make("thread-shutdown-joins-release"); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* allocate; + yield* open(providerSessionId); + + const closingSession = yield* manager + .close(providerSessionId) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(closing); + + const shuttingDown = yield* manager.shutdown.pipe(Effect.exit, Effect.forkChild); + // The release is claimed and parked; shutdown must be waiting on it, + // not reporting teardown over running cleanup. + yield* TestClock.adjust("15 seconds"); + assert.isUndefined(shuttingDown.pollUnsafe()); + + yield* Deferred.succeed(closeGate, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(closingSession))); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(shuttingDown))); + }).pipe( + Effect.ensuring(Deferred.succeed(closeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + beforeClose: Ref.modify(closeCalls, (n) => [n === 0, n + 1] as const).pipe( + Effect.andThen((first) => + first + ? Deferred.succeed(closing, undefined).pipe( + Effect.andThen(Deferred.await(closeGate)), + ) + : Effect.void, + ), + ), + }), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 shutdown releases wedged sessions concurrently", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const hangClose = yield* Deferred.make(); + const closeEntered = yield* Ref.make(0); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const a = yield* makeThreadSessionFixture(ThreadId.make("thread-shutdown-concurrent-a")); + const b = yield* makeThreadSessionFixture(ThreadId.make("thread-shutdown-concurrent-b")); + const aId = yield* a.allocate; + yield* a.open(aId); + const bId = yield* b.allocate; + yield* b.open(bId); + + const stopping = yield* manager.shutdown.pipe(Effect.exit, Effect.forkChild); + // Both releases must START concurrently: beforeClose runs first on + // each scope close, so under a sequential loop only the first + // session's close has entered before the first bound elapses. + yield* Effect.forEach( + Array.from({ length: 200 }, () => undefined), + () => + Ref.get(closeEntered).pipe( + Effect.flatMap((n) => (n < 2 ? Effect.yieldNow : Effect.void)), + ), + { discard: true }, + ); + assert.equal(yield* Ref.get(closeEntered), 2); + // Both scope closes now park on the shared gate; each release is + // bounded at 30s. Sequential release would stack the bounds (60s+ + // here); concurrent release settles at the single bound. + yield* TestClock.adjust("29 seconds"); + yield* Effect.yieldNow; + assert.isUndefined(stopping.pollUnsafe()); + yield* TestClock.adjust("2 seconds"); + yield* Effect.yieldNow; + const exit = stopping.pollUnsafe(); + assert.isTrue(exit !== undefined && Exit.isSuccess(exit) && Exit.isSuccess(exit.value)); + }).pipe( + Effect.ensuring(Deferred.succeed(hangClose, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + hangSessionScopeClose: hangClose, + beforeClose: Ref.update(closeEntered, (n) => n + 1), + }), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 bounds the detach wait on a stalled in-flight attach", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const issuing = yield* Deferred.make(); + const issueGate = yield* Deferred.make(); + const attachThreadId = ThreadId.make("thread-detach-attach-lock"); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (input) => + (input.threadId === attachThreadId + ? Deferred.succeed(issuing, undefined).pipe(Effect.andThen(Deferred.await(issueGate))) + : Effect.void + ).pipe(Effect.andThen(delegate.issue(input))), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const threadId = ThreadId.make("thread-detach-attach-lock-peer"); + const first = yield* makeThreadSessionFixture(threadId); + const second = yield* makeThreadSessionFixture(attachThreadId); + const providerSessionId = yield* first.allocate; + yield* first.open(providerSessionId); + + // The second thread's attach parks inside MCP issuance while holding + // the [session, thread] attach lock. + const attaching = yield* second + .open(providerSessionId, attachThreadId) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(issuing); + + const detaching = yield* manager + .detach({ providerSessionId, threadId: attachThreadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + const detachExit = yield* Fiber.join(detaching); + assert.equal(detachExit._tag, "Failure"); + + yield* Deferred.succeed(issueGate, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(attaching))); + }).pipe( + Effect.ensuring(Deferred.succeed(issueGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + mcpRegistryLayer, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 shutdown joins a tracked credential revocation still running", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const revoking = yield* Deferred.make(); + const revokeGate = yield* Deferred.make(); + const pauseRevoke = yield* Ref.make(true); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + revokeProviderSession: (providerSessionId) => + Ref.getAndSet(pauseRevoke, false).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(revokeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.revokeProviderSession(providerSessionId)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const registry = yield* McpSessionRegistry.McpSessionRegistry; + const threadId = ThreadId.make("thread-shutdown-pending-revocation"); + const peerThreadId = ThreadId.make("thread-shutdown-pending-revocation-peer"); + const first = yield* makeThreadSessionFixture(threadId); + const peer = yield* makeThreadSessionFixture(peerThreadId); + const providerSessionId = yield* first.allocate; + yield* first.open(providerSessionId); + yield* peer.open(providerSessionId, peerThreadId); + const issued = (yield* Ref.get(mcpConfigs)).find((config) => config?.threadId === threadId); + const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(token); + + // The terminal detach prunes the thread's credential and parks inside + // the tracked revocation sweep — outliving the detach caller itself. + const detaching = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(revoking); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(detaching))._tag, "Failure"); + + // Shutdown releases the session (still owned by the peer thread) but + // must not return while the tracked revocation is still running. + const shuttingDown = yield* manager.shutdown.pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("15 seconds"); + assert.isUndefined(shuttingDown.pollUnsafe()); + + yield* Deferred.succeed(revokeGate, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(shuttingDown))); + assert.isUndefined(yield* registry.resolve(token!)); + }).pipe( + Effect.ensuring(Deferred.succeed(revokeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + mcpRegistryLayer, + }), + ), + ); + }), +); + +for (const operation of ["shutdown", "closeInstance"] as const) { + it.effect( + `ProviderSessionManagerV2 ${operation} waits for a revocation tail registered by an in-flight detach`, + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + // Teardown is one-shot, so every offset runs against a fresh manager. + // The detach's first scheduler checkpoints land while it is still + // queued on the [session, thread] lock — its in-flight marker is + // registered, but the masked mutation/registration stretch has not + // run, so no pendingRevocations record exists yet. + for (let ops = 4; ops < 24; ops += 1) { + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const stepper = makeSteppingScheduler(); + const threadId = ThreadId.make(`thread-${operation}-late-tail-${ops}`); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* allocate; + yield* open(providerSessionId); + + const detaching = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe( + Effect.exit, + Effect.forkDetach, + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + ); + yield* stepper.step(detaching, ops); + + const tearingDown = yield* ( + operation === "shutdown" + ? manager.shutdown + : manager.closeInstance(modelSelection.instanceId) + ).pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 16; i += 1) { + yield* Effect.yieldNow; + } + // The detach is in flight but has registered no revocation yet; + // only draining the in-flight marker keeps teardown pending here. + // A one-time snapshot returns Success over the owed sweep. + assert.isUndefined( + tearingDown.pollUnsafe(), + `${operation} returned while an in-flight detach could still register a revocation tail (op ${ops})`, + ); + + yield* stepper.drain; + const teardownExit = yield* Fiber.join(tearingDown); + assert.isTrue( + Exit.isSuccess(teardownExit), + `${operation} returned ${teardownExit._tag} instead of Success (op ${ops})`, + ); + // Teardown may only finish once the in-flight detach has exited — + // its tail registration and bounded join are what the drain waits + // on. + assert.isDefined( + detaching.pollUnsafe(), + `${operation} returned before the in-flight detach settled (op ${ops})`, + ); + const detachExit = yield* Fiber.join(detaching); + assert.isTrue( + Exit.isSuccess(detachExit), + `${operation}: in-flight detach returned ${detachExit._tag} (op ${ops})`, + ); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + }), + ), + ); + } + }), + ); +} + +it.effect( + "ProviderSessionManagerV2 closeInstance does not drain an in-flight detach owned by another instance", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const stepper = makeSteppingScheduler(); + const otherSelection = { + ...modelSelection, + instanceId: ProviderInstanceId.make("codex_other"), + }; + const foreignThread = ThreadId.make("thread-foreign-detach"); + const first = yield* makeThreadSessionFixture(ThreadId.make("thread-foreign-detach-owner")); + yield* makeThreadSessionFixture(foreignThread); + const sessionA = yield* first.allocate; + const sessionB = yield* idAllocator.allocate.providerSession({ + providerInstanceId: otherSelection.instanceId, + threadId: foreignThread, + }); + yield* first.open(sessionA); + yield* manager.open({ + threadId: foreignThread, + providerSessionId: sessionB, + modelSelection: otherSelection, + runtimePolicy, + }); + + // The foreign detach is in flight — marker registered, still queued + // on its session's attach lock — but belongs to codex_other, so this + // instance's teardown must not be held behind it. + const detaching = yield* manager + .detach({ + providerSessionId: sessionB, + threadId: foreignThread, + revokeMcpCredential: true, + }) + .pipe( + Effect.exit, + Effect.forkDetach, + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + ); + yield* stepper.step(detaching, 8); + + const closing = yield* manager + .closeInstance(modelSelection.instanceId) + .pipe(Effect.exit, Effect.forkChild); + const closeExit = yield* Fiber.join(closing); + assert.isTrue( + Exit.isSuccess(closeExit), + `closeInstance returned ${closeExit._tag} instead of Success`, + ); + assert.isUndefined( + detaching.pollUnsafe(), + "closeInstance drained a detach owned by another instance", + ); + + yield* stepper.drain; + yield* Fiber.join(detaching); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + extraAdapters: [ + makeProviderAdapter(state, { + instanceId: ProviderInstanceId.make("codex_other"), + }), + ], + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 closeInstance drains an in-flight detach across a same-id replacement", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const issueStarted = yield* Deferred.make(); + const issueGate = yield* Deferred.make(); + const armIssue = yield* Ref.make(false); + const detachedThread = ThreadId.make("thread-replaced-detach"); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (request) => + request.threadId === detachedThread + ? Ref.get(armIssue).pipe( + Effect.flatMap((armed) => + armed + ? Deferred.succeed(issueStarted, undefined).pipe( + Effect.andThen(Deferred.await(issueGate)), + Effect.andThen(delegate.issue(request)), + ) + : delegate.issue(request), + ), + ) + : delegate.issue(request), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const stepper = makeSteppingScheduler(); + const otherSelection = { + ...modelSelection, + instanceId: ProviderInstanceId.make("codex_other"), + }; + const first = yield* makeThreadSessionFixture(detachedThread); + const providerSessionId = yield* first.allocate; + yield* first.open(providerSessionId); + + // Suspend the detach after it recorded which instance's entry it + // observed (the capture lands around op 10) but before its + // [session, thread] lock section, so its marker attributes this + // instance while the session id is about to change hands. + const detaching = yield* manager + .detach({ + providerSessionId, + threadId: detachedThread, + revokeMcpCredential: true, + }) + .pipe( + Effect.exit, + Effect.forkDetach, + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + ); + yield* stepper.step(detaching, 15); + + // Release the observed session, then reopen the same session id under + // another instance: map lookups now resolve the marker's session to + // codex_other, and only the marker's captured owner still attributes + // the detach to this instance. + yield* manager.close(providerSessionId); + const replacementThread = ThreadId.make("thread-replaced-owner"); + yield* makeThreadSessionFixture(replacementThread); + yield* manager.open({ + threadId: replacementThread, + providerSessionId, + modelSelection: otherSelection, + runtimePolicy, + }); + + // A credential prepare for the detached thread on a third session id + // parks inside mcpPrepareLock[thread] once armed, so the detach's + // late-registered revocation tail is observably still running below. + const holderSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: otherSelection.instanceId, + threadId: detachedThread, + }); + yield* manager.open({ + threadId: ThreadId.make("thread-lock-session-owner"), + providerSessionId: holderSessionId, + modelSelection: otherSelection, + runtimePolicy, + }); + yield* Ref.set(armIssue, true); + const holding = yield* manager + .open({ + threadId: detachedThread, + providerSessionId: holderSessionId, + modelSelection: otherSelection, + runtimePolicy, + }) + .pipe(Effect.exit, Effect.forkChild); + // Mandatory handshake: the prepare must be parked inside the prepare + // lock before the detach's tail can be expected to queue behind it. + yield* Deferred.await(issueStarted); + + const closing = yield* manager + .closeInstance(modelSelection.instanceId) + .pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 16; i += 1) { + yield* Effect.yieldNow; + } + // Captured rather than asserted here: a premature teardown exit must + // still be followed by draining the suspended detach, or the layer's + // own shutdown waits on it forever and masks the real failure. + const skippedDetach = closing.pollUnsafe(); + + yield* stepper.drain; + for (let i = 0; i < 16; i += 1) { + yield* Effect.yieldNow; + } + const skippedTail = closing.pollUnsafe(); + + // The resumed detach registered its revocation tail and is parked on + // its join while the tail queues behind the held prepare lock. + // Interrupting the detach exits it while its tail is still running — + // teardown may only stay pending here by joining the tail itself. The + // interrupt is delivered through the stepping dispatcher the fiber is + // suspended on, so drain is what lands it at the parked join. + yield* Effect.sync(() => detaching.interruptUnsafe()); + yield* stepper.drain; + for (let i = 0; i < 16; i += 1) { + yield* Effect.yieldNow; + } + const detachExitEarly = detaching.pollUnsafe(); + const skippedLateTail = closing.pollUnsafe(); + + yield* Deferred.succeed(issueGate, undefined); + const closeExit = yield* Fiber.join(closing); + const detachExit = yield* Fiber.await(detaching); + yield* Fiber.join(holding); + + // The marker attributes the in-flight detach to this instance even + // though its session id now belongs to codex_other — resolving + // ownership through the current maps returns Success here instead. + assert.isUndefined( + skippedDetach, + `closeInstance skipped an in-flight detach that observed its own instance (${skippedDetach?._tag})`, + ); + // The resumed detach registered its revocation tail, which was then + // parked behind the held prepare lock — teardown must still be + // waiting rather than reporting over a live sweep. + assert.isUndefined( + skippedTail, + `closeInstance returned while the detached thread's revocation tail was still running (${skippedTail?._tag})`, + ); + // The detach exited while its tail was still parked — teardown must + // still be waiting on that tail rather than reporting over it. + assert.isDefined( + detachExitEarly, + "the interrupted in-flight detach did not exit while its tail was parked", + ); + assert.isTrue( + detachExitEarly !== undefined && Exit.isFailure(detachExit), + `in-flight detach returned ${detachExit._tag} instead of the interrupt Failure`, + ); + assert.isUndefined( + skippedLateTail, + `closeInstance returned while the detached thread's revocation tail was still running after the detach exited (${skippedLateTail?._tag})`, + ); + assert.isTrue( + Exit.isSuccess(closeExit), + `closeInstance returned ${closeExit._tag} instead of Success`, + ); + }).pipe( + Effect.ensuring(Deferred.succeed(issueGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + mcpRegistryLayer, + extraAdapters: [ + makeProviderAdapter(state, { + instanceId: ProviderInstanceId.make("codex_other"), + }), + ], + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 closeInstance waits for an in-flight detach that captured no owner", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const stepper = makeSteppingScheduler(); + const otherSelection = { + ...modelSelection, + instanceId: ProviderInstanceId.make("codex_other"), + }; + const detachedThread = ThreadId.make("thread-ownerless-detach"); + const first = yield* makeThreadSessionFixture(detachedThread); + const providerSessionId = yield* first.allocate; + yield* first.open(providerSessionId); + + // Suspend before the detach resolves which instance owns the session + // id (the capture lands around op 10), then close the session while + // it is suspended: the resumed capture finds no entry, so the marker + // records resolved-but-unknown ownership — not "not captured yet". + const detaching = yield* manager + .detach({ + providerSessionId, + threadId: detachedThread, + revokeMcpCredential: true, + }) + .pipe( + Effect.exit, + Effect.forkDetach, + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + ); + yield* stepper.step(detaching, 8); + yield* manager.close(providerSessionId); + yield* stepper.step(detaching, 12); + + // The same session id now belongs to another instance. A marker that + // conflated "captured no owner" with "not captured" would resolve the + // id through the maps here and skip this detach as foreign-owned. + const replacementThread = ThreadId.make("thread-ownerless-replacement"); + yield* makeThreadSessionFixture(replacementThread); + yield* manager.open({ + threadId: replacementThread, + providerSessionId, + modelSelection: otherSelection, + runtimePolicy, + }); + + const closing = yield* manager + .closeInstance(modelSelection.instanceId) + .pipe(Effect.exit, Effect.forkChild); + for (let i = 0; i < 16; i += 1) { + yield* Effect.yieldNow; + } + // Captured rather than asserted here: a premature teardown exit must + // still be followed by draining the suspended detach, or the layer's + // own shutdown waits on it forever and masks the real failure. + const skippedDetach = closing.pollUnsafe(); + + yield* stepper.drain; + for (let i = 0; i < 16; i += 1) { + yield* Effect.yieldNow; + } + const closeExit = yield* Fiber.join(closing); + const detachExit = yield* Fiber.join(detaching); + + assert.isUndefined( + skippedDetach, + `closeInstance skipped an in-flight detach that had resolved no owner for its session id (${skippedDetach?._tag})`, + ); + assert.isTrue( + Exit.isSuccess(closeExit), + `closeInstance returned ${closeExit._tag} instead of Success`, + ); + assert.isTrue( + Exit.isSuccess(detachExit), + `in-flight detach returned ${detachExit._tag} instead of Success`, + ); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + extraAdapters: [ + makeProviderAdapter(state, { + instanceId: ProviderInstanceId.make("codex_other"), + }), + ], + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 closeInstance reports an in-flight detach that outlives the cleanup bound", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const stepper = makeSteppingScheduler(); + const threadId = ThreadId.make("thread-stuck-detach"); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* allocate; + yield* open(providerSessionId); + + // The detach never resumes: its in-flight marker outlives the cleanup + // bound, and teardown must report that instead of discarding the + // timeout and returning success over a still-running detach. + const detaching = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe( + Effect.exit, + Effect.forkDetach, + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + ); + yield* stepper.step(detaching, 8); + + const closing = yield* manager + .closeInstance(modelSelection.instanceId) + .pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + const closeExit = yield* Fiber.join(closing); + // Drain and join before asserting: a teardown that wrongly reported + // success still leaves the suspended detach for the layer's own + // shutdown, which would hang and mask this failure. + yield* stepper.drain; + yield* Fiber.join(detaching); + assert.isTrue( + Exit.isFailure(closeExit), + `closeInstance returned ${closeExit._tag} instead of Failure`, + ); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 closeInstance joins a chained revocation started by its own instance", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const revoking = yield* Deferred.make(); + const revokeGate = yield* Deferred.make(); + const pauseRevoke = yield* Ref.make(true); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + revokeProviderSession: (providerSessionId) => + Ref.getAndSet(pauseRevoke, false).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(revokeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.revokeProviderSession(providerSessionId)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const threadId = ThreadId.make("thread-closeinstance-chained-revoke"); + const peerA = ThreadId.make("thread-closeinstance-chained-revoke-a"); + const peerB = ThreadId.make("thread-closeinstance-chained-revoke-b"); + const otherSelection = { + ...modelSelection, + instanceId: ProviderInstanceId.make("codex_other"), + }; + const firstA = yield* makeThreadSessionFixture(threadId); + const holderA = yield* makeThreadSessionFixture(peerA); + yield* makeThreadSessionFixture(peerB); + const sessionA = yield* firstA.allocate; + const sessionB = yield* idAllocator.allocate.providerSession({ + providerInstanceId: otherSelection.instanceId, + threadId: peerB, + }); + yield* firstA.open(sessionA); + yield* holderA.open(sessionA, peerA); + yield* manager.open({ + threadId: peerB, + providerSessionId: sessionB, + modelSelection: otherSelection, + runtimePolicy, + }); + // The thread is attached to both instances' sessions. + yield* manager.open({ + threadId, + providerSessionId: sessionB, + modelSelection: otherSelection, + runtimePolicy, + }); + + // A's terminal detach parks inside the tracked sweep; B's detach of + // the same thread chains behind it — the tail must still carry A's + // attribution because A's sweep is still running. + const detachA = yield* manager + .detach({ providerSessionId: sessionA, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(revoking); + const detachB = yield* manager + .detach({ providerSessionId: sessionB, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(detachA))._tag, "Failure"); + assert.equal((yield* Fiber.join(detachB))._tag, "Failure"); + + const closingInstance = yield* manager + .closeInstance(modelSelection.instanceId) + .pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("15 seconds"); + // A's predecessor sweep is still parked — teardown must be waiting. + assert.isUndefined(closingInstance.pollUnsafe()); + + yield* Deferred.succeed(revokeGate, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(closingInstance))); + }).pipe( + Effect.ensuring(Deferred.succeed(revokeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + mcpRegistryLayer, + extraAdapters: [ + makeProviderAdapter(state, { + instanceId: ProviderInstanceId.make("codex_other"), + }), + ], + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 closeInstance does not join an unrelated instance's revocation tail", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const revoking = yield* Deferred.make(); + const revokeGate = yield* Deferred.make(); + const pauseRevoke = yield* Ref.make(true); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + revokeProviderSession: (providerSessionId) => + Ref.getAndSet(pauseRevoke, false).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(revoking, undefined).pipe( + Effect.andThen(Deferred.await(revokeGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.revokeProviderSession(providerSessionId)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const threadId = ThreadId.make("thread-closeinstance-foreign-revoke"); + const peerA = ThreadId.make("thread-closeinstance-foreign-revoke-a"); + const threadB = ThreadId.make("thread-closeinstance-foreign-revoke-b"); + const otherSelection = { + ...modelSelection, + instanceId: ProviderInstanceId.make("codex_other"), + }; + const firstA = yield* makeThreadSessionFixture(threadId); + const holderA = yield* makeThreadSessionFixture(peerA); + yield* makeThreadSessionFixture(threadB); + const sessionA = yield* firstA.allocate; + const sessionB = yield* idAllocator.allocate.providerSession({ + providerInstanceId: otherSelection.instanceId, + threadId: threadB, + }); + yield* firstA.open(sessionA); + yield* holderA.open(sessionA, peerA); + yield* manager.open({ + threadId: threadB, + providerSessionId: sessionB, + modelSelection: otherSelection, + runtimePolicy, + }); + + // A's terminal detach parks inside the tracked sweep; the retry finds + // the thread already detached on the still-live session and chains a + // new tail — which must keep A's attribution from the observed entry. + const detaching = yield* manager + .detach({ providerSessionId: sessionA, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(revoking); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(detaching))._tag, "Failure"); + const retry = yield* manager + .detach({ providerSessionId: sessionA, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(retry))._tag, "Failure"); + + // B's teardown must not wait on — and must not fail on — a revocation + // owned by A. + const closingB = yield* manager + .closeInstance(otherSelection.instanceId) + .pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(closingB))); + // A's own teardown still joins its sweep. + const closingA = yield* manager + .closeInstance(modelSelection.instanceId) + .pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("15 seconds"); + assert.isUndefined(closingA.pollUnsafe()); + yield* Deferred.succeed(revokeGate, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(closingA))); + }).pipe( + Effect.ensuring(Deferred.succeed(revokeGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + mcpRegistryLayer, + extraAdapters: [ + makeProviderAdapter(state, { + instanceId: ProviderInstanceId.make("codex_other"), + }), + ], + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 closeInstance does not inherit a finished predecessor's pending successor", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const gateA = yield* Ref.make(undefined); + const gateB = yield* Ref.make(undefined); + const revokingA = yield* Deferred.make(); + const revokingB = yield* Deferred.make(); + const gateFirst = yield* Deferred.make(); + const gateSecond = yield* Deferred.make(); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + revokeProviderSession: (providerSessionId) => + Effect.gen(function* () { + const a = yield* Ref.get(gateA); + const b = yield* Ref.get(gateB); + if (providerSessionId === a) { + yield* Deferred.succeed(revokingA, undefined); + yield* Deferred.await(gateFirst); + } else if (providerSessionId === b) { + yield* Deferred.succeed(revokingB, undefined); + yield* Deferred.await(gateSecond); + } + }).pipe(Effect.andThen(delegate.revokeProviderSession(providerSessionId))), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const threadId = ThreadId.make("thread-closeinstance-settled-predecessor"); + const peerA = ThreadId.make("thread-closeinstance-settled-predecessor-a"); + const peerB = ThreadId.make("thread-closeinstance-settled-predecessor-b"); + const peerC = ThreadId.make("thread-closeinstance-settled-predecessor-c"); + const otherSelection = { + ...modelSelection, + instanceId: ProviderInstanceId.make("codex_other"), + }; + const firstA = yield* makeThreadSessionFixture(threadId); + const holderA = yield* makeThreadSessionFixture(peerA); + yield* makeThreadSessionFixture(peerB); + yield* makeThreadSessionFixture(peerC); + const sessionA = yield* firstA.allocate; + const sessionB = yield* idAllocator.allocate.providerSession({ + providerInstanceId: otherSelection.instanceId, + threadId: peerB, + }); + const sessionC = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: peerC, + }); + yield* firstA.open(sessionA); + yield* holderA.open(sessionA, peerA); + const credentialA = McpProviderSession.readMcpProviderSession(threadId)?.providerSessionId; + yield* Ref.set(gateA, credentialA); + yield* manager.open({ + threadId: peerB, + providerSessionId: sessionB, + modelSelection: otherSelection, + runtimePolicy, + }); + // The instance mismatch rotates the thread's credential so B's + // session claims its own. + yield* manager.open({ + threadId, + providerSessionId: sessionB, + modelSelection: otherSelection, + runtimePolicy, + }); + const credentialB = McpProviderSession.readMcpProviderSession(threadId)?.providerSessionId; + yield* Ref.set(gateB, credentialB); + // A third session re-rotates the binding, then closes so the binding + // is cleared: the first sweep then only carries A's credential and + // can finish while B's own sweep stays parked. + yield* manager.open({ + threadId: peerC, + providerSessionId: sessionC, + modelSelection, + runtimePolicy, + }); + yield* manager.open({ + threadId, + providerSessionId: sessionC, + modelSelection, + runtimePolicy, + }); + yield* manager.close(sessionC); + + const detachA = yield* manager + .detach({ providerSessionId: sessionA, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(revokingA); + const detachB = yield* manager + .detach({ providerSessionId: sessionB, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + assert.equal((yield* Fiber.join(detachA))._tag, "Failure"); + assert.equal((yield* Fiber.join(detachB))._tag, "Failure"); + + // A's sweep finishes; B's chained sweep parks on its own credential. + yield* Deferred.succeed(gateFirst, undefined); + yield* Deferred.await(revokingB); + + // A's teardown must not wait on — or fail on — B's still-running sweep. + const closingA = yield* manager + .closeInstance(modelSelection.instanceId) + .pipe(Effect.exit, Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(closingA))); + + yield* Deferred.succeed(gateSecond, undefined); + }).pipe( + Effect.ensuring( + Deferred.succeed(gateFirst, undefined).pipe( + Effect.andThen(Deferred.succeed(gateSecond, undefined)), + ), + ), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + mcpRegistryLayer, + extraAdapters: [ + makeProviderAdapter(state, { + instanceId: ProviderInstanceId.make("codex_other"), + }), + ], + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 an interrupted pending-release detach cannot strand the revocation tail", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + // Per-iteration gate: each close must stay parked inside its scope + // finalizer so the detach observes a pending release and takes the + // tracked-revocation path. + const closeGate = yield* Ref.make< + | { + readonly entered: Deferred.Deferred; + readonly release: Deferred.Deferred; + } + | undefined + >(undefined); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const stepper = makeSteppingScheduler(); + + // Interrupting between the tail registration and the worker fork + // used to leave a `done` that never settles. There is no suspension + // between the two, so only a per-op boundary reaches the window: + // sweep the interrupt across every op offset instead of racing it. + for (let ops = 0; ops < 128; ops++) { + const threadId = ThreadId.make(`thread-detach-stranded-tail-${ops}`); + const peerId = ThreadId.make(`thread-detach-stranded-tail-peer-${ops}`); + const first = yield* makeThreadSessionFixture(threadId); + const peer = yield* makeThreadSessionFixture(peerId); + const providerSessionId = yield* first.allocate; + yield* first.open(providerSessionId); + yield* peer.open(providerSessionId, peerId); + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + yield* Ref.set(closeGate, { entered, release }); + const closingSession = yield* manager + .close(providerSessionId) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(entered); + const detaching = yield* manager + .detach({ providerSessionId, threadId, revokeMcpCredential: true }) + .pipe( + Effect.exit, + Effect.forkDetach, + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + ); + // A detach legitimately parks once it joins the parked release; + // interrupt wherever the fiber stopped, then end the sweep when the + // target boundary was not reached — later offsets are unreachable. + const reached = yield* stepper.step(detaching, ops); + // Interrupt while the fiber is still suspended: the resumption is + // queued into the stepping dispatcher, so drain delivers the + // interrupt deterministically at this op boundary. + yield* Effect.sync(() => detaching.interruptUnsafe()); + yield* stepper.drain; + yield* Fiber.await(detaching); + + yield* Deferred.succeed(release, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(closingSession))); + + // A stranded tail never settles; the next join parks until the 30s + // bound. A healthy tail settles during the drain above, so the join + // completes without any clock advance. + const closingInstance = yield* manager + .closeInstance(modelSelection.instanceId) + .pipe(Effect.exit, Effect.forkChild); + yield* stepper.drain; + if (closingInstance.pollUnsafe() === undefined) { + yield* TestClock.adjust("30 seconds"); + } + assert.isTrue( + Exit.isSuccess(yield* Fiber.join(closingInstance)), + `interrupt at op ${ops} stranded the revocation tail`, + ); + if (!reached) break; + } + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + beforeClose: Ref.get(closeGate).pipe( + Effect.flatMap((gate) => + gate === undefined + ? Effect.void + : Deferred.succeed(gate.entered, undefined).pipe( + Effect.andThen(Deferred.await(gate.release)), + ), + ), + ), + }), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 an interrupted startup cannot strand the opening record", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const stepper = makeSteppingScheduler(); + const threadId = ThreadId.make("thread-open-stranded-record"); + yield* makeThreadSessionFixture(threadId); + + // Interrupting after `opening.set` but before the unwind handler is + // installed used to strand the record: a later close marked it and + // joined a `settled` that never completed, failing at the 30s bound + // while replacements stayed blocked. The gap has no suspension, so + // sweep the interrupt across every op offset. `cwd: null` skips the + // async `fileSystem.stat`, which would park the fiber mid-sweep and + // leave offsets past the park unreachable. + const startupPolicy = { ...runtimePolicy, cwd: null }; + for (let ops = 0; ops < 160; ops++) { + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const opening = yield* manager + .open({ + providerSessionId, + threadId, + modelSelection, + runtimePolicy: startupPolicy, + }) + .pipe( + Effect.exit, + Effect.forkDetach, + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + ); + // Interrupt wherever the fiber stopped, then end the sweep when the + // target boundary was not reached — later offsets are unreachable. + const reached = yield* stepper.step(opening, ops); + // Interrupt while the fiber is still suspended: the resumption is + // queued into the stepping dispatcher, so drain delivers the + // interrupt deterministically at this op boundary. + yield* Effect.sync(() => opening.interruptUnsafe()); + yield* stepper.drain; + yield* Fiber.await(opening); + + // A stranded record keeps its `settled` pending forever; the close + // marks it and joins, so it only resolves at the 30s bound — and + // fails. A settled or absent record lets the close finish during + // the drain, before any clock advance. + const closing = yield* manager.close(providerSessionId).pipe(Effect.exit, Effect.forkChild); + yield* stepper.drain; + if (closing.pollUnsafe() === undefined) { + yield* TestClock.adjust("30 seconds"); + } + assert.isTrue( + Exit.isSuccess(yield* Fiber.join(closing)), + `interrupt at op ${ops} stranded the opening record`, + ); + if (!reached) break; + } + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + }), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 close joins an open still resolving its workspace", () => + Effect.gen(function* () { + const realFileSystem = yield* FileSystem.FileSystem; + const root = yield* realFileSystem.makeTempDirectoryScoped(); + const cwd = `${root}/workspace`; + yield* realFileSystem.makeDirectory(cwd); + const statEntered = yield* Deferred.make(); + const statRelease = yield* Deferred.make(); + const gatedFileSystem = new Proxy(realFileSystem, { + get: (target, property, receiver) => + property === "stat" + ? (path: string) => + path === cwd + ? Deferred.succeed(statEntered, undefined).pipe( + Effect.andThen(Deferred.await(statRelease)), + Effect.andThen(target.stat(path)), + ) + : target.stat(path) + : Reflect.get(target, property, receiver), + }); + const state = yield* Ref.make(emptyState); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const threadId = ThreadId.make("thread-close-during-workspace-stat"); + const { allocate } = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* allocate; + const opening = yield* manager + .open({ + providerSessionId, + threadId, + modelSelection, + runtimePolicy: { ...runtimePolicy, cwd }, + }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(statEntered); + const closing = yield* manager.close(providerSessionId).pipe(Effect.exit, Effect.forkChild); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + // The startup record is registered before the workspace stat, so the + // close marks it and joins its unwind — it cannot complete while the + // open is still parked. When the stat resolves, the marked record + // fences the spawn and the unwind settles the close. + assert.isTrue(closing.pollUnsafe() === undefined); + yield* Deferred.succeed(statRelease, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(closing))); + assert.equal((yield* Fiber.join(opening))._tag, "Failure"); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + fileSystemLayer: Layer.succeed(FileSystem.FileSystem, gatedFileSystem), + }), + ), + ); + }).pipe(Effect.provide(NodeServices.layer)), +); + +it.effect("ProviderSessionManagerV2 closeInstance joins only its own unfinished sweeps", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const mcpConfigs = yield* Ref.make< + ReadonlyArray + >([]); + const gateA = yield* Ref.make(undefined); + const gateB = yield* Ref.make(undefined); + const revokingA = yield* Deferred.make(); + const revokingB = yield* Deferred.make(); + const gateFirst = yield* Deferred.make(); + const gateSecond = yield* Deferred.make(); + const releaseArmed = yield* Ref.make(false); + const releaseStarted = yield* Deferred.make(); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + revokeProviderSession: (providerSessionId) => + Effect.gen(function* () { + const a = yield* Ref.get(gateA); + const b = yield* Ref.get(gateB); + if (providerSessionId === a) { + yield* Deferred.succeed(revokingA, undefined); + yield* Deferred.await(gateFirst); + } else if (providerSessionId === b) { + yield* Deferred.succeed(revokingB, undefined); + yield* Deferred.await(gateSecond); + } + }).pipe(Effect.andThen(delegate.revokeProviderSession(providerSessionId))), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const threadId = ThreadId.make("thread-closeinstance-own-sweep"); + const peerA = ThreadId.make("thread-closeinstance-own-sweep-a"); + const peerB = ThreadId.make("thread-closeinstance-own-sweep-b"); + const peerC = ThreadId.make("thread-closeinstance-own-sweep-c"); + const otherSelection = { + ...modelSelection, + instanceId: ProviderInstanceId.make("codex_other"), + }; + const firstA = yield* makeThreadSessionFixture(threadId); + const holderA = yield* makeThreadSessionFixture(peerA); + yield* makeThreadSessionFixture(peerB); + yield* makeThreadSessionFixture(peerC); + const sessionA = yield* firstA.allocate; + const sessionB = yield* idAllocator.allocate.providerSession({ + providerInstanceId: otherSelection.instanceId, + threadId: peerB, + }); + const sessionC = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: peerC, + }); + yield* firstA.open(sessionA); + yield* holderA.open(sessionA, peerA); + const credentialA = McpProviderSession.readMcpProviderSession(threadId)?.providerSessionId; + yield* Ref.set(gateA, credentialA); + yield* manager.open({ + threadId: peerB, + providerSessionId: sessionB, + modelSelection: otherSelection, + runtimePolicy, + }); + yield* manager.open({ + threadId, + providerSessionId: sessionB, + modelSelection: otherSelection, + runtimePolicy, + }); + const credentialB = McpProviderSession.readMcpProviderSession(threadId)?.providerSessionId; + yield* Ref.set(gateB, credentialB); + yield* manager.open({ + threadId: peerC, + providerSessionId: sessionC, + modelSelection, + runtimePolicy, + }); + yield* manager.open({ + threadId, + providerSessionId: sessionC, + modelSelection, + runtimePolicy, + }); + yield* manager.close(sessionC); + + // Sweep A parks on its credential; B's detach chains a successor tail + // behind it. + const detachA = yield* manager + .detach({ providerSessionId: sessionA, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(revokingA); + const detachB = yield* manager + .detach({ providerSessionId: sessionB, threadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + // closeInstance starts while A's sweep is still pending. The release + // handshake means it is parked on the revocation join by the time the + // test resumes. + yield* Ref.set(releaseArmed, true); + const closingA = yield* manager + .closeInstance(modelSelection.instanceId) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(releaseStarted); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + // A's sweep finishes while B's chained sweep stays parked on its own + // credential. Joining the matching record — not the tail — lets A's + // teardown complete now. + yield* Deferred.succeed(gateFirst, undefined); + yield* Deferred.await(revokingB); + yield* Effect.yieldNow; + const early = closingA.pollUnsafe(); + assert.isTrue(early !== undefined && Exit.isSuccess(early)); + + // detachA joined its own tail, which settled when A's sweep finished; + // detachB is still parked behind B's sweep and fails at the bound. + yield* TestClock.adjust("30 seconds"); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(detachA))); + assert.equal((yield* Fiber.join(detachB))._tag, "Failure"); + yield* Deferred.succeed(gateSecond, undefined); + assert.isTrue(Exit.isSuccess(yield* Fiber.join(closingA))); + }).pipe( + Effect.ensuring( + Deferred.succeed(gateFirst, undefined).pipe( + Effect.andThen(Deferred.succeed(gateSecond, undefined)), + ), + ), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpConfigs, + mcpRegistryLayer, + beforeClose: Ref.get(releaseArmed).pipe( + Effect.flatMap((armed) => + armed ? Deferred.succeed(releaseStarted, undefined).pipe(Effect.asVoid) : Effect.void, + ), + ), + extraAdapters: [ + makeProviderAdapter(state, { + instanceId: ProviderInstanceId.make("codex_other"), + }), + ], + }), + ), + ); + }), +); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index d6f9d9059744..c50ed554f707 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -4,6 +4,7 @@ import { OrchestrationV2DomainEvent, OrchestrationV2ProviderSession, OrchestrationV2RuntimeRequest, + ProviderDriverKind, ProviderInstanceId, ProviderSessionId, ThreadId, @@ -12,6 +13,7 @@ import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; @@ -37,6 +39,7 @@ import { makeKeyedSerialExecutor } from "./KeyedSerialExecutor.ts"; import { ProviderEventIngestorV2 } from "./ProviderEventIngestor.ts"; import { ProviderAdapterEventStreamError, + ProviderAdapterProtocolError, ProviderAdapterV2RuntimePolicy, type ProviderAdapterV2Error, type ProviderAdapterV2Event, @@ -49,6 +52,18 @@ import { ProjectionStoreV2 } from "./ProjectionStore.ts"; const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000; const RELEASE_SCOPE_CLOSE_TIMEOUT_MS = 30 * 1000; +// OpenCode runs turn work in-band: compactThread/startTurn yield* the whole +// session.summarize inside the call instead of forking it into the session +// scope. An unbounded drain would let a stalled summarize wedge the release, +// so turn calls for these drivers get a bounded wait — the scope close is +// what aborts them. Every other adapter's turn call is acquire-then-return +// and keeps the unbounded drain: abandoning a pending acquisition (Cursor's +// runner.open) would let it install a provider resource after the session +// scope has already closed, with no finalizer left to release it. +const ADAPTER_OP_DRAIN_TIMEOUT_MS = 20 * 1000; +const INBAND_TURN_CALL_DRIVERS: ReadonlySet = new Set([ + ProviderDriverKind.make("opencode"), +]); export const ProviderSessionReleaseReason = Schema.Literals([ "idle_timeout", @@ -116,6 +131,8 @@ export class ProviderSessionReleaseError extends Schema.TaggedError()( "ProviderSessionActivityError", { @@ -209,6 +226,22 @@ interface LiveSessionEntry { readonly pinnedSinceMs: number | null; } +interface PendingSessionRelease { + readonly entry: LiveSessionEntry; + readonly threadIds: ReadonlySet; + readonly mcpCredentialIdByThread: Map; + readonly done: Deferred.Deferred; +} + +interface OpeningSessionRecord { + readonly providerSessionId: ProviderSessionId; + readonly threadId: ThreadId; + readonly instanceId: ProviderInstanceId; + closeRequested: boolean; + failed: ProviderSessionReleaseError | undefined; + readonly settled: Deferred.Deferred; +} + type ProviderSessionEventSignal = | { readonly type: "event"; readonly event: ProviderAdapterV2Event } | { @@ -359,14 +392,303 @@ export const layerWithOptions = ( ); const layerScope = yield* Effect.scope; const sessions = yield* Ref.make(new Map()); + + // Keep ownership after removal from the live map until cleanup actually finishes. + const releasing = new Map(); + // Adapter operations that may create native resources (ensureThread, + // resumeThread) register here before invoking the adapter and settle the + // deferred when they exit. A release drains the set before closing the + // session scope so a late adapter result can never be installed after + // the finalizer that would have closed it. + const inflightAdapterOps = new Map< + string, + Set<{ + readonly done: Deferred.Deferred; + readonly drainTimeout: Duration.Duration | undefined; + }> + >(); + // Startup ownership before a session entry exists: the open registers + // here before any adapter work so a racing close/detach can mark it and + // join its unwind instead of reporting success over a spawning provider + // process. `closeRequested`/`failed` are only written inside + // Ref.modify(sessions) or the opening fiber's uninterruptible unwind, so + // the registration fence below reads them atomically. + const opening = new Map(); + // Teardown fences: once closeInstance bumps its instance's count (or + // shutdown flips the flag), no new startup record may be created and + // no in-flight startup may register — a record created afterwards + // would escape the mark-and-join the close performs on the records it + // saw. Read and written only inside synchronous steps (Ref.modify or + // straight-line gen code), where no other fiber can interleave. + const closingInstanceCounts = new Map(); + let shutdownInitiated = false; + // Terminal-detach revocations outlive the call that started them: each + // tracked sweep owns the credentials its caller pruned, so a retry + // joins this record (bounded like any cleanup wait) and queues its own + // captured set behind it instead of discarding or duplicating work. + interface PendingRevocation { + // Attribution reflects only sweeps still running: each record carries + // its own caller's instance and a link to the sweep it waits behind, + // and `settled` marks a record whose own sweep finished — a completed + // predecessor must not keep its caller's teardown waiting on a + // successor it does not own. undefined means the sweep could not be + // attributed and must be joined by every closeInstance. + readonly instanceId: ProviderInstanceId | undefined; + readonly predecessor: PendingRevocation | undefined; + settled: boolean; + readonly done: Deferred.Deferred; + } + const pendingRevocations = new Map(); + // Every running detach carries one marker from its first uninterruptible + // step to its exit. Teardown drains this set before snapshotting + // pendingRevocations: a detach that started earlier can still be inside + // its lock, not yet at the forkTrackedRevocation call that registers the + // sweep, and a one-time snapshot would return while that tail is still + // being created. + interface InflightDetach { + readonly providerSessionId: ProviderSessionId; + readonly threadId: ThreadId; + readonly detached: Deferred.Deferred; + // Captured once the detach's body resolves which instance's entry it + // is detaching from: the maps can re-resolve the session id to a + // same-id replacement owned by another instance, so drain attribution + // reads what the detach observed, not what owns the key now. + // `ownershipResolved` distinguishes "has not looked yet" (fall back to + // the maps) from "looked and found no entry" (an unattributed tail — + // owned by every teardown, so the marker must still be drained). + ownershipResolved: boolean; + instanceId: ProviderInstanceId | undefined; + // Published when the detach registers its tracked sweep. The record + // leaves pendingRevocations on settle, so a tail that registers and + // finishes while teardown is draining this marker is joined through + // this deferred instead. + revokeDone: Deferred.Deferred | undefined; + } + const inflightDetaches = new Set(); + // Joins every unfinished revocation record matching `owned` and drains + // in-flight detaches that may still register a matching sweep — a tail + // registered while an earlier pass was joining is still awaited rather + // than left running past teardown. Each marker is awaited at most once: + // one that outlives its bound is reported through `awaitInflight`, not + // re-waited — the detach is already past the contract and re-waiting + // would make teardown unbounded. Every pass collects records, markers, + // and published tails in one synchronous stretch before deciding, so + // nothing can register between the last scan and the return. + const joinTrackedRevocations = ( + owned: (record: PendingRevocation) => boolean, + join: ( + threadId: ThreadId, + done: Deferred.Deferred, + ) => Effect.Effect, + awaitInflight: (marker: InflightDetach) => Effect.Effect, + ownsInflight: ( + marker: InflightDetach, + live: ReadonlyMap, + ) => boolean, + ) => + Effect.gen(function* () { + const outcomes: Array> = []; + const joined = new Set>(); + const awaited = new Set(); + for (;;) { + const live = yield* Ref.get(sessions); + const fresh: Array< + readonly [ThreadId, Deferred.Deferred] + > = []; + for (const [threadId, record] of pendingRevocations) { + for ( + let current: PendingRevocation | undefined = record; + current !== undefined; + current = current.predecessor + ) { + if (!current.settled && owned(current) && !joined.has(current.done)) { + joined.add(current.done); + fresh.push([threadId, current.done]); + } + } + } + const pending: Array = []; + const lateTails: Array< + readonly [ThreadId, Deferred.Deferred] + > = []; + for (const marker of inflightDetaches) { + if (!ownsInflight(marker, live)) continue; + if (!awaited.has(marker)) { + awaited.add(marker); + pending.push(marker); + } + const tail = marker.revokeDone; + if (tail !== undefined && !joined.has(tail)) { + joined.add(tail); + lateTails.push([marker.threadId, tail]); + } + } + if (fresh.length === 0 && pending.length === 0 && lateTails.length === 0) { + return outcomes; + } + const joins: Array>> = [ + ...[...fresh, ...lateTails].map(([threadId, done]) => + join(threadId, done).pipe(Effect.exit), + ), + ...pending.map((marker) => awaitInflight(marker).pipe(Effect.exit)), + ]; + const pass = yield* Effect.all(joins, { concurrency: "unbounded" }); + outcomes.push(...pass); + // A marker that just exited published its tail at most once; join + // it here, while the detach's completion has made the field final. + const tailJoins: Array>> = + []; + for (const marker of pending) { + const tail = marker.revokeDone; + if (tail !== undefined && !joined.has(tail)) { + joined.add(tail); + tailJoins.push(join(marker.threadId, tail).pipe(Effect.exit)); + } + } + outcomes.push(...(yield* Effect.all(tailJoins, { concurrency: "unbounded" }))); + } + }); + const forkTrackedRevocation = ( + providerSessionId: ProviderSessionId, + threadId: ThreadId, + revoke: Effect.Effect, + instanceId: ProviderInstanceId | undefined, + onRegistered?: (done: Deferred.Deferred) => void, + ) => + // Registration and the worker fork are one uninterruptible handoff: + // an interrupt landing between them would leave a tail whose `done` + // never settles, parking every later join and chaining every retry + // behind it. The forked worker is a separate fiber and is unaffected. + // `onRegistered` runs inside the same handoff so a caller publishing + // the deferred (an in-flight detach marker) cannot be interrupted out + // of that publication while the record is already visible. + Effect.uninterruptible( + Effect.gen(function* () { + // A tracked sweep captures only the credentials its caller pruned; + // joining it must not discard this caller's own set. Every call + // registers its own completion and chains its sweep behind the + // previous one, so a later retry always joins the newest tail. + const predecessor = pendingRevocations.get(threadId); + const done = Deferred.makeUnsafe(); + const record: PendingRevocation = { + instanceId, + predecessor, + settled: false, + done, + }; + pendingRevocations.set(threadId, record); + onRegistered?.(done); + yield* Effect.gen(function* () { + if (predecessor !== undefined) { + // A failed predecessor reports on its own deferred; this + // caller's captured credentials still need their sweep. + yield* Effect.ignore(Deferred.await(predecessor.done)); + } + const exit = yield* Effect.exit(revoke); + // The record must come down with its completion: a stale entry + // would let a later retry join a finished sweep and skip + // revoking a freshly configured binding. A chained successor + // may already have replaced the map entry — only delete our own. + yield* Effect.uninterruptible( + Effect.gen(function* () { + record.settled = true; + if (pendingRevocations.get(threadId) === record) { + pendingRevocations.delete(threadId); + } + yield* Deferred.done( + done, + Exit.isSuccess(exit) + ? Exit.void + : Exit.fail( + new ProviderSessionReleaseError({ + providerSessionId, + reason: "runtime_error", + cause: exit.cause, + }), + ), + ); + }), + ); + }).pipe(Effect.forkDetach({ startImmediately: true })); + return done; + }), + ); + const findPendingRelease = ( + providerSessionId: ProviderSessionId, + threadId: ThreadId, + existing: LiveSessionEntry | undefined, + ) => + Array.from(releasing.values()).find( + (release) => + release.entry.runtime.providerSessionId === providerSessionId || + (release.threadIds.has(threadId) && !existing?.attachedThreadIds.has(threadId)), + ); + // A startup record that failed its unwind, or is still unwinding after a + // close request, may still own provider resources for its session id and + // thread — the same contract findPendingRelease enforces for releases. + const findBlockingStartup = (providerSessionId: ProviderSessionId, threadId: ThreadId) => + Array.from(opening.values()).find( + (record) => + (record.providerSessionId === providerSessionId || record.threadId === threadId) && + (record.closeRequested || record.failed !== undefined), + ); + // Runtime methods attach threads outside `open`, so the pending-release + // block must hold here too — and it must propagate: observeActivity only + // logs bookkeeping failures. + const rejectPendingThreadAttachment = (input: { + readonly providerSessionId: ProviderSessionId; + readonly threadId: ThreadId; + readonly driver: ProviderDriverKind; + }) => + Effect.gen(function* () { + const existing = (yield* Ref.get(sessions)).get(sessionKey(input.providerSessionId)); + if ( + findPendingRelease(input.providerSessionId, input.threadId, existing) !== undefined || + findBlockingStartup(input.providerSessionId, input.threadId) !== undefined + ) { + return yield* new ProviderAdapterProtocolError({ + driver: input.driver, + detail: "A previous provider session has not finished cleanup.", + }); + } + }); const nextSubscriberId = yield* Ref.make(0); const sessionOpen = yield* makeKeyedSerialExecutor(); + const threadLifecycle = yield* makeKeyedSerialExecutor(); + const releaseStatus = yield* makeKeyedSerialExecutor(); + // Serializes attaches of one thread to one session: a concurrent caller + // must wait for an in-flight attach to become durable instead of riding + // its provisional attachment — an attach that later fails unwinds the + // bookkeeping and credential claim the second caller was relying on. + // The key carries the runtime's identity, not just session+thread: a + // same-id reopen always installs a new runtime, and an abandoned in-band + // call (e.g. an OpenCode turn call still running past its drain bound) + // must not serialize the replacement's attaches behind a lock the dead + // runtime may never release. + const threadAttach = yield* makeKeyedSerialExecutor(); + const threadAttachRuntimeSeq = new WeakMap(); + let nextThreadAttachRuntimeSeq = 0; + const threadAttachKey = ( + providerSessionId: ProviderSessionId, + runtime: ProviderAdapterV2SessionRuntime | undefined, + threadId: ThreadId, + ) => { + let seq = -1; + if (runtime !== undefined) { + const existing = threadAttachRuntimeSeq.get(runtime); + if (existing === undefined) { + seq = nextThreadAttachRuntimeSeq++; + threadAttachRuntimeSeq.set(runtime, seq); + } else { + seq = existing; + } + } + return `${providerSessionId}\0${seq}\0${threadId}`; + }; const idleTimeoutMs = Math.max(1, options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS); const maxIdlePinMs = Math.max(0, options.maxIdlePinMs ?? DEFAULT_MAX_IDLE_PIN_MS); interface PreparedMcpCredential { readonly mcpCredentialId: string | undefined; - /** True when this call minted the credential (vs reusing a live one). */ - readonly issued: boolean; } /** * Reservations protect a credential between prepareMcpSession handing it @@ -395,7 +717,91 @@ export const layerWithOptions = ( }; const isMcpCredentialReserved = (threadId: ThreadId, mcpCredentialId: string) => (mcpCredentialReservations.get(mcpReservationKey(threadId, mcpCredentialId)) ?? 0) > 0; + // Anything still able to claim a credential: a held reservation, a live + // session's or pending release's recorded credential. Claims are + // credential-specific: a session attached to the thread claims only the + // credential it recorded at attach — a rotated-out token may still be + // held by a different session, and a session's own record is the claim + // that decides that. The attach-to-record gap is covered by the + // reservation prepareMcpSession holds until the entry records it. + // Read `sessions` first and pass the same snapshot in. + const isMcpCredentialClaimed = ( + current: ReadonlyMap, + threadId: ThreadId, + mcpCredentialId: string, + ) => + isMcpCredentialReserved(threadId, mcpCredentialId) || + Array.from(current.values()).some( + (other) => other.mcpCredentialIdByThread.get(threadId) === mcpCredentialId, + ) || + Array.from(releasing.values()).some( + (other) => other.mcpCredentialIdByThread.get(threadId) === mcpCredentialId, + ); const mcpPrepareLock = yield* makeKeyedSerialExecutor(); + /** + * Revokes the credential when no live session, pending release, or + * in-flight reservation still claims it. Callers must already hold + * mcpPrepareLock for this thread — every reservation is taken there, so + * the claim check and the delete must run under it too or a prepare + * could reserve and resolve the credential in between and end up + * holding a revoked token. + */ + const releaseUnclaimedMcpCredentialPrepared = (threadId: ThreadId, mcpCredentialId: string) => + Effect.gen(function* () { + const claimed = isMcpCredentialClaimed( + yield* Ref.get(sessions), + threadId, + mcpCredentialId, + ); + if (!claimed) { + yield* clearMcpSession(threadId, mcpCredentialId).pipe(Effect.ignore); + } + }); + const releaseUnclaimedMcpCredential = (threadId: ThreadId, mcpCredentialId: string) => + mcpPrepareLock.withLock( + threadId, + releaseUnclaimedMcpCredentialPrepared(threadId, mcpCredentialId), + ); + + /** + * Terminal-detach credential revocation: the thread is gone, so its + * config slot is cleared while it still binds one of the credentials + * this detach captured — a different session may have reconfigured the + * thread while the sweep was suspended, and that binding is not ours + * to clear. Each known credential is revoked only once nothing claims + * it: a credential shared with a still-live session or pending release + * stays valid until its final holder's own cleanup revokes it. + */ + const revokeUnclaimedThreadCredentials = ( + threadId: ThreadId, + extraCredentialIds: Iterable, + ) => + Effect.gen(function* () { + const captured = new Set(extraCredentialIds); + const candidates = new Set(captured); + const configured = McpProviderSession.readMcpProviderSession(threadId)?.providerSessionId; + if (configured !== undefined) { + candidates.add(configured); + } + yield* Effect.forEach( + candidates, + (mcpCredentialId) => releaseUnclaimedMcpCredential(threadId, mcpCredentialId), + { discard: true }, + ); + // Re-read under the prepare lock: a peer prepare may have + // configured a fresh credential while the candidate sweep ran. + yield* mcpPrepareLock.withLock( + threadId, + Effect.sync(() => { + const configuredNow = + McpProviderSession.readMcpProviderSession(threadId)?.providerSessionId; + if (configuredNow === undefined || captured.has(configuredNow)) { + McpProviderSession.clearMcpProviderSession(threadId); + } + }), + ); + }); + /** * Resolves (or mints) the thread's MCP credential and returns it with a * reservation held; the caller must drop the reservation exactly once. @@ -405,11 +811,20 @@ export const layerWithOptions = ( const prepareMcpSession = ( threadId: ThreadId, providerInstanceId: ProviderInstanceId, + /** + * Invoked synchronously every time this call's held reservation + * changes: with the credential id as each one is reserved, and with + * undefined when a reservation is dropped internally (a resolved + * credential that mismatched is released before issuing a fresh one). + * Callers track it so a failed or interrupted prepare can still drop + * the reservation it currently holds. + */ + onReservationChanged?: (mcpCredentialId: string | undefined) => void, ): Effect.Effect => options.configureMcp === false ? Effect.sync((): PreparedMcpCredential => { McpProviderSession.clearMcpProviderSession(threadId); - return { mcpCredentialId: undefined, issued: false }; + return { mcpCredentialId: undefined }; }) : mcpPrepareLock.withLock( threadId, @@ -431,7 +846,11 @@ export const layerWithOptions = ( if (existing !== undefined) { // Reserve before the async resolve so a release cannot // revoke the credential between validation and reservation. + // The caller tracks the in-flight reservation through + // onReservationChanged so a failed or interrupted prepare + // can still unwind it. reserveMcpCredential(threadId, existing.providerSessionId); + onReservationChanged?.(existing.providerSessionId); const rawToken = existing.authorizationHeader.replace(/^Bearer\s+/, ""); const resolved = yield* mcpSessionRegistry.resolve(rawToken); if ( @@ -443,11 +862,22 @@ export const layerWithOptions = ( resolved.capabilities.has("preview") === browserToolsAvailable && resolved.capabilities.has("device") === deviceToolsAvailable ) { - return { mcpCredentialId: existing.providerSessionId, issued: false }; + return { mcpCredentialId: existing.providerSessionId }; } dropMcpCredentialReservation(threadId, existing.providerSessionId); + // The dropped reservation may have been the credential's + // last claim: holders that closed while resolve was + // suspended already discarded their records. + onReservationChanged?.(undefined); + yield* releaseUnclaimedMcpCredentialPrepared( + threadId, + existing.providerSessionId, + ); } - yield* mcpSessionRegistry.revokeThread(threadId); + // No thread-wide revocation here: a mismatched credential was + // already revoked above when unclaimed, and a still-claimed + // one belongs to a live process that keeps using it until its + // final holder releases it. const credential = yield* mcpSessionRegistry.issue({ threadId, providerInstanceId, @@ -456,36 +886,28 @@ export const layerWithOptions = ( }); McpProviderSession.setMcpProviderSession(credential.config); reserveMcpCredential(threadId, credential.config.providerSessionId); - return { mcpCredentialId: credential.config.providerSessionId, issued: true }; + onReservationChanged?.(credential.config.providerSessionId); + return { mcpCredentialId: credential.config.providerSessionId }; }), ); /** - * With a credential id, revocation is scoped to that credential and the - * config slot is cleared only while it still holds it; a replacement - * session's newer credential survives. Without one (attach failed before - * a credential was recorded), fall back to thread-wide revocation. + * Revocation is always scoped to one credential, and the config slot is + * cleared only while it still holds it; a replacement session's newer + * credential survives. */ - const clearMcpSession = (threadId: ThreadId, mcpCredentialId?: string) => - mcpCredentialId === undefined - ? mcpSessionRegistry - .revokeThread(threadId) - .pipe( - Effect.tap(() => - Effect.sync(() => McpProviderSession.clearMcpProviderSession(threadId)), - ), - ) - : mcpSessionRegistry.revokeProviderSession(mcpCredentialId).pipe( - Effect.tap(() => - Effect.sync(() => { - if ( - McpProviderSession.readMcpProviderSession(threadId)?.providerSessionId === - mcpCredentialId - ) { - McpProviderSession.clearMcpProviderSession(threadId); - } - }), - ), - ); + const clearMcpSession = (threadId: ThreadId, mcpCredentialId: string) => + mcpSessionRegistry.revokeProviderSession(mcpCredentialId).pipe( + Effect.tap(() => + Effect.sync(() => { + if ( + McpProviderSession.readMcpProviderSession(threadId)?.providerSessionId === + mcpCredentialId + ) { + McpProviderSession.clearMcpProviderSession(threadId); + } + }), + ), + ); const publishToSubscribers = ( subscribers: Ref.Ref< @@ -707,148 +1129,344 @@ export const layerWithOptions = ( readonly cancelIdleFiber?: boolean; readonly onlyIfIdleGeneration?: number; readonly gracefulSubscribers?: boolean; + readonly alreadyLocked?: boolean; + // A caller that already observed a pending release must join that exact + // cleanup. If it finished meanwhile, the old session is gone and a + // same-id replacement must not be claimed in its place. + readonly joinRelease?: { + readonly done: Deferred.Deferred; + }; + // Join pending cleanup for this session id but never claim a live + // entry: the caller's target may already be replaced. + readonly joinOnly?: boolean; + // Only act on the session instance that owns this runtime: a stale + // caller must not release a same-id replacement opened meanwhile. + readonly expectedRuntime?: ProviderAdapterV2SessionRuntime; + // A release record the caller already registered atomically with its + // own state mutation (detach moves the last exclusive attachment into + // `releasing` inside its Ref.modify): skip selection and run the + // cleanup worker plus the bounded join for that exact record. + readonly preclaimed?: PendingSessionRelease; }) => - Effect.acquireUseRelease( - Ref.modify(sessions, (current) => { + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { const key = sessionKey(input.providerSessionId); - const existing = current.get(key); - if (existing === undefined) { - return [Option.none(), current] as const; - } - if ( - input.onlyIfIdleGeneration !== undefined && - (existing.busyCount > 0 || existing.idleGeneration !== input.onlyIfIdleGeneration) - ) { - return [Option.none(), current] as const; + const selected = + input.preclaimed !== undefined + ? Option.some({ release: input.preclaimed, start: true } as const) + : yield* Ref.modify( + sessions, + ( + current, + ): readonly [ + Option.Option< + | { readonly release: PendingSessionRelease; readonly start: boolean } + | { readonly opening: OpeningSessionRecord } + >, + Map, + ] => { + const pending = releasing.get(key); + if (input.joinRelease !== undefined) { + return [ + pending !== undefined && pending.done === input.joinRelease.done + ? Option.some({ release: pending, start: false }) + : Option.none(), + current, + ] as const; + } + if (pending !== undefined) { + return [ + input.expectedRuntime === undefined || + pending.entry.runtime === input.expectedRuntime + ? Option.some({ release: pending, start: false }) + : Option.none(), + current, + ] as const; + } + if (input.joinOnly === true) { + return [Option.none(), current] as const; + } + const existing = current.get(key); + if ( + existing === undefined || + (input.expectedRuntime !== undefined && + existing.runtime !== input.expectedRuntime) + ) { + // An in-flight startup has no entry to claim but can already + // own resources (a spawning process, a held credential + // reservation). A pinned-runtime caller targets a different + // instance and must not touch it; everyone else marks it so + // registration fences, then joins its unwind below. + const openingRecord = + input.expectedRuntime === undefined ? opening.get(key) : undefined; + if (openingRecord !== undefined) { + if (openingRecord.failed === undefined) { + openingRecord.closeRequested = true; + } + return [Option.some({ opening: openingRecord }), current] as const; + } + return [Option.none(), current] as const; + } + if ( + input.onlyIfIdleGeneration !== undefined && + (existing.busyCount > 0 || + existing.idleGeneration !== input.onlyIfIdleGeneration) + ) { + return [Option.none(), current] as const; + } + const release = { + entry: existing, + threadIds: new Set([ + ...existing.attachedThreadIds, + ...existing.mcpCredentialIdByThread.keys(), + ]), + mcpCredentialIdByThread: new Map(existing.mcpCredentialIdByThread), + done: Deferred.makeUnsafe(), + }; + releasing.set(key, release); + const updated = new Map(current); + updated.delete(key); + return [Option.some({ release, start: true }), updated] as const; + }, + ); + if (Option.isNone(selected)) return; + if ("opening" in selected.value) { + const openingRecord = selected.value.opening; + // The open's own unwind performs the cleanup; this join is + // bounded like any cleanup wait and the record stays (marked + // closeRequested, or failed) until the unwind finishes, so a + // timeout or interruption here cannot let a replacement open + // race ahead of a dying startup. + const settled = yield* restore( + Deferred.await(openingRecord.settled).pipe( + Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), + ), + ); + if (Option.isNone(settled)) { + return yield* new ProviderSessionReleaseError({ + providerSessionId: input.providerSessionId, + reason: input.reason, + cause: + "Provider session startup cleanup did not finish within 30 seconds. " + + "Replacement sessions are blocked until cleanup completes.", + }); + } + return; } - const updated = new Map(current); - updated.delete(key); - return [Option.some(existing), updated] as const; - }), - (entry) => - Option.match(entry, { - onNone: () => Effect.void, - onSome: (entry) => - Effect.gen(function* () { - if (input.cancelIdleFiber !== false) { - yield* cancelIdleFiber(entry.idleFiber); - } - if (input.gracefulSubscribers === true) { - yield* endSubscribers(entry); - } else if (input.reason === "server_shutdown") { - yield* closeSubscribers(entry); - } else { - yield* failSubscribers( - entry, - input.detail ?? `Provider session released: ${input.reason}.`, - ); - } - // Scope close can wedge on a misbehaving adapter finalizer - // (e.g. a provider process that never yields its message - // stream). Time-box it so release still persists released - // events and leaves a diagnosable trail instead of silently - // parking the session as "ready" forever. - const closeFiber = yield* Scope.close(entry.scope, Exit.void).pipe( - Effect.exit, - Effect.forkDetach({ startImmediately: true }), + const { release, start } = selected.value; + if (start) { + const releaseCredentials = Effect.gen(function* () { + // Scope cleanup succeeded. Stop claiming these credentials before + // checking peers, even if persisting the released status later fails. + // Each credential re-reads `sessions`: a peer may record a claim + // while an earlier revocation in this sweep is suspended, and a + // snapshot taken once here would miss it. + const credentials = [...release.mcpCredentialIdByThread]; + release.mcpCredentialIdByThread.clear(); + yield* Effect.forEach( + credentials, + ([threadId, mcpCredentialId]) => + releaseUnclaimedMcpCredential(threadId, mcpCredentialId), + { discard: true }, + ); + }); + const closeScope = Effect.gen(function* () { + const entry = release.entry; + if (input.cancelIdleFiber !== false) { + yield* cancelIdleFiber(entry.idleFiber); + } + if (input.gracefulSubscribers === true) { + yield* endSubscribers(entry); + } else if (input.reason === "server_shutdown") { + yield* closeSubscribers(entry); + } else { + yield* failSubscribers( + entry, + input.detail ?? `Provider session released: ${input.reason}.`, ); - const closeExit = yield* Fiber.join(closeFiber).pipe( - Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), + } + yield* Scope.close(entry.scope, Exit.void); + // On a failed or interrupted close the release stays pending and + // its provider process may still be live — the claims in + // `release.mcpCredentialIdByThread` must remain recorded until a + // cleanup actually completes, or a sweep could revoke a credential + // that process still uses. + yield* releaseCredentials; + }); + // Drain in-flight opens, then release their locks before closing the scope. + // The adapter-op drain is lock-free: an admitted + // runtime.ensureThread/resumeThread registers in + // inflightAdapterOps before invoking the adapter, so it either + // settles before the scope closes or is refused by its ownership + // recheck — a provider resource created mid-close must never + // outlive the finalizer that would have closed it. The set is + // closed at claim time (registration requires the live entry), + // and an adapter call never joins this release — no drain + // deadlock. The alreadyLocked caller holds the open/lifecycle + // locks (skip reacquiring them) but its session is still + // reachable through `get`, so the op drain still applies. + const drainAdapterOps = Effect.gen(function* () { + const ops = inflightAdapterOps.get(key); + if (ops === undefined) return; + let abandoned = 0; + yield* Effect.forEach( + ops, + (op) => + op.drainTimeout === undefined + ? Deferred.await(op.done) + : Deferred.await(op.done).pipe( + Effect.timeoutOption(op.drainTimeout), + Effect.tap((settled) => + Option.isNone(settled) + ? Effect.sync(() => { + abandoned += 1; + }) + : Effect.void, + ), + ), + { discard: true, concurrency: "unbounded" }, + ); + // An op that outlives its drain bound is left to the scope + // close — the finalizer aborts the work it was doing. This is + // the long-lived turn shape (e.g. OpenCode summarize), not an + // acquisition, so proceeding does not strand a late install. + if (abandoned > 0) { + yield* Effect.logWarning( + "orchestration-v2.driver-session.adapter-op-drain-timeout", + { + providerSessionId: input.providerSessionId, + abandoned, + }, ); - if (Option.isNone(closeExit)) { - yield* Effect.logWarning( - "orchestration-v2.provider-session-scope-close-timeout", - { - providerSessionId: input.providerSessionId, - reason: input.reason, - timeoutMs: RELEASE_SCOPE_CLOSE_TIMEOUT_MS, - }, - ); - yield* Fiber.join(closeFiber).pipe( - Effect.flatMap((exit) => - Exit.isFailure(exit) - ? Effect.logWarning( - "orchestration-v2.provider-session-scope-close-failed", - { - providerSessionId: input.providerSessionId, - reason: input.reason, - cause: exit.cause, - }, - ) - : Effect.logInfo( - "orchestration-v2.provider-session-scope-close-completed-late", - { - providerSessionId: input.providerSessionId, - reason: input.reason, - }, - ), + } + inflightAdapterOps.delete(key); + }); + const waitForOpens = + input.alreadyLocked === true + ? drainAdapterOps + : Array.from(release.threadIds) + .sort() + .reduceRight( + (effect, threadId) => threadLifecycle.withLock(threadId, effect), + sessionOpen.withLock(input.providerSessionId, drainAdapterOps), + ); + const completeRelease = (exit: Exit.Exit) => + releaseStatus.withLock( + input.providerSessionId, + Effect.gen(function* () { + const reportExit = yield* Effect.gen(function* () { + if (Exit.isFailure(exit)) { + yield* Effect.logWarning("orchestration-v2.driver-session.cleanup-failed", { + providerSessionId: input.providerSessionId, + reason: input.reason, + cause: exit.cause, + }); + } + yield* writeReleasedSessionEvents({ + entry: release.entry, + reason: Exit.isFailure(exit) ? "runtime_error" : input.reason, + ...(Exit.isFailure(exit) + ? { detail: "Provider session cleanup failed." } + : input.detail === undefined + ? {} + : { detail: input.detail }), + }); + yield* writeReleasedRuntimeRequestEvents({ + entry: release.entry, + reason: Exit.isFailure(exit) ? "runtime_error" : input.reason, + }).pipe(release.entry.requestEventPermit.withPermits(1)); + }).pipe( + Effect.mapError( + (cause) => + new ProviderSessionReleaseError({ + providerSessionId: input.providerSessionId, + reason: input.reason, + cause, + }), ), - Effect.forkDetach, + Effect.exit, ); - } - yield* writeReleasedSessionEvents({ - entry, - reason: input.reason, - ...(input.detail === undefined ? {} : { detail: input.detail }), - }); - yield* writeReleasedRuntimeRequestEvents({ - entry, - reason: input.reason, - }).pipe(entry.requestEventPermit.withPermits(1)); - if (Option.isSome(closeExit) && Exit.isFailure(closeExit.value)) { - return yield* Effect.failCause(closeExit.value.cause); - } - }), - }), - (entry) => - Option.match(entry, { - onNone: () => Effect.void, - onSome: (entry) => - // Revoke every credential this session recorded, including for - // threads that detached without re-attaching: the provider - // process is gone, so nothing holds them anymore. Skip threads - // a live replacement session took over, since credential reuse - // means the replacement may hold this very credential. - Ref.get(sessions).pipe( - Effect.flatMap((current) => - Effect.forEach( - entry.mcpCredentialIdByThread, - ([threadId, mcpCredentialId]) => { - // Id-sensitive: a stale record for the same thread but - // a DIFFERENT credential (left behind by an old session - // the thread rotated away from) must not veto revoking - // this session's own credential, or it leaks forever. - // A reservation means an in-flight open is configuring - // a provider process with this credential right now; - // revoking it here would strand that process (eager - // adapters cannot pick up a rotated token). - const heldElsewhere = - isMcpCredentialReserved(threadId, mcpCredentialId) || - Array.from(current.values()).some( - (other) => - other !== entry && - (other.attachedThreadIds.has(threadId) || - other.mcpCredentialIdByThread.get(threadId) === mcpCredentialId), - ); - return heldElsewhere - ? Effect.void - : clearMcpSession(threadId, mcpCredentialId); + const completed = Exit.isFailure(exit) ? exit : reportExit; + if (Exit.isSuccess(exit)) releasing.delete(key); + yield* Deferred.done(release.done, completed); + }), + ); + yield* waitForOpens.pipe( + Effect.andThen(closeScope), + Effect.catchCause((cause) => + Effect.fail( + new ProviderSessionReleaseError({ + providerSessionId: input.providerSessionId, + reason: input.reason, + cause, + }), + ), + ), + Effect.onExit(completeRelease), + Effect.forkDetach({ startImmediately: true }), + ); + } + const result = yield* restore( + Deferred.await(release.done).pipe( + Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), + ), + ); + if (Option.isNone(result)) { + const detail = + "Provider session cleanup did not finish within 30 seconds. Replacement sessions are blocked until cleanup completes."; + // Report through the same releaseStatus lock so the error lands + // in order with an attach or completion write still holding it — + // but report detached: that write may be suspended indefinitely, + // and this caller's bounded wait must not depend on it. + yield* releaseStatus + .withLock( + input.providerSessionId, + Effect.gen(function* () { + if (yield* Deferred.isDone(release.done)) return; + yield* writeReleasedSessionEvents({ + entry: release.entry, + reason: "runtime_error", + detail, + }); + yield* writeReleasedRuntimeRequestEvents({ + entry: release.entry, + reason: "runtime_error", + }).pipe(release.entry.requestEventPermit.withPermits(1)); + }), + ) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning( + "orchestration-v2.driver-session.cleanup-timeout-report-failed", + { + providerSessionId: input.providerSessionId, + cause, }, - { discard: true }, ), ), - ), - }), + Effect.forkDetach({ startImmediately: true }), + ); + if (yield* Deferred.isDone(release.done)) return yield* Deferred.await(release.done); + return yield* new ProviderSessionReleaseError({ + providerSessionId: input.providerSessionId, + reason: input.reason, + }); + } + }), ).pipe( - Effect.catchCause((cause) => - Effect.fail( + Effect.catchCause((cause) => { + const error = Cause.findErrorOption(cause); + if (Option.isSome(error) && isProviderSessionReleaseError(error.value)) { + return Effect.fail(error.value); + } + return Effect.fail( new ProviderSessionReleaseError({ providerSessionId: input.providerSessionId, reason: input.reason, cause, }), - ), - ), + ); + }), ); // Annotated to break the releaseIfStillIdle <-> scheduleIdleReleaseInternal @@ -856,6 +1474,7 @@ export const layerWithOptions = ( const releaseIfStillIdle = (input: { readonly providerSessionId: ProviderSessionId; readonly generation: number; + readonly expectedRuntime?: ProviderAdapterV2SessionRuntime; }): Effect.Effect => Effect.gen(function* () { const current = yield* Ref.get(sessions); @@ -864,7 +1483,11 @@ export const layerWithOptions = ( if ( entry === undefined || entry.busyCount > 0 || - entry.idleGeneration !== input.generation + entry.idleGeneration !== input.generation || + // An orphaned timer (displaced by a concurrent scheduler before + // being recorded in the entry) must never act on a same-id + // replacement's entry. + (input.expectedRuntime !== undefined && entry.runtime !== input.expectedRuntime) ) { return; } @@ -924,6 +1547,7 @@ export const layerWithOptions = ( reason: "idle_timeout", cancelIdleFiber: false, onlyIfIdleGeneration: input.generation, + expectedRuntime: probedRuntime, }).pipe( Effect.catchCause((cause) => Effect.logWarning("orchestration-v2.driver-session.idle-release-failed", { @@ -949,90 +1573,180 @@ export const layerWithOptions = ( ), ); - const scheduleIdleReleaseInternal = (providerSessionId: ProviderSessionId) => - Effect.gen(function* () { - const key = sessionKey(providerSessionId); - const current = yield* Ref.get(sessions); - const entry = current.get(key); - if (entry === undefined || entry.busyCount > 0) { - return; - } - - yield* cancelIdleFiber(entry.idleFiber); - const generation = entry.idleGeneration + 1; - const idleFiber = yield* Effect.sleep(Duration.millis(idleTimeoutMs)).pipe( - Effect.andThen(releaseIfStillIdle({ providerSessionId, generation })), - Effect.forkIn(layerScope), - ); - const lastActivityAtMs = yield* Clock.currentTimeMillis; - yield* Ref.update(sessions, (latest) => { - const latestEntry = latest.get(key); - if (latestEntry === undefined || latestEntry.busyCount > 0) { - return latest; + const scheduleIdleReleaseInternal = ( + providerSessionId: ProviderSessionId, + expectedRuntime?: ProviderAdapterV2SessionRuntime, + ) => + // Uninterruptible: cancelling the tracked timer suspends until that + // fiber dies, and an interruption landing before the replacement is + // installed would leave the entry pointing at a dead fiber with no + // idle cleanup left. The atomic install check still decides whether + // the new timer is kept or cancelled. + Effect.uninterruptible( + Effect.gen(function* () { + const key = sessionKey(providerSessionId); + const current = yield* Ref.get(sessions); + const entry = current.get(key); + if ( + entry === undefined || + entry.busyCount > 0 || + (expectedRuntime !== undefined && entry.runtime !== expectedRuntime) + ) { + return; } - const updated = new Map(latest); - updated.set(key, { - ...latestEntry, - idleGeneration: generation, - idleFiber, - lastActivityAtMs, + + yield* cancelIdleFiber(entry.idleFiber); + const generation = entry.idleGeneration + 1; + const idleFiber = yield* Effect.sleep(Duration.millis(idleTimeoutMs)).pipe( + Effect.andThen( + releaseIfStillIdle({ + providerSessionId, + generation, + ...(expectedRuntime === undefined ? {} : { expectedRuntime }), + }), + ), + Effect.forkIn(layerScope), + ); + const lastActivityAtMs = yield* Clock.currentTimeMillis; + const installed = yield* Ref.modify(sessions, (latest) => { + const latestEntry = latest.get(key); + if ( + latestEntry === undefined || + latestEntry.busyCount > 0 || + // A concurrent scheduler installed its timer since our read. + // Yield to it: keeping ours would orphan the tracked fiber. + latestEntry.idleFiber !== entry.idleFiber || + (expectedRuntime !== undefined && latestEntry.runtime !== expectedRuntime) + ) { + return [false, latest] as const; + } + const updated = new Map(latest); + updated.set(key, { + ...latestEntry, + idleGeneration: generation, + idleFiber, + lastActivityAtMs, + }); + return [true, updated] as const; }); - return updated; - }); - }); + if (!installed) { + // Our timer lost registration; cancel it so it can never fire + // against a same-id replacement or double-release this entry. + yield* cancelIdleFiber(idleFiber); + } + }), + ); - const scheduleIdleRelease = (providerSessionId: ProviderSessionId) => - withActivityError(providerSessionId, scheduleIdleReleaseInternal(providerSessionId)); + const scheduleIdleRelease = ( + providerSessionId: ProviderSessionId, + expectedRuntime?: ProviderAdapterV2SessionRuntime, + ) => + withActivityError( + providerSessionId, + scheduleIdleReleaseInternal(providerSessionId, expectedRuntime), + ); - const touchActivity = (providerSessionId: ProviderSessionId) => + const touchActivity = ( + providerSessionId: ProviderSessionId, + expectedRuntime?: ProviderAdapterV2SessionRuntime, + ) => withActivityError( providerSessionId, Effect.gen(function* () { const lastActivityAtMs = yield* Clock.currentTimeMillis; - yield* Ref.update(sessions, (current) => { + const owned = yield* Ref.modify(sessions, (current) => { const entry = current.get(sessionKey(providerSessionId)); - if (entry === undefined) { - return current; + if ( + entry === undefined || + (expectedRuntime !== undefined && entry.runtime !== expectedRuntime) + ) { + return [false, current] as const; } const updated = new Map(current); updated.set(sessionKey(providerSessionId), { ...entry, lastActivityAtMs, }); - return updated; + return [true, updated] as const; }); - yield* scheduleIdleReleaseInternal(providerSessionId); + if (owned) { + yield* scheduleIdleReleaseInternal(providerSessionId, expectedRuntime); + } }), ); const attachThread = (input: { readonly providerSessionId: ProviderSessionId; readonly threadId: ThreadId; + readonly expectedRuntime: ProviderAdapterV2SessionRuntime; }) => withActivityError( input.providerSessionId, - Ref.modify(sessions, (current) => { - const entry = current.get(sessionKey(input.providerSessionId)); - if (entry === undefined || entry.attachedThreadIds.has(input.threadId)) { - return [false, current] as const; - } - const updated = new Map(current); - updated.set(sessionKey(input.providerSessionId), { - ...entry, - attachedThreadIds: new Set([...entry.attachedThreadIds, input.threadId]), - }); - return [true, updated] as const; - }), + Ref.modify( + sessions, + ( + current, + ): readonly [ + ( + | { readonly outcome: "closing" | "released" } + | { + readonly outcome: "attached" | "alreadyAttached"; + readonly entry: LiveSessionEntry; + } + ), + Map, + ] => { + const key = sessionKey(input.providerSessionId); + const entry = current.get(key); + if (entry === undefined || entry.runtime !== input.expectedRuntime) { + // Registration into `releasing` and removal from `sessions` + // happen in one Ref.modify, so whichever lands first wins: an + // attach that loses the race sees a closing or released session + // instead of silently skipping the attachment. A mismatched + // runtime is a stale handle on a replaced same-id session. + return [ + { outcome: releasing.has(key) ? ("closing" as const) : ("released" as const) }, + current, + ]; + } + if (entry.attachedThreadIds.has(input.threadId)) { + return [{ outcome: "alreadyAttached" as const, entry }, current]; + } + // The pending-release/startup gates in attachThreadOrReject + // ran before this caller queued on threadAttach — a release + // claiming the thread or a dying startup that owns it may have + // landed meanwhile, so recheck inside the atomic attach + // decision. + if ( + findPendingRelease(input.providerSessionId, input.threadId, entry) !== undefined || + findBlockingStartup(input.providerSessionId, input.threadId) !== undefined + ) { + return [{ outcome: "closing" as const }, current]; + } + const updatedEntry = { + ...entry, + attachedThreadIds: new Set([...entry.attachedThreadIds, input.threadId]), + }; + const updated = new Map(current); + updated.set(key, updatedEntry); + return [{ outcome: "attached" as const, entry: updatedEntry }, updated]; + }, + ), ); const removeThreadAttachment = (input: { readonly providerSessionId: ProviderSessionId; readonly threadId: ThreadId; + readonly expectedRuntime?: ProviderAdapterV2SessionRuntime; }) => Ref.update(sessions, (current) => { const key = sessionKey(input.providerSessionId); const entry = current.get(key); - if (entry === undefined || !entry.attachedThreadIds.has(input.threadId)) { + if ( + entry === undefined || + (input.expectedRuntime !== undefined && entry.runtime !== input.expectedRuntime) || + !entry.attachedThreadIds.has(input.threadId) + ) { return current; } const attachedThreadIds = new Set(entry.attachedThreadIds); @@ -1052,25 +1766,29 @@ export const layerWithOptions = ( readonly providerSessionId: ProviderSessionId; readonly threadId: ThreadId; readonly providerThreadKey: string; + readonly expectedRuntime: ProviderAdapterV2SessionRuntime; }) => Ref.get(sessions).pipe( - Effect.map( - (current) => - current - .get(sessionKey(input.providerSessionId)) - ?.loadedProviderThreadKeyByThread.get(input.threadId) === input.providerThreadKey, - ), + Effect.map((current) => { + const entry = current.get(sessionKey(input.providerSessionId)); + return ( + entry !== undefined && + entry.runtime === input.expectedRuntime && + entry.loadedProviderThreadKeyByThread.get(input.threadId) === input.providerThreadKey + ); + }), ); const markProviderThreadLoaded = (input: { readonly providerSessionId: ProviderSessionId; readonly threadId: ThreadId; readonly providerThreadKey: string; + readonly expectedRuntime: ProviderAdapterV2SessionRuntime; }) => Ref.update(sessions, (current) => { const key = sessionKey(input.providerSessionId); const entry = current.get(key); - if (entry === undefined) { + if (entry === undefined || entry.runtime !== input.expectedRuntime) { return current; } const loadedProviderThreadKeyByThread = new Map(entry.loadedProviderThreadKeyByThread); @@ -1084,72 +1802,271 @@ export const layerWithOptions = ( readonly providerSessionId: ProviderSessionId; readonly threadId: ThreadId; readonly providerInstanceId: ProviderInstanceId; + readonly expectedRuntime: ProviderAdapterV2SessionRuntime; }) => - Effect.suspend(() => { - let preparedForCleanup: PreparedMcpCredential | undefined; - let reservationDropped = false; - const dropReservation = () => { - if (!reservationDropped && preparedForCleanup?.mcpCredentialId !== undefined) { - reservationDropped = true; - dropMcpCredentialReservation(input.threadId, preparedForCleanup.mcpCredentialId); - } - }; - return Effect.gen(function* () { - const attached = yield* attachThread(input); - if (attached) { - const prepared = yield* prepareMcpSession(input.threadId, input.providerInstanceId); - preparedForCleanup = prepared; - if (prepared.mcpCredentialId !== undefined) { - const mcpCredentialId = prepared.mcpCredentialId; - yield* Ref.update(sessions, (current) => { - const key = sessionKey(input.providerSessionId); - const entry = current.get(key); - if (entry === undefined) return current; - const mcpCredentialIdByThread = new Map(entry.mcpCredentialIdByThread); - mcpCredentialIdByThread.set(input.threadId, mcpCredentialId); - const updated = new Map(current); - updated.set(key, { ...entry, mcpCredentialIdByThread }); - return updated; - }); - } - const entry = (yield* Ref.get(sessions)).get(sessionKey(input.providerSessionId)); - if (entry !== undefined) { - yield* withActivityError( - input.providerSessionId, - writeProviderSessionEvents({ - runtime: entry.runtime, - threadIds: [input.threadId], - type: "provider-session.attached", - payload: entry.runtime.providerSession, - }), - ); + threadAttach.withLock( + threadAttachKey(input.providerSessionId, input.expectedRuntime, input.threadId), + Effect.suspend(() => { + // Set synchronously when prepare reserves a credential, so a prepare + // that never returns (failed or interrupted mid-resolve/issue) can + // still have its in-flight reservation unwound. + let pendingMcpCredentialId: string | undefined; + // Whether the post-prepare step introduced the entry's credential + // claim; a claim recorded before this attach (a plain detach + // retains it for the still-live process) is not ours to remove. + let recordOwned = false; + // A retained credential this attach's record overwrote. Once it is + // out of the entry it has no remaining cleanup path, so the unwind + // must cover it if the normal sweep below is interrupted first. + let supersededMcpCredentialId: string | undefined; + let reservationDropped = false; + const dropReservation = () => { + if (!reservationDropped && pendingMcpCredentialId !== undefined) { + reservationDropped = true; + dropMcpCredentialReservation(input.threadId, pendingMcpCredentialId); } - } - }).pipe( - Effect.tapError(() => - removeThreadAttachment(input).pipe( - // Revoke only a credential this attach freshly minted: a REUSED - // credential is by definition held by another live provider - // process, and revoking it thread-wide would break that - // process's MCP client mid-conversation. - Effect.andThen( - Effect.suspend(() => { - dropReservation(); - return preparedForCleanup?.issued === true - ? clearMcpSession(input.threadId, preparedForCleanup.mcpCredentialId) - : Effect.void; - }), - ), + }; + const expectedRuntime = input.expectedRuntime; + // Entry identity rides on the runtime reference: bookkeeping updates + // replace the entry record but never the runtime, while a reopened + // same-id session always carries a new one. + const entryStillOwned = (current: ReadonlyMap) => { + const key = sessionKey(input.providerSessionId); + const entry = current.get(key); + return entry !== undefined && entry.runtime === expectedRuntime + ? entry + : ((releasing.has(key) ? "closing" : "released") as "closing" | "released"); + }; + // The guard in rejectPendingThreadAttachment ran before this + // caller queued on the attach lock and suspended in credential + // preparation — a dying startup that still owns this thread's + // resources may have been marked meanwhile, so every ownership + // recheck below must see it too. + const attachOwnership = (current: ReadonlyMap) => { + const owned = entryStillOwned(current); + return owned !== "closing" && + owned !== "released" && + findBlockingStartup(input.providerSessionId, input.threadId) !== undefined + ? ("closing" as const) + : owned; + }; + const abandonPrepared = ( + outcome: "closing" | "released", + ): Effect.Effect<"closing" | "released", ProviderSessionActivityError> => + Effect.gen(function* () { + // "closing"/"released" return normally, so the onExit unwind + // below never runs for them — the provisional attachment and + // any recorded credential claim must be removed here or a live + // entry keeps a thread whose attach was rejected. + yield* removeThreadAttachment({ ...input, expectedRuntime }); + yield* releasePrepared(); + return outcome; + }); + // Unwind a failed attach's credential bookkeeping while the entry may + // still be owned: drop our reservation and the claim this attempt + // recorded, then revoke only when no other holder claims the + // credential. + const releasePrepared = () => + Effect.suspend(() => { + // When prepare never reserved — the attach was interrupted + // while queued on mcpPrepareLock — the thread's configured + // credential may still need a claim recheck once the + // provisional attachment is gone. + const mcpCredentialId = + pendingMcpCredentialId ?? + McpProviderSession.readMcpProviderSession(input.threadId)?.providerSessionId; + dropReservation(); + const removeRecord = + recordOwned && mcpCredentialId !== undefined + ? Ref.update(sessions, (current) => { + const key = sessionKey(input.providerSessionId); + const entry = current.get(key); + if ( + entry === undefined || + entry.runtime !== expectedRuntime || + entry.mcpCredentialIdByThread.get(input.threadId) !== mcpCredentialId + ) { + return current; + } + const mcpCredentialIdByThread = new Map(entry.mcpCredentialIdByThread); + mcpCredentialIdByThread.delete(input.threadId); + const updated = new Map(current); + updated.set(key, { ...entry, mcpCredentialIdByThread }); + return updated; + }) + : Effect.void; + return removeRecord.pipe( + Effect.andThen( + mcpCredentialId === undefined + ? Effect.void + : releaseUnclaimedMcpCredential(input.threadId, mcpCredentialId), + ), + Effect.andThen( + supersededMcpCredentialId === undefined + ? Effect.void + : releaseUnclaimedMcpCredential(input.threadId, supersededMcpCredentialId), + ), + ); + }); + return Effect.gen(function* () { + const attach = yield* attachThread(input); + if (attach.outcome === "closing" || attach.outcome === "released") { + return attach.outcome; + } + const attached = attach.outcome; + if (attached === "attached") { + const prepared = yield* prepareMcpSession( + input.threadId, + input.providerInstanceId, + (mcpCredentialId) => { + pendingMcpCredentialId = mcpCredentialId; + }, + ); + // Re-check ownership after prepare suspends: a release may have + // claimed the session in between. Recording the credential and + // deciding happen in the same Ref.modify. + const postPrepare = yield* Ref.modify( + sessions, + ( + current, + ): readonly [ + ( + | "closing" + | "released" + | { + readonly entry: LiveSessionEntry; + readonly priorMcpCredentialId: string | undefined; + } + ), + Map, + ] => { + const key = sessionKey(input.providerSessionId); + const owned = attachOwnership(current); + if (owned === "closing" || owned === "released") { + return [owned, current]; + } + const entry = owned; + if (prepared.mcpCredentialId === undefined) { + return [{ entry, priorMcpCredentialId: undefined } as const, current]; + } + const priorMcpCredentialId = entry.mcpCredentialIdByThread.get(input.threadId); + recordOwned = priorMcpCredentialId !== prepared.mcpCredentialId; + supersededMcpCredentialId = + priorMcpCredentialId !== undefined && + priorMcpCredentialId !== prepared.mcpCredentialId + ? priorMcpCredentialId + : undefined; + const mcpCredentialIdByThread = new Map(entry.mcpCredentialIdByThread); + mcpCredentialIdByThread.set(input.threadId, prepared.mcpCredentialId); + const updated = new Map(current); + updated.set(key, { ...entry, mcpCredentialIdByThread }); + return [{ entry, priorMcpCredentialId } as const, updated]; + }, + ); + if (postPrepare === "closing" || postPrepare === "released") { + return yield* abandonPrepared(postPrepare); + } + const superseded = postPrepare.priorMcpCredentialId; + if ( + superseded !== undefined && + superseded !== prepared.mcpCredentialId && + superseded !== pendingMcpCredentialId + ) { + // A retained record from an earlier attach was just + // overwritten: revoke the superseded credential once nothing + // else claims it, or it lives on unclaimed. + yield* releaseUnclaimedMcpCredential(input.threadId, superseded); + } + // The release path reports terminal status under releaseStatus, + // so checking ownership and writing the attach event inside the + // same lock orders this event strictly before any stopped/error + // write: a release that claimed the session first is observed + // here and the late write is skipped instead of resurrecting the + // persisted live status. + const written = yield* releaseStatus + .withLock( + input.providerSessionId, + Ref.get(sessions).pipe( + Effect.flatMap( + ( + current, + ): Effect.Effect< + LiveSessionEntry | "closing" | "released", + ProviderSessionActivityError + > => { + const owned = attachOwnership(current); + if (owned === "closing" || owned === "released") { + return Effect.succeed(owned); + } + return withActivityError( + input.providerSessionId, + writeProviderSessionEvents({ + runtime: owned.runtime, + threadIds: [input.threadId], + type: "provider-session.attached", + payload: owned.runtime.providerSession, + }), + ).pipe(Effect.as(owned)); + }, + ), + ), + ) + .pipe( + // A failed write skips the post-write recheck, so run it on + // the error path too: a release that landed mid-write must + // still reject the caller instead of returning a closed + // adapter. + Effect.catch((writeError) => + Ref.get(sessions).pipe( + Effect.flatMap((current) => { + const owned = attachOwnership(current); + return owned === "closing" || owned === "released" + ? Effect.succeed(owned) + : Effect.fail(writeError); + }), + ), + ), + ); + if (written === "closing" || written === "released") { + return yield* abandonPrepared(written); + } + // Removal from `sessions` does not take the releaseStatus lock, + // so a release may still have claimed the session between the + // in-lock check and now. Re-check before reporting success. + const postWrite = yield* Ref.modify(sessions, (current) => [ + attachOwnership(current), + current, + ]); + if (postWrite === "closing" || postWrite === "released") { + return yield* abandonPrepared(postWrite); + } + } + return attached; + }).pipe( + // Unwind on any non-success exit: typed failures, defects, and + // interruption all must drop the provisional thread attachment — + // while it stays in the entry it claims the credential, which + // keeps a token alive that no live holder recorded — and then + // release the in-flight credential bookkeeping. + Effect.onExit((exit) => + Exit.isSuccess(exit) + ? Effect.void + : removeThreadAttachment({ ...input, expectedRuntime }).pipe( + Effect.andThen(releasePrepared()), + ), ), - ), - // The entry's own record (written above while the thread is - // attached) guards the credential from here on; the reservation - // is only needed until then. Ensuring covers defects/interrupts. - Effect.ensuring(Effect.sync(dropReservation)), - ); - }); + // The entry's own record (written above while the thread is + // attached) guards the credential from here on; the reservation + // is only needed until then. Ensuring covers defects/interrupts. + Effect.ensuring(Effect.sync(dropReservation)), + ); + }), + ); - const markBusy = (providerSessionId: ProviderSessionId) => + const markBusy = ( + providerSessionId: ProviderSessionId, + expectedRuntime?: ProviderAdapterV2SessionRuntime, + ) => withActivityError( providerSessionId, Effect.gen(function* () { @@ -1157,7 +2074,10 @@ export const layerWithOptions = ( const now = yield* Clock.currentTimeMillis; const idleFiber = yield* Ref.modify(sessions, (current) => { const entry = current.get(key); - if (entry === undefined) { + if ( + entry === undefined || + (expectedRuntime !== undefined && entry.runtime !== expectedRuntime) + ) { return [null, current] as const; } const updated = new Map(current); @@ -1174,16 +2094,22 @@ export const layerWithOptions = ( }), ); - const markIdle = (providerSessionId: ProviderSessionId) => + const markIdle = ( + providerSessionId: ProviderSessionId, + expectedRuntime?: ProviderAdapterV2SessionRuntime, + ) => withActivityError( providerSessionId, Effect.gen(function* () { const key = sessionKey(providerSessionId); const now = yield* Clock.currentTimeMillis; - yield* Ref.update(sessions, (current) => { + const owned = yield* Ref.modify(sessions, (current) => { const entry = current.get(key); - if (entry === undefined) { - return current; + if ( + entry === undefined || + (expectedRuntime !== undefined && entry.runtime !== expectedRuntime) + ) { + return [false, current] as const; } const updated = new Map(current); updated.set(key, { @@ -1191,15 +2117,17 @@ export const layerWithOptions = ( busyCount: Math.max(0, entry.busyCount - 1), lastActivityAtMs: now, }); - return updated; + return [true, updated] as const; }); - yield* scheduleIdleReleaseInternal(providerSessionId); + if (owned) { + yield* scheduleIdleReleaseInternal(providerSessionId, expectedRuntime); + } }), ); - const observeActivity = ( + const observeActivity = ( providerSessionId: ProviderSessionId, - activity: Effect.Effect, + activity: Effect.Effect, ) => activity.pipe( Effect.catchCause((cause) => @@ -1210,6 +2138,162 @@ export const layerWithOptions = ( ), ); + // Attachment rejection must propagate to the caller, so it cannot live + // inside observeActivity (which only logs bookkeeping failures). The + // pending-release check runs twice: the gate rejects attachments whose + // thread a pending release recorded, and attachThread atomically rejects + // a session that releaseEntry claimed between the gate and the attach. + const attachThreadOrReject = (input: { + readonly providerSessionId: ProviderSessionId; + readonly threadId: ThreadId; + readonly providerInstanceId: ProviderInstanceId; + readonly driver: ProviderDriverKind; + readonly expectedRuntime: ProviderAdapterV2SessionRuntime; + }) => + Effect.andThen( + rejectPendingThreadAttachment({ + providerSessionId: input.providerSessionId, + threadId: input.threadId, + driver: input.driver, + }), + Effect.gen(function* () { + const attached = yield* ensureThreadAttached({ + providerSessionId: input.providerSessionId, + threadId: input.threadId, + providerInstanceId: input.providerInstanceId, + expectedRuntime: input.expectedRuntime, + }).pipe( + Effect.catch((error) => + Ref.get(sessions).pipe( + Effect.flatMap( + ( + current, + ): Effect.Effect<"closing" | "released", ProviderAdapterProtocolError> => { + const key = sessionKey(input.providerSessionId); + const entry = current.get(key); + if (entry === undefined || entry.runtime !== input.expectedRuntime) { + // The attach failed because the session was claimed + // mid-flight — report the lifecycle outcome. + return Effect.succeed( + releasing.has(key) ? ("closing" as const) : ("released" as const), + ); + } + // The session is still owned but the attach bookkeeping + // failed: proceeding would invoke the adapter for an + // untracked attachment missing its prepared credential. + return Effect.logWarning("orchestration-v2.driver-session.attach-failed", { + providerSessionId: input.providerSessionId, + error, + }).pipe( + Effect.andThen( + Effect.fail( + new ProviderAdapterProtocolError({ + driver: input.driver, + detail: "The provider session could not attach the thread.", + }), + ), + ), + ); + }, + ), + ), + ), + ); + if (attached === "closing" || attached === "released") { + return yield* new ProviderAdapterProtocolError({ + driver: input.driver, + detail: + attached === "closing" + ? "A previous provider session has not finished cleanup." + : "The provider session is no longer running.", + }); + } + }), + ); + + // Entry points that don't attach a thread still must not reach a stale + // adapter: a same-id reopen hands callers a new runtime, and a released + // session's runtime is already closing. + const requireLiveRuntime = (input: { + readonly providerSessionId: ProviderSessionId; + readonly expectedRuntime: ProviderAdapterV2SessionRuntime; + readonly driver: ProviderDriverKind; + }) => + Effect.gen(function* () { + const key = sessionKey(input.providerSessionId); + const entry = (yield* Ref.get(sessions)).get(key); + if (entry !== undefined && entry.runtime === input.expectedRuntime) { + return; + } + return yield* new ProviderAdapterProtocolError({ + driver: input.driver, + detail: releasing.has(key) + ? "A previous provider session has not finished cleanup." + : "The provider session is no longer running.", + }); + }); + + /** + * Runs an adapter operation that can create native resources under an + * admission record the release path drains. Registration and the + * ownership check are one Ref.modify: the claim deletes the sessions + * entry and writes `releasing` atomically, so an operation admitted + * before the claim is always in the drained set and an operation after + * the claim is refused before the adapter runs. + */ + const runResourceCreatingAdapterOp = (input: { + readonly providerSessionId: ProviderSessionId; + readonly expectedRuntime: ProviderAdapterV2SessionRuntime; + readonly driver: ProviderDriverKind; + readonly operation: Effect.Effect; + readonly drainTimeout: Duration.Duration | undefined; + }): Effect.Effect => + Effect.uninterruptibleMask((_restore) => + Effect.gen(function* () { + const key = sessionKey(input.providerSessionId); + const record = { + done: Deferred.makeUnsafe(), + drainTimeout: input.drainTimeout, + }; + const admitted = yield* Ref.modify(sessions, (current) => { + const entry = current.get(key); + if (entry === undefined || entry.runtime !== input.expectedRuntime) { + return [false, current] as const; + } + const ops = inflightAdapterOps.get(key) ?? new Set(); + ops.add(record); + inflightAdapterOps.set(key, ops); + return [true, current] as const; + }); + if (!admitted) { + return yield* new ProviderAdapterProtocolError({ + driver: input.driver, + detail: releasing.has(key) + ? "A previous provider session has not finished cleanup." + : "The provider session is no longer running.", + }); + } + // The operation runs uninterruptibly inside the mask: adapters + // acquire native resources through Effect.tryPromise (Cursor's + // Agent.create/resume), whose interruption detaches the promise + // rather than cancelling it. Signalling `done` at interrupt time + // would let the drain complete while a late result still lands + // with no cleanup owner — an interruptible caller instead waits + // for the acquisition to settle into tracked ownership. + return yield* input.operation.pipe( + Effect.ensuring( + Effect.sync(() => { + const ops = inflightAdapterOps.get(key); + if (ops !== undefined) { + ops.delete(record); + if (ops.size === 0) inflightAdapterOps.delete(key); + } + }).pipe(Effect.andThen(Deferred.done(record.done, Exit.void))), + ), + ); + }), + ); + const makeEventSubscription = ( subscribers: Ref.Ref< ReadonlyMap> @@ -1263,18 +2347,35 @@ export const layerWithOptions = ( subscribeEvents.pipe(Effect.map((subscription) => subscription.events)), ), ensureThread: (input) => - observeActivity( + attachThreadOrReject({ providerSessionId, - ensureThreadAttached({ - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, - }), - ).pipe( - Effect.andThen(runtime.ensureThread(input)), + threadId: input.threadId, + providerInstanceId: runtime.instanceId, + driver: runtime.driver, + expectedRuntime: runtime, + }).pipe( + Effect.andThen( + // The adapter call holds the attach lock so a detach cannot + // strip this thread's bookkeeping mid-call, and it runs under + // an admission record a release drains before closing the + // session scope — a resource the adapter creates (e.g. + // Cursor's late runner.open result) is never installed after + // the finalizer that would have closed it already ran. + threadAttach.withLock( + threadAttachKey(providerSessionId, runtime, input.threadId), + runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.ensureThread(input), + drainTimeout: undefined, + }), + ), + ), Effect.tap((providerThread) => markProviderThreadLoaded({ providerSessionId, + expectedRuntime: runtime, threadId: input.threadId, providerThreadKey: providerThreadLoadKey({ providerThread, @@ -1287,7 +2388,19 @@ export const layerWithOptions = ( resumeThread: (input) => { const threadId = input.threadId ?? input.providerThread.appThreadId; if (threadId === null || threadId === undefined) { - return runtime.resumeThread(input); + // No attach key exists without a thread, so the ownership check + // and the adapter call share the session-open lock a release + // drains before closing the scope. + return sessionOpen.withLock( + providerSessionId, + runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.resumeThread(input), + drainTimeout: undefined, + }), + ); } const providerThreadKey = providerThreadLoadKey({ providerThread: input.providerThread, @@ -1296,23 +2409,39 @@ export const layerWithOptions = ( : { modelSelection: input.modelSelection }), ...(input.runtimePolicy === undefined ? {} : { runtimePolicy: input.runtimePolicy }), }); - return observeActivity( + return attachThreadOrReject({ providerSessionId, - ensureThreadAttached({ - providerSessionId, - threadId, - providerInstanceId: runtime.instanceId, - }), - ).pipe( + threadId, + providerInstanceId: runtime.instanceId, + driver: runtime.driver, + expectedRuntime: runtime, + }).pipe( Effect.andThen( - isProviderThreadLoaded({ providerSessionId, threadId, providerThreadKey }), + isProviderThreadLoaded({ + providerSessionId, + threadId, + providerThreadKey, + expectedRuntime: runtime, + }), ), Effect.flatMap((loaded) => - loaded ? Effect.succeed(input.providerThread) : runtime.resumeThread(input), + loaded + ? Effect.succeed(input.providerThread) + : threadAttach.withLock( + threadAttachKey(providerSessionId, runtime, threadId), + runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.resumeThread(input), + drainTimeout: undefined, + }), + ), ), Effect.tap((providerThread) => markProviderThreadLoaded({ providerSessionId, + expectedRuntime: runtime, threadId, providerThreadKey: providerThreadLoadKey({ providerThread, @@ -1328,18 +2457,33 @@ export const layerWithOptions = ( ); }, forkThread: (input) => - observeActivity( + attachThreadOrReject({ providerSessionId, - ensureThreadAttached({ - providerSessionId, - threadId: input.targetThreadId, - providerInstanceId: runtime.instanceId, - }), - ).pipe( - Effect.andThen(runtime.forkThread(input)), + threadId: input.targetThreadId, + providerInstanceId: runtime.instanceId, + driver: runtime.driver, + expectedRuntime: runtime, + }).pipe( + // Fork acquires provider-side state for the target thread + // (ACP session/fork may restart the runtime) — the same + // acquire-then-return shape as ensure/resume, so it drains + // under the same admission record. + Effect.andThen( + threadAttach.withLock( + threadAttachKey(providerSessionId, runtime, input.targetThreadId), + runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.forkThread(input), + drainTimeout: undefined, + }), + ), + ), Effect.tap((providerThread) => markProviderThreadLoaded({ providerSessionId, + expectedRuntime: runtime, threadId: input.targetThreadId, providerThreadKey: providerThreadLoadKey({ providerThread, @@ -1354,34 +2498,236 @@ export const layerWithOptions = ( ), ), startTurn: (input) => - observeActivity( - providerSessionId, - ensureThreadAttached({ + Effect.andThen( + attachThreadOrReject({ providerSessionId, threadId: input.threadId, providerInstanceId: runtime.instanceId, + driver: runtime.driver, + expectedRuntime: runtime, }), - ).pipe( - Effect.andThen(observeActivity(providerSessionId, markBusy(providerSessionId))), - Effect.andThen(runtime.startTurn(input)), - Effect.catch((error) => - observeActivity(providerSessionId, markIdle(providerSessionId)).pipe( - Effect.andThen(Effect.fail(error)), + observeActivity(providerSessionId, markBusy(providerSessionId, runtime)).pipe( + // markBusy can suspend waiting on the idle fiber; a close may + // have claimed the session meanwhile — re-check before + // invoking the adapter. + Effect.andThen( + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }), + ), + // The adapter call returns once the turn is started — most + // adapters fork the turn's lifetime into the session scope — + // so the admission record spans only the acquisition window + // (Cursor's openAgent/runner.open, ACP session activate+prompt + // submit). Only in-band turn calls get the bounded drain — + // abandoning an acquire-then-return call could strand a + // resource the scope close already finished checking for. + Effect.andThen( + threadAttach.withLock( + threadAttachKey(providerSessionId, runtime, input.threadId), + runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.startTurn(input), + drainTimeout: INBAND_TURN_CALL_DRIVERS.has(runtime.driver) + ? Duration.millis(ADAPTER_OP_DRAIN_TIMEOUT_MS) + : undefined, + }), + ), + ), + Effect.catch((error) => + observeActivity(providerSessionId, markIdle(providerSessionId, runtime)).pipe( + Effect.andThen(Effect.fail(error)), + ), ), ), ), steerTurn: (input) => - observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }).pipe( + Effect.andThen( + observeActivity(providerSessionId, touchActivity(providerSessionId, runtime)), + ), + // touchActivity can suspend waiting on the idle fiber; a close + // may have claimed the session meanwhile — re-check before + // invoking the adapter. + Effect.andThen( + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }), + ), Effect.andThen(runtime.steerTurn(input)), ), interruptTurn: (input) => - observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }).pipe( + Effect.andThen( + observeActivity(providerSessionId, touchActivity(providerSessionId, runtime)), + ), + Effect.andThen( + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }), + ), Effect.andThen(runtime.interruptTurn(input)), ), respondToRuntimeRequest: (input) => - observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }).pipe( + Effect.andThen( + observeActivity(providerSessionId, touchActivity(providerSessionId, runtime)), + ), + Effect.andThen( + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }), + ), Effect.andThen(runtime.respondToRuntimeRequest(input)), ), + // Every exposed adapter operation must reject a stale handle: a + // released session's runtime is already closing, and a same-id + // reopen hands callers a new runtime — the spread would otherwise + // carry these through unguarded. + ...(runtime.compactThread === undefined + ? {} + : { + compactThread: (input: Parameters>[0]) => + Effect.andThen( + attachThreadOrReject({ + providerSessionId, + threadId: input.threadId, + providerInstanceId: runtime.instanceId, + driver: runtime.driver, + expectedRuntime: runtime, + }), + observeActivity(providerSessionId, markBusy(providerSessionId, runtime)).pipe( + Effect.andThen( + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }), + ), + // Compaction delegates to the same acquire-then-start + // path as startTurn, so it shares the admission record + // and the same driver-keyed drain policy. + Effect.andThen( + threadAttach.withLock( + threadAttachKey(providerSessionId, runtime, input.threadId), + runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.compactThread!(input), + drainTimeout: INBAND_TURN_CALL_DRIVERS.has(runtime.driver) + ? Duration.millis(ADAPTER_OP_DRAIN_TIMEOUT_MS) + : undefined, + }), + ), + ), + Effect.catch((error) => + observeActivity( + providerSessionId, + markIdle(providerSessionId, runtime), + ).pipe(Effect.andThen(Effect.fail(error))), + ), + ), + ), + }), + readThreadSnapshot: (input) => + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }).pipe( + Effect.andThen( + observeActivity(providerSessionId, touchActivity(providerSessionId, runtime)), + ), + Effect.andThen( + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }), + ), + Effect.andThen( + runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.readThreadSnapshot(input), + drainTimeout: undefined, + }), + ), + ), + rollbackThread: (input) => + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }).pipe( + Effect.andThen( + observeActivity(providerSessionId, touchActivity(providerSessionId, runtime)), + ), + Effect.andThen( + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }), + ), + Effect.andThen( + runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.rollbackThread(input), + drainTimeout: undefined, + }), + ), + ), + ...(runtime.uploadFeedback === undefined + ? {} + : { + uploadFeedback: ( + input: Parameters>[0], + ) => + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }).pipe( + Effect.andThen( + observeActivity(providerSessionId, touchActivity(providerSessionId, runtime)), + ), + Effect.andThen( + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }), + ), + Effect.andThen(runtime.uploadFeedback!(input)), + ), + }), }; }; @@ -1389,27 +2735,37 @@ export const layerWithOptions = ( entry: LiveSessionEntry, event: Extract, ) => - Effect.gen(function* () { - const current = (yield* Ref.get(sessions)).get( - sessionKey(entry.runtime.providerSessionId), - ); - if (current?.runtime !== entry.runtime) { - return; - } - yield* writeProviderSessionEvents({ - runtime: entry.runtime, - threadIds: current.attachedThreadIds, - type: "provider-session.updated", - payload: event.providerSession, - }); - }).pipe( - Effect.catchCause((cause) => - Effect.logWarning("orchestration-v2.driver-session.status-persist-failed", { - providerSessionId: entry.runtime.providerSessionId, - cause, + // The release path reports terminal status under releaseStatus, so + // checking ownership and writing inside the same lock orders this + // update strictly before any stopped/error write: a release that + // claimed the session first is observed here and the late write is + // skipped instead of resurrecting the persisted live status. + releaseStatus + .withLock( + entry.runtime.providerSessionId, + Effect.gen(function* () { + const current = (yield* Ref.get(sessions)).get( + sessionKey(entry.runtime.providerSessionId), + ); + if (current?.runtime !== entry.runtime) { + return; + } + yield* writeProviderSessionEvents({ + runtime: entry.runtime, + threadIds: current.attachedThreadIds, + type: "provider-session.updated", + payload: event.providerSession, + }); }), - ), - ); + ) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.driver-session.status-persist-failed", { + providerSessionId: entry.runtime.providerSessionId, + cause, + }), + ), + ); const startEventPump = (entry: LiveSessionEntry) => { let stoppedByProvider = false; @@ -1424,8 +2780,8 @@ export const layerWithOptions = ( return observeActivity( entry.runtime.providerSessionId, event.type === "turn.terminal" - ? markIdle(entry.runtime.providerSessionId) - : touchActivity(entry.runtime.providerSessionId), + ? markIdle(entry.runtime.providerSessionId, entry.runtime) + : touchActivity(entry.runtime.providerSessionId, entry.runtime), ).pipe( Effect.andThen( event.type === "provider_session.updated" @@ -1484,6 +2840,7 @@ export const layerWithOptions = ( providerSessionId: entry.runtime.providerSessionId, reason: "manual_shutdown", gracefulSubscribers: true, + expectedRuntime: entry.runtime, }).pipe(Effect.ignore); return; } @@ -1505,6 +2862,7 @@ export const layerWithOptions = ( providerSessionId: entry.runtime.providerSessionId, reason: "runtime_error", detail: Cause.pretty(cause), + expectedRuntime: entry.runtime, }).pipe(Effect.ignore); }), ), @@ -1513,202 +2871,576 @@ export const layerWithOptions = ( }; const shutdown = Effect.gen(function* () { + // Fence first, in one synchronous pass: a startup that already + // registered is found by the session scan below; one still opening + // was created before this flag and is marked here so its + // registration fence loses instead of slipping in while the + // releases run. The flag is permanent — the manager is going away. + shutdownInitiated = true; + const starting = Array.from(opening.values()); + for (const record of starting) { + if (record.failed === undefined) { + record.closeRequested = true; + } + } const activeSessions = [...(yield* Ref.get(sessions)).values()]; - yield* Effect.forEach( - activeSessions, - (entry) => - releaseEntry({ - providerSessionId: entry.runtime.providerSessionId, - reason: "server_shutdown", - }).pipe( - Effect.catchCause((cause) => - Effect.logWarning("orchestration-v2.driver-session.shutdown-release-failed", { + // Every wait below is independently bounded, and the phases target + // disjoint resources — run all of them concurrently so a few wedged + // provider processes cannot multiply the teardown wait. Ownership is + // unchanged: shutdown still joins each tracked cleanup before it + // returns. + yield* Effect.all( + [ + Effect.forEach( + activeSessions, + (entry) => + releaseEntry({ providerSessionId: entry.runtime.providerSessionId, - cause, - }), - ), + reason: "server_shutdown", + expectedRuntime: entry.runtime, + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.driver-session.shutdown-release-failed", { + providerSessionId: entry.runtime.providerSessionId, + cause, + }), + ), + ), + { concurrency: "unbounded", discard: true }, + ), + // Releases already claimed before this shutdown (a racing close, + // detach, or idle sweep) are invisible to the sessions scan; join + // each pending record so teardown does not return while cleanup + // still runs against the manager's dependencies. + Effect.forEach( + [...releasing.values()], + (release) => + releaseEntry({ + providerSessionId: release.entry.runtime.providerSessionId, + reason: "server_shutdown", + joinOnly: true, + expectedRuntime: release.entry.runtime, + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.driver-session.shutdown-release-failed", { + providerSessionId: release.entry.runtime.providerSessionId, + cause, + }), + ), + ), + { concurrency: "unbounded", discard: true }, + ), + // Terminal-detach credential sweeps outlive their callers in + // pendingRevocations; join each tracked tail (bounded like every + // cleanup wait) so teardown does not return while a revocation is + // still running. In-flight detaches are drained first: one still + // inside its lock can register a tail after a single snapshot. + joinTrackedRevocations( + () => true, + (threadId, done) => + Deferred.await(done).pipe( + Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), + Effect.flatMap((settled) => + Option.isNone(settled) + ? Effect.logWarning( + "orchestration-v2.driver-session.shutdown-revocation-pending", + { threadId }, + ) + : Effect.void, + ), + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.driver-session.shutdown-release-failed", { + threadId, + cause, + }), + ), + ), + (marker) => + Deferred.await(marker.detached).pipe( + Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), + Effect.flatMap((settled) => + Option.isNone(settled) + ? Effect.logWarning( + "orchestration-v2.driver-session.shutdown-detach-pending", + { + providerSessionId: marker.providerSessionId, + threadId: marker.threadId, + }, + ) + : Effect.void, + ), + ), + () => true, + ).pipe(Effect.ignore), + // In-flight startups own provider resources without a live entry; + // they were marked above — join their bounded unwind during + // shutdown too. + Effect.forEach( + starting, + (record) => + releaseEntry({ + providerSessionId: record.providerSessionId, + reason: "server_shutdown", + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.driver-session.shutdown-release-failed", { + providerSessionId: record.providerSessionId, + cause, + }), + ), + ), + { concurrency: "unbounded", discard: true }, ), - { discard: true }, + ], + { concurrency: "unbounded", discard: true }, ); }); yield* Effect.addFinalizer(() => shutdown); return ProviderSessionManagerV2.of({ shutdown, - open: (input) => - sessionOpen.withLock( + open: (input) => { + const open = sessionOpen.withLock( input.providerSessionId, Effect.gen(function* () { - const cwd = input.runtimePolicy.cwd; - if (cwd !== null) { - const workspaceIsDirectory = yield* fileSystem.stat(cwd).pipe( - Effect.map((stat) => stat.type === "Directory"), - Effect.catch((error) => Effect.succeed(error.reason._tag !== "NotFound")), - ); - if (!workspaceIsDirectory) { - return yield* new ProviderWorkspaceMissingError({ - threadId: input.threadId, - cwd, - }); - } - } const key = sessionKey(input.providerSessionId); - const existing = (yield* Ref.get(sessions)).get(key); - if (existing !== undefined) { - if ( - !existing.attachedThreadIds.has(input.threadId) && - !existing.supportsMultipleProviderThreads - ) { - return yield* new ProviderSessionOpenError({ - instanceId: input.modelSelection.instanceId, - providerSessionId: input.providerSessionId, - cause: `Provider ${existing.runtime.driver} does not support attaching multiple app threads to one session.`, - }); - } - yield* ensureThreadAttached({ - providerSessionId: input.providerSessionId, - threadId: input.threadId, - providerInstanceId: existing.runtime.instanceId, - }); - yield* touchActivity(input.providerSessionId); - return existing.exposedRuntime; - } - - const adapter = yield* registry.get(input.modelSelection.instanceId).pipe( - Effect.mapError( - (cause) => - new ProviderSessionOpenError({ - instanceId: input.modelSelection.instanceId, - providerSessionId: input.providerSessionId, - cause, - }), - ), - ); - const prepared = yield* prepareMcpSession( - input.threadId, - input.modelSelection.instanceId, - ); - const mcpCredentialId = prepared.mcpCredentialId; // The reservation from prepare protects the credential (which // eager adapters bake into the provider process during // openSession) from racing releases until this session's entry - // is recorded below. Dropped exactly once on every path. - let reservationDropped = mcpCredentialId === undefined; - const dropReservation = Effect.sync(() => { - if (!reservationDropped && mcpCredentialId !== undefined) { + // is recorded below. Dropped exactly once on every path — the + // pending id is captured as soon as prepare reserves it, and the + // single onExit below covers every non-success exit of the + // whole prepare→open→register sequence so no interleave can + // strand the reservation. + let pendingMcpCredentialId: string | undefined; + let reservationDropped = false; + const dropReservationNow = () => { + if (!reservationDropped && pendingMcpCredentialId !== undefined) { reservationDropped = true; - dropMcpCredentialReservation(input.threadId, mcpCredentialId); + dropMcpCredentialReservation(input.threadId, pendingMcpCredentialId); } - }); + }; + const dropReservation = Effect.sync(dropReservationNow); const sessionScope = yield* Scope.make(); - const runtime = yield* adapter - .openSession({ - threadId: input.threadId, - providerSessionId: input.providerSessionId, - modelSelection: input.modelSelection, - runtimePolicy: input.runtimePolicy, - ...(input.resumeFromSession === undefined - ? {} - : { resumeFromSession: input.resumeFromSession }), - ...(input.initialNativeThreadId === undefined - ? {} - : { initialNativeThreadId: input.initialNativeThreadId }), - ...(input.initialProviderItemIdentityVersion === undefined - ? {} - : { - initialProviderItemIdentityVersion: - input.initialProviderItemIdentityVersion, - }), - }) - .pipe( - Effect.provideService(Scope.Scope, sessionScope), - Effect.tapError(() => - Scope.close(sessionScope, Exit.void).pipe( - Effect.ignore, - Effect.andThen(dropReservation), - // Revoke only a credential this open freshly minted: a - // reused credential is held by another live provider - // process and must survive this open's failure. - Effect.andThen( - prepared.issued - ? clearMcpSession(input.threadId, mcpCredentialId) - : Effect.void, - ), - ), - ), - Effect.onInterrupt(() => dropReservation), - Effect.mapError( - (cause) => - new ProviderSessionOpenError({ - instanceId: input.modelSelection.instanceId, + // Track the startup before any suspension — the workspace stat + // and adapter lookup included: a close or detach racing the open + // marks closeRequested and joins this record's unwind instead of + // reporting success while provider resources are being spawned. + // sessionOpen serializes same-id opens, so a record found here + // later can only be a failed leftover. The flag check and the + // insert share one synchronous step so a startup can never + // begin tracking after its instance's close already scanned. + let entryRegistered = false; + let registeredRuntime: ProviderAdapterV2SessionRuntime | undefined; + // Registration and the unwind handler are one uninterruptible + // handoff: an interrupt landing after `opening.set` but before + // the onExit is installed would strand the record — a racing + // close would mark it and join a `settled` that never + // completes. The open itself stays interruptible under + // `restore`; every exit from it still runs the unwind. + const exposedRuntime = yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const openingRecord = yield* Effect.sync( + (): OpeningSessionRecord | "blocked" | undefined => { + if ( + shutdownInitiated || + closingInstanceCounts.has(input.modelSelection.instanceId) + ) { + return undefined; + } + // sessionOpen serializes same-id opens, so a record + // found here can only be a failed or still-unwinding + // leftover — it must keep blocking replacements. + if (opening.has(key)) { + return "blocked"; + } + const record: OpeningSessionRecord = { providerSessionId: input.providerSessionId, - cause, - }), - ), - ); - const eventSubscribers = yield* Ref.make< - ReadonlyMap> - >(new Map()); - const exposedRuntime = decorateRuntime(runtime, eventSubscribers); - const now = yield* Clock.currentTimeMillis; - const entry: LiveSessionEntry = { - attachedThreadIds: new Set([input.threadId]), - loadedProviderThreadKeyByThread: new Map(), - mcpCredentialIdByThread: - mcpCredentialId === undefined - ? new Map() - : new Map([[input.threadId, mcpCredentialId]]), - supportsMultipleProviderThreads: - runtime.providerSession.capabilities.sessions - .supportsMultipleProviderThreadsPerSession, - runtime, - exposedRuntime, - eventSubscribers, - requestEventPermit: yield* Semaphore.make(1), - scope: sessionScope, - idleGeneration: 0, - busyCount: 0, - lastActivityAtMs: now, - idleFiber: null, - pinnedSinceMs: null, - }; - yield* Ref.update(sessions, (current) => { - const updated = new Map(current); - updated.set(key, entry); - return updated; - }); - // The entry now guards the credential via its recorded id, so - // the pre-open reservation can be dropped. - yield* dropReservation; - yield* withActivityError( - input.providerSessionId, - writeProviderSessionEvents({ - runtime, - threadIds: [input.threadId], - type: "provider-session.attached", - payload: runtime.providerSession, + threadId: input.threadId, + instanceId: input.modelSelection.instanceId, + closeRequested: false, + failed: undefined, + settled: Deferred.makeUnsafe(), + }; + opening.set(key, record); + return record; + }, + ); + if (openingRecord === "blocked") { + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: "A previous provider session has not finished cleanup.", + }); + } + if (openingRecord === undefined) { + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: "The provider session was closed before it finished opening.", + }); + } + return yield* restore( + Effect.gen(function* () { + const cwd = input.runtimePolicy.cwd; + if (cwd !== null) { + const workspaceIsDirectory = yield* fileSystem.stat(cwd).pipe( + Effect.map((stat) => stat.type === "Directory"), + Effect.catch((error) => Effect.succeed(error.reason._tag !== "NotFound")), + ); + if (!workspaceIsDirectory) { + return yield* new ProviderWorkspaceMissingError({ + threadId: input.threadId, + cwd, + }); + } + } + const existing = (yield* Ref.get(sessions)).get(key); + const pending = findPendingRelease( + input.providerSessionId, + input.threadId, + existing, + ); + if (pending !== undefined) { + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: `Provider session ${pending.entry.runtime.providerSessionId} has not finished cleanup.`, + }); + } + // Block the replacement like a pending release: a failed or + // still-unwinding startup for the session id or the same thread + // may still own provider resources. + const blockingStartup = findBlockingStartup( + input.providerSessionId, + input.threadId, + ); + if (blockingStartup !== undefined) { + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: `Provider session ${blockingStartup.providerSessionId} has not finished cleanup.`, + }); + } + if (existing !== undefined) { + if ( + !existing.attachedThreadIds.has(input.threadId) && + !existing.supportsMultipleProviderThreads + ) { + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: `Provider ${existing.runtime.driver} does not support attaching multiple app threads to one session.`, + }); + } + const attached = yield* ensureThreadAttached({ + providerSessionId: input.providerSessionId, + threadId: input.threadId, + providerInstanceId: existing.runtime.instanceId, + expectedRuntime: existing.runtime, + }); + if (attached === "closing" || attached === "released") { + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: `Provider session ${input.providerSessionId} ${ + attached === "closing" + ? "has not finished cleanup." + : "is no longer running." + }`, + }); + } + yield* touchActivity(input.providerSessionId, existing.runtime); + return existing.exposedRuntime; + } + + const adapter = yield* registry.get(input.modelSelection.instanceId).pipe( + Effect.mapError( + (cause) => + new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause, + }), + ), + ); + const prepared = yield* prepareMcpSession( + input.threadId, + input.modelSelection.instanceId, + (mcpCredentialId) => { + pendingMcpCredentialId = mcpCredentialId; + }, + ); + const mcpCredentialId = prepared.mcpCredentialId; + // A close marked this startup while credentials were + // resolving: skip the spawn entirely and unwind. A mark + // landing after this read is still caught by the + // registration fence below. + if (openingRecord.closeRequested) { + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: "The provider session was closed before it finished opening.", + }); + } + const runtime = yield* adapter + .openSession({ + threadId: input.threadId, + providerSessionId: input.providerSessionId, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + ...(input.resumeFromSession === undefined + ? {} + : { resumeFromSession: input.resumeFromSession }), + ...(input.initialNativeThreadId === undefined + ? {} + : { initialNativeThreadId: input.initialNativeThreadId }), + ...(input.initialProviderItemIdentityVersion === undefined + ? {} + : { + initialProviderItemIdentityVersion: + input.initialProviderItemIdentityVersion, + }), + }) + .pipe( + Effect.provideService(Scope.Scope, sessionScope), + Effect.mapError( + (cause) => + new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause, + }), + ), + ); + const eventSubscribers = yield* Ref.make< + ReadonlyMap> + >(new Map()); + const exposedRuntime = decorateRuntime(runtime, eventSubscribers); + const now = yield* Clock.currentTimeMillis; + const entry: LiveSessionEntry = { + attachedThreadIds: new Set([input.threadId]), + loadedProviderThreadKeyByThread: new Map(), + mcpCredentialIdByThread: + mcpCredentialId === undefined + ? new Map() + : new Map([[input.threadId, mcpCredentialId]]), + supportsMultipleProviderThreads: + runtime.providerSession.capabilities.sessions + .supportsMultipleProviderThreadsPerSession, + runtime, + exposedRuntime, + eventSubscribers, + requestEventPermit: yield* Semaphore.make(1), + scope: sessionScope, + idleGeneration: 0, + busyCount: 0, + lastActivityAtMs: now, + idleFiber: null, + pinnedSinceMs: null, + }; + // Fenced in the same atomic step: a close that marked this + // startup (or a closeInstance/shutdown that fenced its + // instance) before the adapter returned must not see a live + // entry appear behind its back — the unwind below closes + // the scope and completes `settled` instead. + yield* Ref.modify( + sessions, + (current): readonly [boolean, Map] => { + if ( + openingRecord.closeRequested || + shutdownInitiated || + closingInstanceCounts.has(openingRecord.instanceId) + ) { + return [false, current] as const; + } + const updated = new Map(current); + updated.set(key, entry); + // Set inside the atomic update: from here the release path + // owns cleanup and the entry's credential record guards the + // credential. + entryRegistered = true; + registeredRuntime = runtime; + return [true, updated] as const; + }, + ); + if (entryRegistered) { + opening.delete(key); + yield* Deferred.done(openingRecord.settled, Exit.void); + } else { + // A close won the race: skip the spawn bookkeeping entirely + // and fail so the unwind closes the provider scope and + // settles the startup record the close is joining. + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: "The provider session was closed before it finished opening.", + }); + } + // The pre-open reservation can be dropped now; the onExit + // below also drops it so an interruption landing before this + // line cannot strand it. + yield* dropReservation; + // Ownership check, attach-event write, and recheck all run under + // releaseStatus — the lock the release path reports terminal + // status through. A close can claim the session while this write + // suspends (removal does not take the lock), so the recheck keeps + // open from returning a runtime that is already being released, + // and the in-lock check keeps a late attach event from + // overwriting the persisted stopped/error status. + const persisted = yield* releaseStatus + .withLock( + input.providerSessionId, + Effect.gen(function* () { + const owned = (yield* Ref.get(sessions)).get(key)?.runtime === runtime; + if (!owned) { + return false; + } + yield* withActivityError( + input.providerSessionId, + writeProviderSessionEvents({ + runtime, + threadIds: [input.threadId], + type: "provider-session.attached", + payload: runtime.providerSession, + }), + ); + return (yield* Ref.get(sessions)).get(key)?.runtime === runtime; + }), + ) + .pipe( + Effect.tapError(() => + releaseEntry({ + providerSessionId: input.providerSessionId, + reason: "runtime_error", + detail: "Failed to persist the provider-session attachment.", + alreadyLocked: true, + expectedRuntime: runtime, + }).pipe(Effect.ignore), + ), + ); + if (!persisted) { + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: "The provider session was closed before it finished opening.", + }); + } + yield* startEventPump(entry); + yield* scheduleIdleRelease(input.providerSessionId, runtime); + return exposedRuntime; + }), + ).pipe( + Effect.onExit((exit) => + Exit.isSuccess(exit) + ? entryRegistered + ? Effect.void + : // Early returns — a reused live session or a + // rejected attach — never hand the record to the + // release path; settle it here or a racing close + // joins a deferred that never completes. + Effect.uninterruptible( + Effect.gen(function* () { + if (opening.get(key) === openingRecord) { + opening.delete(key); + } + yield* Deferred.done(openingRecord.settled, Exit.void); + }), + ) + : Effect.suspend(() => { + if (entryRegistered) { + // A registered entry guards the credential by its + // own record, so the reservation can go before the + // half-initialized entry is handed to the release + // path. Forked: this fiber still holds sessionOpen, + // which the release's waitForOpens needs. + dropReservationNow(); + return releaseEntry({ + providerSessionId: input.providerSessionId, + reason: "runtime_error", + detail: "Provider session open was interrupted.", + ...(registeredRuntime === undefined + ? {} + : { expectedRuntime: registeredRuntime }), + }).pipe(Effect.forkDetach({ startImmediately: true }), Effect.asVoid); + } + // A defecting finalizer must not keep the rest of the + // unwind from running — but its failure must stay + // visible: closing over a provider process that may + // still be alive keeps the opening record as a failed + // cleanup so replacement opens stay blocked until + // restart, and a racing close's join sees the error. + return Effect.gen(function* () { + // The record stays blocking while the unwind runs: + // attachments and replacements must wait for the + // scope close rather than racing a provider process + // that is still shutting down. + openingRecord.closeRequested = true; + const closeExit = yield* Scope.close(sessionScope, Exit.void).pipe( + Effect.exit, + ); + // The reservation outlives the scope close: the + // provider process can still present the credential + // while it is shutting down, so a peer's terminal + // detach must keep seeing this claim until the close + // settles. + dropReservationNow(); + const sweepExit = + pendingMcpCredentialId === undefined + ? Exit.void + : yield* releaseUnclaimedMcpCredential( + input.threadId, + pendingMcpCredentialId, + ).pipe(Effect.exit); + const failed = Exit.isFailure(closeExit) + ? closeExit + : Exit.isFailure(sweepExit) + ? sweepExit + : undefined; + if (failed === undefined) { + opening.delete(key); + yield* Deferred.done(openingRecord.settled, Exit.void); + return; + } + const error = new ProviderSessionReleaseError({ + providerSessionId: input.providerSessionId, + reason: "runtime_error", + cause: failed.cause, + }); + openingRecord.failed = error; + yield* Deferred.done(openingRecord.settled, Exit.fail(error)); + }); + }), + ), + ); }), - ).pipe( - Effect.tapError(() => - releaseEntry({ - providerSessionId: input.providerSessionId, - reason: "runtime_error", - detail: "Failed to persist the provider-session attachment.", - }).pipe(Effect.ignore), - ), ); - yield* startEventPump(entry); - yield* scheduleIdleRelease(input.providerSessionId); return exposedRuntime; }), - ), + ); + return Effect.gen(function* () { + // Reusing an attached live session starts no new provider work. + const existing = (yield* Ref.get(sessions)).get(sessionKey(input.providerSessionId)); + if ( + findPendingRelease(input.providerSessionId, input.threadId, existing) !== undefined || + findBlockingStartup(input.providerSessionId, input.threadId) !== undefined + ) { + return yield* new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause: "A previous provider session has not finished cleanup.", + }); + } + return yield* threadLifecycle.withLock(input.threadId, open); + }); + }, get: (providerSessionId) => Effect.gen(function* () { const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId)); if (entry === undefined) { return Option.none(); } - yield* touchActivity(providerSessionId); + yield* touchActivity(providerSessionId, entry.runtime); return Option.some(entry.exposedRuntime); }).pipe( Effect.mapError( @@ -1731,24 +3463,144 @@ export const layerWithOptions = ( ), closeInstance: (instanceId) => Effect.gen(function* () { - const active = [...(yield* Ref.get(sessions)).values()].filter( - (entry) => entry.runtime.instanceId === instanceId, + // Fence first, in one synchronous pass (same reasoning as + // shutdown): startups created after this point are rejected at + // the record gate; the ones already tracked are marked so their + // registration fence loses while the releases below run. + closingInstanceCounts.set(instanceId, (closingInstanceCounts.get(instanceId) ?? 0) + 1); + const starting = Array.from(opening.values()).filter( + (record) => record.instanceId === instanceId, ); + for (const record of starting) { + if (record.failed === undefined) { + record.closeRequested = true; + } + } + const active = [ + ...(yield* Ref.get(sessions)).values(), + ...Array.from(releasing.values(), (release) => release.entry), + ].filter((entry) => entry.runtime.instanceId === instanceId); const outcomes = yield* Effect.forEach( - active, - (entry) => + active.map((entry) => ({ + providerSessionId: entry.runtime.providerSessionId, + expectedRuntime: entry.runtime, + })), + (target) => releaseEntry({ - providerSessionId: entry.runtime.providerSessionId, + providerSessionId: target.providerSessionId, reason: "manual_shutdown", detail: `Provider instance ${instanceId} logged out.`, + expectedRuntime: target.expectedRuntime, }).pipe(Effect.exit), { concurrency: "unbounded" }, ); - const failure = outcomes.find(Exit.isFailure); + // In-flight startups for this instance own provider resources + // without a live entry; they were marked above — join their + // bounded unwind as part of closing the instance. Join the + // captured record's settled deferred directly: re-resolving by + // session id could claim a different instance's same-id + // replacement that registered after this record was removed. + const startupOutcomes = yield* Effect.forEach( + starting, + (record) => + Deferred.await(record.settled).pipe( + Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), + Effect.flatMap((settled) => + Option.isNone(settled) + ? Effect.fail( + new ProviderSessionReleaseError({ + providerSessionId: record.providerSessionId, + reason: "manual_shutdown", + cause: + "Provider session startup cleanup did not finish within 30 seconds. " + + "Replacement sessions are blocked until cleanup completes.", + }), + ) + : Effect.void, + ), + Effect.exit, + ), + { concurrency: "unbounded" }, + ); + // Tracked credential sweeps outlive their detach callers; the + // ones owned by this instance's sessions (or unattributed, where + // the owning entry was already gone) still run under teardown. + // Only unfinished records count, and each matching record is + // joined on its own completion — not the tail's: a chained + // successor may still be sweeping for another instance after + // this instance's own sweep finishes. A completed predecessor's + // caller is done and must not be kept waiting on a successor it + // does not own. In-flight detaches are drained first: one still + // inside its lock can register a tail after a single snapshot. + const revocationOutcomes = yield* joinTrackedRevocations( + (record) => record.instanceId === undefined || record.instanceId === instanceId, + (threadId, done) => + Deferred.await(done).pipe( + Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), + Effect.flatMap((settled) => + Option.isNone(settled) + ? Effect.fail( + new ProviderSessionReleaseError({ + providerSessionId: ProviderSessionId.make( + `provider-session:thread:${threadId}`, + ), + reason: "manual_shutdown", + cause: + "MCP credential revocation did not finish within 30 seconds and is still running.", + }), + ) + : Effect.void, + ), + ), + (marker) => + Deferred.await(marker.detached).pipe( + Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), + Effect.flatMap((settled) => + Option.isNone(settled) + ? Effect.fail( + new ProviderSessionReleaseError({ + providerSessionId: marker.providerSessionId, + reason: "manual_shutdown", + cause: + "An in-flight detach did not finish within 30 seconds and is still running.", + }), + ) + : Effect.void, + ), + ), + // Drain only detaches whose sweep could match `owned` above: a + // detach on a session provably owned by another instance cannot + // register a tail this close must join, and waiting on one would + // only stretch logout behind an unrelated stuck detach. The + // marker's captured owner wins over a map re-resolution — the + // session id may already name a same-id replacement. + (marker, live) => { + const key = sessionKey(marker.providerSessionId); + const ownerInstance = marker.ownershipResolved + ? marker.instanceId + : (live.get(key)?.runtime.instanceId ?? + releasing.get(key)?.entry.runtime.instanceId ?? + opening.get(key)?.instanceId); + return ownerInstance === undefined || ownerInstance === instanceId; + }, + ); + const failure = [...outcomes, ...startupOutcomes, ...revocationOutcomes].find( + Exit.isFailure, + ); if (failure !== undefined && Exit.isFailure(failure)) { return yield* Effect.failCause(failure.cause); } }).pipe( + Effect.ensuring( + Effect.sync(() => { + const remaining = (closingInstanceCounts.get(instanceId) ?? 1) - 1; + if (remaining <= 0) { + closingInstanceCounts.delete(instanceId); + } else { + closingInstanceCounts.set(instanceId, remaining); + } + }), + ), Effect.mapError( (cause) => new ProviderSessionCloseError({ @@ -1761,122 +3613,428 @@ export const layerWithOptions = ( ), release: releaseEntry, detach: (input) => - Effect.gen(function* () { - const key = sessionKey(input.providerSessionId); - const currentEntry = (yield* Ref.get(sessions)).get(key); - if (currentEntry?.supportsMultipleProviderThreads === true) { - const projection = yield* Effect.option( - projectionStore.getThreadRecords(input.threadId, [ - "providerThreads", - "providerTurns", - ]), - ); - if (Option.isSome(projection)) { - const providerThreads = new Map( - projection.value.providerThreads - .filter((thread) => thread.providerSessionId === input.providerSessionId) - .map((thread) => [thread.id, thread] as const), - ); - const activeTurns = projection.value.providerTurns.filter( - (turn) => turn.status === "running" && providerThreads.has(turn.providerThreadId), - ); - yield* Effect.forEach( - activeTurns, - (turn) => - currentEntry.exposedRuntime - .interruptTurn({ - providerThread: providerThreads.get(turn.providerThreadId)!, - providerTurnId: turn.id, - }) - .pipe( - Effect.catchCause((cause) => - Effect.logWarning( - "orchestration-v2.driver-session.detach-interrupt-failed", - { - providerSessionId: input.providerSessionId, - threadId: input.threadId, - providerTurnId: turn.id, - cause, - }, - ), + Effect.uninterruptibleMask((restore) => + Effect.suspend(() => { + // Registered inside the mask before the body can suspend, and + // removed on any exit below: teardown drains this set before + // snapshotting pendingRevocations, so a detach already in + // flight cannot register a credential sweep the join misses. + const inflight: InflightDetach = { + providerSessionId: input.providerSessionId, + threadId: input.threadId, + detached: Deferred.makeUnsafe(), + ownershipResolved: false, + instanceId: undefined, + revokeDone: undefined, + }; + inflightDetaches.add(inflight); + return restore( + Effect.gen(function* () { + const key = sessionKey(input.providerSessionId); + const pendingRelease = releasing.get(key); + if (pendingRelease !== undefined) { + inflight.instanceId = pendingRelease.entry.runtime.instanceId; + inflight.ownershipResolved = true; + let revokeDone: + | Deferred.Deferred + | undefined; + if (input.revokeMcpCredential === true) { + const recorded = pendingRelease.mcpCredentialIdByThread.get(input.threadId); + // The claim-aware revocation can wait on a peer's stalled + // prepare; run it detached so this retry reaches the bounded + // join instead of blocking indefinitely first. The pending + // release's own sweep is the primary revocation path for a + // credential it still records — but a credential an earlier + // detach already pruned is absent from its record, so this + // tracked sweep is the only owner for that work. + revokeDone = yield* forkTrackedRevocation( + input.providerSessionId, + input.threadId, + revokeUnclaimedThreadCredentials( + input.threadId, + recorded === undefined ? [] : [recorded], ), + pendingRelease.entry.runtime.instanceId, + (done) => { + inflight.revokeDone = done; + }, + ); + } + yield* releaseEntry({ + providerSessionId: input.providerSessionId, + reason: "manual_shutdown", + joinRelease: pendingRelease, + }); + // The release's sweep does not cover a credential pruned before + // its record was captured — joining only the release would + // report success while that revocation is still running. + if (revokeDone !== undefined) { + const settled = yield* Deferred.await(revokeDone).pipe( + Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), + ); + if (Option.isNone(settled)) { + return yield* new ProviderSessionReleaseError({ + providerSessionId: input.providerSessionId, + reason: "runtime_error", + cause: + "MCP credential revocation did not finish within 30 seconds and is still running.", + }); + } + } + return; + } + const currentEntry = (yield* Ref.get(sessions)).get(key); + inflight.instanceId = + currentEntry?.runtime.instanceId ?? opening.get(key)?.instanceId; + inflight.ownershipResolved = true; + const openingRecord = currentEntry === undefined ? opening.get(key) : undefined; + if (openingRecord !== undefined && openingRecord.threadId === input.threadId) { + // Detaching the thread an in-flight startup belongs to aborts + // it: mark the open like a close and join its bounded unwind + // instead of returning success while a provider process may + // still be spawning for this thread. + return yield* releaseEntry({ + providerSessionId: input.providerSessionId, + reason: "manual_shutdown", + }); + } + if (currentEntry?.supportsMultipleProviderThreads === true) { + const projection = yield* Effect.option( + projectionStore.getThreadRecords(input.threadId, ["providerThreads", "providerTurns"]), + ); + if (Option.isSome(projection)) { + const providerThreads = new Map( + projection.value.providerThreads + .filter((thread) => thread.providerSessionId === input.providerSessionId) + .map((thread) => [thread.id, thread] as const), + ); + const activeTurns = projection.value.providerTurns.filter( + (turn) => + turn.status === "running" && providerThreads.has(turn.providerThreadId), + ); + yield* Effect.forEach( + activeTurns, + (turn) => + currentEntry.exposedRuntime + .interruptTurn({ + providerThread: providerThreads.get(turn.providerThreadId)!, + providerTurnId: turn.id, + }) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning( + "orchestration-v2.driver-session.detach-interrupt-failed", + { + providerSessionId: input.providerSessionId, + threadId: input.threadId, + providerTurnId: turn.id, + cause, + }, + ), + ), + ), + { concurrency: 1, discard: true }, + ); + } + } + // The attachment mutation and credential cleanup run under the same + // [session, thread] lock an attach holds through its provisional + // bookkeeping, MCP preparation, and event write. Without it a detach + // can land between an in-flight attach's provisional add and its + // post-prepare completion: the attach would then record a credential + // and emit its attached event for a thread the entry no longer + // tracks, resurrecting a detached binding. + // The acquisition itself is bounded: an attach can hold this + // lock through MCP preparation, and a stalled holder must not + // keep a cleanup caller waiting past the bound. Timing out here + // leaves ownership with the lock holder — nothing this detach + // could have claimed is touched. + const locked = yield* threadAttach + .withLock( + threadAttachKey( + input.providerSessionId, + currentEntry?.runtime, + input.threadId, ), - { concurrency: 1, discard: true }, - ); - } - } - const detached = yield* Ref.modify(sessions, (current) => { - const entry = current.get(key); - if (entry === undefined || !entry.attachedThreadIds.has(input.threadId)) { - return [Option.none(), current] as const; - } - const attachedThreadIds = new Set(entry.attachedThreadIds); - attachedThreadIds.delete(input.threadId); - const loadedProviderThreadKeyByThread = new Map( - entry.loadedProviderThreadKeyByThread, + // Masked so the attachment mutation and the release handoff + // stay atomic with respect to interruption: once the last + // exclusive attachment is removed, the release must exist + // before this fiber can be interrupted out of the revocation + // wait below. + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const detached = yield* Ref.modify(sessions, (current) => { + const entry = current.get(key); + if ( + entry === undefined || + currentEntry === undefined || + // The entry read above may belong to a released session: a + // same-id reopen installs a new runtime, and this detach must + // never mutate the replacement's attachments. + entry.runtime !== currentEntry.runtime || + !entry.attachedThreadIds.has(input.threadId) + ) { + return [ + Option.none<{ + readonly entry: LiveSessionEntry; + readonly priorMcpCredentialId: string | undefined; + readonly release: PendingSessionRelease | undefined; + }>(), + current, + ] as const; + } + const attachedThreadIds = new Set(entry.attachedThreadIds); + attachedThreadIds.delete(input.threadId); + const loadedProviderThreadKeyByThread = new Map( + entry.loadedProviderThreadKeyByThread, + ); + loadedProviderThreadKeyByThread.delete(input.threadId); + // For a plain (workspace-change) detach, the credential id stays + // recorded: the thread may re-attach and reuse it, and + // releaseEntry revokes it when the provider process finally goes + // away. A terminal detach (archive/delete) prunes the record so + // nothing vetoes the revocation below. + const mcpCredentialIdByThread = + input.revokeMcpCredential === true + ? (() => { + const pruned = new Map(entry.mcpCredentialIdByThread); + pruned.delete(input.threadId); + return pruned; + })() + : entry.mcpCredentialIdByThread; + const updatedEntry = { + ...entry, + attachedThreadIds, + loadedProviderThreadKeyByThread, + mcpCredentialIdByThread, + }; + // An exclusive session whose last attachment just + // detached must move into `releasing` in this same + // atomic step: a close landing between the mutation and + // a later claim would otherwise register the emptied + // entry — a release record with no thread or credential + // ownership — and the thread-scoped cleanup gate would + // let this thread's replacement open while the provider + // process still runs. The record keeps the detached + // thread and its pruned credential so the release's own + // bounded sweep and its joiners cover them. + const release: PendingSessionRelease | undefined = + attachedThreadIds.size === 0 && !entry.supportsMultipleProviderThreads + ? { + entry: { + ...updatedEntry, + attachedThreadIds: new Set([input.threadId]), + }, + threadIds: new Set([ + input.threadId, + ...entry.mcpCredentialIdByThread.keys(), + ]), + mcpCredentialIdByThread: new Map(entry.mcpCredentialIdByThread), + done: Deferred.makeUnsafe(), + } + : undefined; + const updated = new Map(current); + if (release === undefined) { + updated.set(key, updatedEntry); + } else { + releasing.set(key, release); + updated.delete(key); + } + return [ + Option.some({ + entry: updatedEntry, + priorMcpCredentialId: entry.mcpCredentialIdByThread.get( + input.threadId, + ), + release, + }), + updated, + ] as const; + }); + // Forked while masked so the handoff cannot be skipped by + // an interruption delivered during the revocation wait + // below, and joined by the caller after the lock is + // released. The record was already claimed atomically + // above, so this only starts the cleanup worker. + const releaseFiber = + Option.isSome(detached) && detached.value.release !== undefined + ? yield* releaseEntry({ + providerSessionId: input.providerSessionId, + reason: "manual_shutdown", + preclaimed: detached.value.release, + ...(input.detail === undefined ? {} : { detail: input.detail }), + }).pipe(Effect.forkDetach({ startImmediately: true })) + : undefined; + // Plain detaches deliberately do not revoke: a detached thread's + // provider process may still be alive (shared multi-thread codex + // session across a workspace handoff) and holds its MCP client's + // credential for the thread it will re-attach with. Credentials + // are revoked when the session entry is released (process gone) + // or rotated on the next attach if they stopped resolving. + // Terminal detaches (thread archived or deleted) revoke the + // thread's credentials — claim-aware: a credential a live peer + // session still records survives until that holder's own cleanup + // revokes it, and a credential the releasing session still records + // is handled by that pending cleanup. + if (input.revokeMcpCredential === true) { + // Read the recorded credential only from a pending release + // that belongs to the session this detach targeted; a + // pending record for a same-id replacement owns a different + // credential map. + const pendingNow = releasing.get(key); + const sameInstancePending = + pendingNow !== undefined && + currentEntry !== undefined && + pendingNow.entry.runtime === currentEntry.runtime + ? pendingNow + : undefined; + const priorMcpCredentialId = Option.isSome(detached) + ? detached.value.priorMcpCredentialId + : sameInstancePending?.mcpCredentialIdByThread.get(input.threadId); + const revoke = revokeUnclaimedThreadCredentials( + input.threadId, + priorMcpCredentialId === undefined ? [] : [priorMcpCredentialId], + ); + // Forked detached and tracked so interrupting this detach + // cannot strand the pruned credential: the sweep keeps the + // captured id and finishes under mcpPrepareLock even after + // this fiber is gone, and a retry joins `revokeDone` + // instead of forking a second sweep or reporting success + // over a running one. The fork stays masked — a pending + // interruption delivered by restore() below must not skip + // it — while the join stays interruptible so a completed + // detach still waits for revocation. + const revokeDone = yield* forkTrackedRevocation( + input.providerSessionId, + input.threadId, + revoke, + Option.isSome(detached) + ? detached.value.entry.runtime.instanceId + : (sameInstancePending?.entry.runtime.instanceId ?? + currentEntry?.runtime.instanceId), + (done) => { + inflight.revokeDone = done; + }, + ); + if (Option.isSome(detached) && releaseFiber === undefined) { + // No release owns this credential's sweep (the session + // stays live for other threads), so this join is the + // only wait on revocation — and the sweep acquires + // mcpPrepareLock, which a stalled peer prepare can hold + // indefinitely. Bound the wait like any cleanup: the + // detached sweep still finishes the revocation late, + // and the timeout keeps the reported state honest + // instead of blocking past the cleanup contract. + const settled = yield* restore( + Deferred.await(revokeDone).pipe( + Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), + ), + ); + if (Option.isNone(settled)) { + return yield* new ProviderSessionReleaseError({ + providerSessionId: input.providerSessionId, + reason: "runtime_error", + cause: + "MCP credential revocation did not finish within 30 seconds and is still running.", + }); + } + } + // When a release was forked above, the pruned credential + // was handed to its record via extraMcpCredentials and its + // own bounded sweep revokes it — the bounded release join + // below is this detach's wait, so joining the tracked + // sweep too would only re-expose the unbounded + // mcpPrepareLock wait. When nothing detached, the post- + // lock tail joins `revokeDone` so a retry never reports + // success over a running revocation. + return { detached, releaseFiber, revokeDone }; + } + return { detached, releaseFiber, revokeDone: undefined }; + }), + ), + ) + .pipe(Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS)); + if (Option.isNone(locked)) { + return yield* new ProviderSessionReleaseError({ + providerSessionId: input.providerSessionId, + reason: "runtime_error", + cause: + "An in-flight attachment or cleanup held this session's lock for over " + + "30 seconds; the detach could not run and ownership stays with the holder.", + }); + } + const { detached, releaseFiber, revokeDone } = locked.value; + if (Option.isNone(detached)) { + // A close may have moved the entry into `releasing` while this + // detach was reading the projection or interrupting turns; join + // that cleanup like any retry instead of reporting success over + // still-pending work. The join is pinned to the ownership this + // detach observed at read time: a pending record for a same-id + // replacement is a different session and must not be joined. + // When no entry was seen at all, the pending-release check at + // the top already passed, so anything releasing now belongs to + // another instance. + if (currentEntry !== undefined && releasing.has(key)) { + yield* releaseEntry({ + providerSessionId: input.providerSessionId, + reason: "manual_shutdown", + joinOnly: true, + expectedRuntime: currentEntry.runtime, + }); + } + // An earlier terminal detach for this thread may have started + // a revocation this call must not outrun: the caller that + // forked it may already have timed out or been interrupted. + // Join the tracked sweep (bounded like any cleanup wait) so a + // retry only reports success once revocation actually finished. + // The release record above does not cover a credential pruned + // before it was captured, so this join runs whether or not a + // release was pending. + if (revokeDone !== undefined) { + const settled = yield* Deferred.await(revokeDone).pipe( + Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), + ); + if (Option.isNone(settled)) { + return yield* new ProviderSessionReleaseError({ + providerSessionId: input.providerSessionId, + reason: "runtime_error", + cause: + "MCP credential revocation did not finish within 30 seconds and is still running.", + }); + } + } + return; + } + if (releaseFiber !== undefined) { + // Forked inside the lock before the revocation wait so an + // interrupted detach cannot strand the session; the join keeps + // a completed detach waiting on the release outcome. + yield* Fiber.join(releaseFiber); + return; + } + yield* scheduleIdleRelease(input.providerSessionId, detached.value.entry.runtime); + }).pipe( + Effect.catchCause((cause) => + Effect.fail( + new ProviderSessionReleaseError({ + providerSessionId: input.providerSessionId, + reason: "manual_shutdown", + cause, + }), + ), + ), + ), + ).pipe( + Effect.onExit(() => + Effect.andThen( + Deferred.done(inflight.detached, Exit.void), + Effect.sync(() => { + inflightDetaches.delete(inflight); + }), + ), + ), ); - loadedProviderThreadKeyByThread.delete(input.threadId); - // For a plain (workspace-change) detach, the credential id stays - // recorded: the thread may re-attach and reuse it, and - // releaseEntry revokes it when the provider process finally goes - // away. A terminal detach (archive/delete) prunes the record so - // nothing vetoes the revocation below. - const mcpCredentialIdByThread = - input.revokeMcpCredential === true - ? (() => { - const pruned = new Map(entry.mcpCredentialIdByThread); - pruned.delete(input.threadId); - return pruned; - })() - : entry.mcpCredentialIdByThread; - const updatedEntry = { - ...entry, - attachedThreadIds, - loadedProviderThreadKeyByThread, - mcpCredentialIdByThread, - }; - const updated = new Map(current); - updated.set(key, updatedEntry); - return [Option.some(updatedEntry), updated] as const; - }); - // Plain detaches deliberately do not revoke: a detached thread's - // provider process may still be alive (shared multi-thread codex - // session across a workspace handoff) and holds its MCP client's - // credential for the thread it will re-attach with. Credentials - // are revoked when the session entry is released (process gone) - // or rotated on the next attach if they stopped resolving. - // Terminal detaches (thread archived or deleted) revoke the - // thread's credentials immediately, even on a retry where the - // entry is already gone: there is no legitimate future re-attach, - // and the token must not outlive the thread. - if (input.revokeMcpCredential === true) { - yield* clearMcpSession(input.threadId); - } - if (Option.isNone(detached)) { - return; - } - if ( - detached.value.attachedThreadIds.size === 0 && - !detached.value.supportsMultipleProviderThreads - ) { - yield* releaseEntry({ - providerSessionId: input.providerSessionId, - reason: "manual_shutdown", - ...(input.detail === undefined ? {} : { detail: input.detail }), - }); - return; - } - yield* scheduleIdleRelease(input.providerSessionId); - }).pipe( - Effect.catchCause((cause) => - Effect.fail( - new ProviderSessionReleaseError({ - providerSessionId: input.providerSessionId, - reason: "manual_shutdown", - cause, - }), - ), - ), + }), ), } satisfies ProviderSessionManagerV2Shape); }), diff --git a/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts b/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts index f2af0e44b9cc..d919bdaa38bb 100644 --- a/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts @@ -156,6 +156,7 @@ const runCursorRecovery = Effect.fn("runCursorRecovery")(function* (input: { steps: phase1Steps, projectionThreadIds: materialized.projectionThreadIds, runtimePolicyOverride: { cwd: tempDir }, + runtimeRestart: true, }, harness, options, @@ -171,6 +172,7 @@ const runCursorRecovery = Effect.fn("runCursorRecovery")(function* (input: { steps: phase2Steps, projectionThreadIds: materialized.projectionThreadIds, runtimePolicyOverride: { cwd: tempDir }, + runtimeRestart: true, }, harness, options, diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 54114648ebca..57a60fbebc8b 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -26,7 +26,8 @@ import { layer as checkpointRollbackServiceLayer } from "../CheckpointRollbackSe import { layer as commandPolicyLayer } from "../CommandPolicy.ts"; import { layer as commandReceiptStoreLayer } from "../CommandReceiptStore.ts"; import { layer as contextHandoffServiceLayer } from "../ContextHandoffService.ts"; -import { layer as effectOutboxLayer } from "../EffectOutbox.ts"; +import * as Context from "effect/Context"; +import * as EffectOutbox from "../EffectOutbox.ts"; import { executorLayer as effectExecutorLayer, layer as effectWorkerLayer, @@ -44,7 +45,7 @@ import { ProviderAuthService } from "../../provider/Services/ProviderAuthService import { layer as providerContinuationRequestsLayer } from "../ProviderContinuationRequests.ts"; import { workerLive as providerContinuationWorkerLive } from "../ProviderContinuationService.ts"; import { layer as providerEventIngestorLayer } from "../ProviderEventIngestor.ts"; -import { layerWithOptions as providerSessionManagerLayerWithOptions } from "../ProviderSessionManager.ts"; +import * as ProviderSessionManager from "../ProviderSessionManager.ts"; import { layer as providerSwitchServiceLayer } from "../ProviderSwitchService.ts"; import { layer as providerTurnControlServiceLayer } from "../ProviderTurnControlService.ts"; import { layer as providerTurnStartServiceLayer } from "../ProviderTurnStartService.ts"; @@ -161,6 +162,15 @@ export interface OrchestratorV2ProviderReplayScenario< > extends OrchestratorV2Scenario { readonly transcript: Transcript; readonly runtimePolicyOverride?: RuntimePolicyV2Override; + /** + * Models a production restart boundary around the scenario run: reconcile + * the effect outbox for the (recreated) runtime before the first step, the + * same rows serverRuntimeStartup retires via ProviderRuntimeRecoveryService + * on boot, and drive provider-session shutdown to completion before the + * caller's scope teardown so in-flight releases finish instead of dying + * mid-close with the scope. + */ + readonly runtimeRestart?: boolean; } export interface OrchestratorV2ProviderReplayHarness< @@ -214,8 +224,22 @@ export function runOrchestratorV2ProviderReplayScenario< ...options, replayGate, }); + const run = runOrchestratorV2Scenario(scenario, { replayGate }); - return runOrchestratorV2Scenario(scenario, { replayGate }).pipe(Effect.provide(layer)); + if (scenario.runtimeRestart !== true) { + return run.pipe(Effect.provide(layer)); + } + return Effect.gen(function* () { + yield* Effect.flatMap(EffectOutbox.EffectOutboxV2, (outbox) => + outbox.reconcileAfterProcessLoss.pipe(Effect.orDie), + ); + const result = yield* run; + yield* Effect.flatMap( + ProviderSessionManager.ProviderSessionManagerV2, + (sessions) => sessions.shutdown, + ); + return result; + }).pipe(Effect.provide(layer)); } export function makeOrchestratorV2ProviderReplayLayer< @@ -236,7 +260,11 @@ export function makeOrchestratorV2ProviderReplayLayer< readonly replayGate?: ProviderReplayGate; } = {}, ): Layer.Layer< - OrchestratorV2 | OrchestrationEffectWorkerV2 | EventSinkV2, + | OrchestratorV2 + | OrchestrationEffectWorkerV2 + | EventSinkV2 + | EffectOutbox.EffectOutboxV2 + | ProviderSessionManager.ProviderSessionManagerV2, Error | MigrationError | PlatformError.PlatformError | SqlError > { const registryLayer = harness.makeProviderAdapterRegistryLayer( @@ -260,7 +288,11 @@ export function makeOrchestratorV2ReplayLayerWithRegistry( readonly runContinuationWorker?: boolean; } = {}, ): Layer.Layer< - OrchestratorV2 | OrchestrationEffectWorkerV2 | EventSinkV2, + | OrchestratorV2 + | OrchestrationEffectWorkerV2 + | EventSinkV2 + | EffectOutbox.EffectOutboxV2 + | ProviderSessionManager.ProviderSessionManagerV2, Error | MigrationError | PlatformError.PlatformError | SqlError > { const serverConfigLayer = Layer.effect( @@ -286,7 +318,7 @@ export function makeOrchestratorV2ReplayLayerWithRegistry( eventStoreLayer, projectionStoreLayer, commandReceiptStoreLayer, - effectOutboxLayer, + EffectOutbox.layer, turnItemPositionStoreLayer, ).pipe(Layer.provide(databaseLayer)); const eventSinkProvided = eventSinkLayer.pipe( @@ -318,7 +350,7 @@ export function makeOrchestratorV2ReplayLayerWithRegistry( idAllocatorLayer, providerEventIngestorProvided, ); - const providerSessionManagerProvided = providerSessionManagerLayerWithOptions({ + const providerSessionManagerProvided = ProviderSessionManager.layerWithOptions({ configureMcp: false, }).pipe( Layer.provide( @@ -444,11 +476,23 @@ export function makeOrchestratorV2ReplayLayerWithRegistry( const effectWorkerProvided = effectWorkerLayer.pipe( Layer.provide(Layer.merge(storesLayer, effectExecutorProvided)), ); + const lifecycleServicesLayer = Layer.unwrap( + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + return Layer.succeedContext( + Context.make(EffectOutbox.EffectOutboxV2, outbox).pipe( + Context.add(ProviderSessionManager.ProviderSessionManagerV2, sessions), + ), + ); + }), + ).pipe(Layer.provide(Layer.mergeAll(storesLayer, providerSessionManagerProvided))); const replayRuntime = Layer.mergeAll( orchestratorProvided, effectWorkerProvided, eventSinkProvided, continuationWorkerProvided, + lifecycleServicesLayer, ).pipe(Layer.provide(worktreeRepairDependenciesTestLayer), Layer.provide(NodeServices.layer)); // Build the daemon from the exact worker instance exposed alongside the diff --git a/docs/orchestration-v2/provider-switching-and-context.md b/docs/orchestration-v2/provider-switching-and-context.md index 17515dda2133..a6164d9a0364 100644 --- a/docs/orchestration-v2/provider-switching-and-context.md +++ b/docs/orchestration-v2/provider-switching-and-context.md @@ -121,6 +121,16 @@ The user message remains the actual run input. The handoff context is a separate ## Strategy Selection +### Session cleanup + +Closing a provider session waits up to 30 seconds for cleanup. A timeout returns an error, records the session as `error`, and leaves cleanup running. It does not confirm that the old process has stopped. + +While cleanup is pending, the session manager rejects reuse of the releasing session ID, creation of sessions for its recorded app threads, and new attachments of those threads to another session. A thread already attached to a different live session can keep reusing that session. Repeated close or detach requests wait for the original cleanup. Successful cleanup removes this block; failed cleanup keeps it until the environment restarts. Other threads can still open sessions. + +This applies when a session is released. Detaching a thread from a shared session does not necessarily release that session, so this is not a complete shared-process account-switching protocol. + +### Handoff selection + Recommended strategy order: 1. No handoff: target provider thread already has current app run coverage. From 83bfa41f266bbda026c6b03a3afd265035cc48d7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:12:09 +1000 Subject: [PATCH 02/12] fix(server): hold attach lock through adapter ops and propagate cleanup failures A terminal detach could win the reacquire race between attachThreadOrReject releasing the thread-attach lock and the resource-creating adapter call reacquiring it, stripping the attachment and its credential claim before the op was admitted. The lock now covers bookkeeping and the adapter call in one hold, and markBusy moves before the attach to avoid a lifecycle-lock cycle. Native close failures were also swallowed by Effect.ignore in the Cursor, Claude, ACP, and OpenCode session finalizers, converting failed cleanup into a clean release over still-owned provider resources. They now propagate (or die inside Effect.addFinalizer slots) so the manager keeps ownership, and a failed startup close retains the MCP credential reservation until cleanup status is known. stopSessions now delegates to the closeInstance barrier so sign-in/out joins in-flight startups and failed cleanups instead of trusting projected status. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../orchestration-v2/Adapters/AcpAdapterV2.ts | 9 +- .../Adapters/ClaudeAdapterV2.test.ts | 63 ++ .../Adapters/ClaudeAdapterV2.ts | 56 +- .../Adapters/CursorAdapterV2.test.ts | 56 + .../Adapters/CursorAdapterV2.ts | 11 +- .../Adapters/OpenCodeAdapterV2.test.ts | 65 +- .../Adapters/OpenCodeAdapterV2.ts | 32 +- .../ProviderSessionManager.test.ts | 129 ++- .../ProviderSessionManager.ts | 1001 +++++++++-------- .../Layers/ProviderAuthService.test.ts | 304 ++--- .../provider/Layers/ProviderAuthService.ts | 73 +- 11 files changed, 1001 insertions(+), 798 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts index b448ba13e007..30bb4ed8a232 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts @@ -7014,7 +7014,12 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV !restartRequired && !transportHadOutstandingResponses ) { - yield* runtime.closeSession().pipe(Effect.ignore); + // A failed native close means the ACP subprocess may still + // own the thread — the failure must reach Scope.close so + // the manager keeps cleanup ownership instead of releasing + // a live process. Finalizers cannot carry typed errors, + // so the failure surfaces as a defect. + yield* runtime.closeSession().pipe(Effect.orDie); } }), ); @@ -7023,7 +7028,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV yield* flavor.assertComplete.pipe(Effect.orDie); } if (runtimeScope !== undefined) { - yield* Scope.close(runtimeScope, Exit.void).pipe(Effect.ignore); + yield* Scope.close(runtimeScope, Exit.void); } }), ); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 01318f945ec8..ef6196485e3e 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -7109,3 +7109,66 @@ describe("ClaudeAdapterV2 query message stream", () => { }), ); }); + +describe("ClaudeAdapterV2 session cleanup", () => { + it.effect("propagates a native query close failure through the session scope", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-close-", + }); + const scope = yield* Scope.make(); + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + path: yield* Path.Path, + idAllocator: yield* IdAllocatorV2, + queryRunner: { + allocateSessionId: Effect.succeed("native-thread-claude-close"), + open: () => + Effect.succeed({ + messages: Stream.never, + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + close: Effect.die("claude query close failed"), + }), + forkSession: () => Effect.die("unused"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-close"); + const runtime = yield* adapter + .openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-close"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }) + .pipe(Effect.provideService(Scope.Scope, scope)); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-claude-close"), + text: "hello", + attachments: [], + }), + ); + + // The native close failure must reach the scope close so the session + // manager records a failed cleanup instead of a clean release. + const closeExit = yield* Scope.close(scope, Exit.void).pipe(Effect.exit); + assert.isTrue(Exit.isFailure(closeExit)); + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, idAllocatorLayer))), + ); +}); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index b7cc51ac67ba..d93539778fff 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -6230,7 +6230,10 @@ export function makeClaudeAdapterV2( // the replacement open succeeds or fails below. const closedExistingNativeThreadId = existing !== null ? existing.nativeThreadId : null; if (existing !== null) { - yield* existing.query.close.pipe(Effect.ignore); + // A failed close means the replaced query's CLI may still be + // alive: the failure must surface instead of opening a second + // process that silently co-owns the native thread. + yield* existing.query.close; if (existing.nativeThreadId !== nativeThreadId) { yield* clearWakeStateForNativeThread(existing.nativeThreadId); yield* resetBackgroundTaskStateForNativeThreadProcess(existing.nativeThreadId, { @@ -6566,29 +6569,31 @@ export function makeClaudeAdapterV2( return next; }); yield* existing.query.interrupt; - yield* existing.query.close.pipe(Effect.ignore); + // The interrupt bookkeeping completes either way, but a failed + // close means the query's CLI may still be alive — the failure + // must reach the caller instead of reporting a clean interrupt. + const closeExit = yield* existing.query.close.pipe(Effect.exit); const closed = yield* Deferred.await(existing.closed).pipe( Effect.timeoutOption("10 seconds"), ); - if (Option.isSome(closed)) { - return; + if (Option.isNone(closed)) { + const completedAt = yield* DateTime.now; + yield* Effect.logWarning("orchestration-v2.claude-query-interrupt-timeout", { + providerSessionId: input.providerSessionId, + providerThreadId: turnInput.providerThread.id, + providerTurnId: turnInput.providerTurnId, + }); + yield* Ref.update(queryContext, (current) => + current?.query === existing.query ? null : current, + ); + yield* finalizeActiveTurn({ + context: currentTurn, + status: "interrupted", + completedAt, + }); + yield* Deferred.succeed(existing.closed, undefined); } - - const completedAt = yield* DateTime.now; - yield* Effect.logWarning("orchestration-v2.claude-query-interrupt-timeout", { - providerSessionId: input.providerSessionId, - providerThreadId: turnInput.providerThread.id, - providerTurnId: turnInput.providerTurnId, - }); - yield* Ref.update(queryContext, (current) => - current?.query === existing.query ? null : current, - ); - yield* finalizeActiveTurn({ - context: currentTurn, - status: "interrupted", - completedAt, - }); - yield* Deferred.succeed(existing.closed, undefined); + return yield* closeExit; }, (effect, turnInput) => effect.pipe( @@ -6655,7 +6660,11 @@ export function makeClaudeAdapterV2( const closeSession = Effect.fnUntraced(function* () { const existing = yield* Ref.get(queryContext); if (existing !== null) { - yield* existing.query.close.pipe(Effect.ignore); + // A failed native close must reach the session scope's close so + // the manager keeps ownership instead of recording a clean + // release. Finalizers cannot carry typed errors, so the failure + // surfaces as a defect that fails the close the same way. + yield* existing.query.close.pipe(Effect.orDie); } yield* Effect.yieldNow; yield* queryRunner.assertComplete.pipe( @@ -6674,7 +6683,10 @@ export function makeClaudeAdapterV2( return; } - yield* existing.query.close.pipe(Effect.ignore); + // A close failure means the live query may still own the native + // thread — propagate it instead of letting a rollback or fork spawn + // a second query on top of it. + yield* existing.query.close; const closed = yield* Deferred.await(existing.closed).pipe( Effect.timeoutOption("10 seconds"), ); diff --git a/apps/server/src/orchestration-v2/Adapters/CursorAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CursorAdapterV2.test.ts index 7a4732b159d5..9033bb38bd77 100644 --- a/apps/server/src/orchestration-v2/Adapters/CursorAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CursorAdapterV2.test.ts @@ -15,11 +15,13 @@ import { } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import { ServerConfig, layerTest as serverConfigLayerTest } from "../../config.ts"; @@ -815,6 +817,60 @@ describe("CursorAdapterV2", () => { assert.isFalse(isCursorCancellationError(null)); }); + it.effect("propagates a native session close failure through the session scope", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const workspace = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "cursor-v2-close-", + }); + const scope = yield* Scope.make(); + const instanceId = ProviderInstanceId.make("cursor"); + const threadId = ThreadId.make("cursor-close-thread"); + const modelSelection = { instanceId, model: "composer-2.5" }; + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: workspace, + }); + const adapter = makeCursorAdapterV2({ + instanceId, + settings: yield* decodeCursorSettings({}), + environment: { HOME: workspace }, + fileSystem, + path, + idAllocator: yield* IdAllocatorV2, + serverConfig: yield* ServerConfig.pipe( + Effect.provide(serverConfigLayerTest(workspace, { prefix: "cursor-v2-close-config-" })), + ), + runner: { + assertComplete: Effect.void, + open: () => + Effect.succeed({ + agentId: "native-cursor-close", + listMessages: Effect.succeed([]), + close: Effect.die("cursor session close failed"), + send: () => Effect.die("unused in close test"), + }), + }, + }); + const runtime = yield* adapter + .openSession({ + threadId, + providerSessionId: ProviderSessionId.make("cursor-close-session"), + modelSelection, + runtimePolicy, + }) + .pipe(Effect.provideService(Scope.Scope, scope)); + yield* runtime.ensureThread({ threadId, modelSelection, runtimePolicy }); + + // The native close failure must reach the scope close so the session + // manager records a failed cleanup instead of a clean release. + const closeExit = yield* Scope.close(scope, Exit.void).pipe(Effect.exit); + assert.isTrue(Exit.isFailure(closeExit)); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, idAllocatorLayer))), + ); + it("preserves failed nested read calls when Cursor omits their path", () => { assert.deepEqual( nestedToolCallFromEnvelope({ diff --git a/apps/server/src/orchestration-v2/Adapters/CursorAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CursorAdapterV2.ts index 69ef6d899c7a..c19fcc8c25e0 100644 --- a/apps/server/src/orchestration-v2/Adapters/CursorAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CursorAdapterV2.ts @@ -2078,7 +2078,9 @@ export function makeCursorAdapterV2( return existing; } if (existing !== null) { - yield* existing.session.close.pipe(Effect.ignore); + // A close failure means the old agent may still own the native + // thread — propagate it instead of opening a second agent on top. + yield* existing.session.close; yield* Ref.set(liveAgent, null); } const sdkSession = yield* runner.open({ @@ -2351,7 +2353,12 @@ export function makeCursorAdapterV2( const closeSession = Effect.fnUntraced(function* () { const existing = yield* Ref.get(liveAgent); if (existing !== null) { - yield* existing.session.close.pipe(Effect.ignore); + // A failed native close must reach the session scope's close so + // the manager keeps ownership instead of recording a clean + // release — and the live agent stays recorded for a retry. + // Finalizers cannot carry typed errors, so the failure surfaces + // as a defect that fails the close the same way. + yield* existing.session.close.pipe(Effect.orDie); yield* Ref.set(liveAgent, null); } yield* runner.assertComplete.pipe( diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.test.ts index 4b4ef583d52a..133d663b4ffd 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.test.ts @@ -161,7 +161,17 @@ const makeOpenCodeRuntimeHarness = Effect.fn("makeOpenCodeRuntimeHarness")(funct environment: {}, runtime: { connectToOpenCodeServer: () => Effect.succeed({ url: "http://test.invalid", external: true }), - createOpenCodeSdkClient: () => client, + createOpenCodeSdkClient: () => ({ + ...(client as Record), + session: { + // Real SDK clients always implement the teardown calls the session + // finalizer invokes; fakes that omit them get benign stubs so scope + // close can complete. Tests that stub their own keep theirs. + abort: async () => ({ data: true }), + children: async () => ({ data: [] }), + ...(client as { session?: Record }).session, + }, + }), } as unknown as OpenCodeRuntimeShape, idAllocator, serverConfig: { @@ -493,12 +503,58 @@ describe("OpenCodeAdapterV2", () => { }).pipe(Effect.provide(idAllocatorLayer), Effect.scoped), ); + it.effect("propagates an external session abort failure while continuing cleanup", () => + Effect.gen(function* () { + const scope = yield* Scope.make(); + const nativeEvents = asyncEventStream(); + const calls: string[] = []; + yield* makeOpenCodeRuntimeHarness("release-failed-abort", "root", { + event: { + subscribe: async (_input: unknown, options: { signal: AbortSignal }) => { + options.signal.addEventListener("abort", () => { + calls.push("stream.close"); + nativeEvents.close(); + }); + return { stream: nativeEvents.stream }; + }, + }, + session: { + create: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), + abort: async ({ sessionID }: { sessionID: string }) => { + calls.push(`abort:${sessionID}`); + if (sessionID === "root") throw new Error("root abort failed"); + return { data: true }; + }, + children: async ({ sessionID }: { sessionID: string }) => { + calls.push(`children:${sessionID}`); + return { data: sessionID === "root" ? [{ id: "child" }] : [] }; + }, + }, + }).pipe(Effect.provideService(Scope.Scope, scope)); + // The abort failure must reach the scope close so the session manager + // records a failed cleanup instead of a clean release — while the + // descendant sweep and stream teardown still run to completion. + const closeExit = yield* Scope.close(scope, Exit.void).pipe(Effect.exit); + assert.isTrue(Exit.isFailure(closeExit)); + assert.deepEqual(calls, [ + "abort:root", + "children:root", + "abort:child", + "children:child", + "stream.close", + ]); + }).pipe(Effect.provide(idAllocatorLayer), Effect.scoped), + ); + for (const failure of ["enumeration", "abort", "not-found", "timeout"] as const) { it.effect(`reports descendant cleanup ${failure}`, () => Effect.gen(function* () { const nativeEvents = asyncEventStream(); const called = promiseGate(); let childSignal: AbortSignal | undefined; + // The failure under test only applies to the interruptTurn sweep; once + // it settles, teardown aborts succeed so scope close stays clean. + let settled = false; const harness = yield* makeOpenCodeRuntimeHarness(`cleanup-${failure}`, "root", { event: { subscribe: async () => ({ stream: nativeEvents.stream }) }, session: { @@ -507,14 +563,14 @@ describe("OpenCodeAdapterV2", () => { get: async () => ({ data: { id: "root", time: { created: 1, updated: 1 } } }), messages: async () => ({ data: [] }), children: async ({ sessionID }: { sessionID: string }) => { - if (failure === "enumeration") throw new Error("cannot enumerate"); + if (failure === "enumeration" && !settled) throw new Error("cannot enumerate"); return { data: sessionID === "root" ? [{ id: "child" }] : [] }; }, abort: async ( { sessionID }: { sessionID: string }, options: { signal: AbortSignal }, ) => { - if (sessionID === "root") return { data: true }; + if (sessionID === "root" || settled) return { data: true }; if (failure === "timeout" && childSignal?.aborted) return { data: true }; childSignal = options.signal; called.resolve(); @@ -543,6 +599,7 @@ describe("OpenCodeAdapterV2", () => { const result = yield* Fiber.join(stop); assert.equal(Exit.isSuccess(result), failure === "not-found"); if (failure === "timeout") assert.isTrue(childSignal?.aborted); + settled = true; }).pipe(Effect.provide(idAllocatorLayer), Effect.scoped), ); } @@ -1967,6 +2024,8 @@ describe("OpenCodeAdapterV2", () => { createInputs.push(input); return { data: { id: `ses_native_${createCount}`, time: { created: 1, updated: 1 } } }; }, + abort: async () => ({ data: true }), + children: async () => ({ data: [] }), }, } as unknown as OpencodeClient; const unused = (operation: string) => () => Effect.die(`${operation} is not used`); diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.ts index 403208d03725..a987dea1071f 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.ts @@ -2866,22 +2866,32 @@ export function makeOpenCodeAdapterV2(options: OpenCodeAdapterV2Options): Provid yield* Scope.addFinalizer( scope, Effect.suspend(() => + // Every root's abort and every descendant sweep runs even when a + // sibling fails — but the first failure propagates through the + // scope close so the manager keeps ownership instead of + // recording a clean release over a live provider process. Effect.forEach( Array.from(threads.values()).filter((state) => state.parentSubagent === null), (state) => - sdkCall("session.abort", { sessionID: state.nativeSessionId }, (signal) => - client.session.abort({ sessionID: state.nativeSessionId }, { signal }), - ).pipe( - Effect.timeout("1 second"), - Effect.ignore({ log: true }), - Effect.andThen( - abortDescendants(state.nativeSessionId).pipe( - Effect.timeout("1 second"), - Effect.ignore({ log: true }), - ), + Effect.zip( + sdkCall("session.abort", { sessionID: state.nativeSessionId }, (signal) => + client.session.abort({ sessionID: state.nativeSessionId }, { signal }), + ).pipe(Effect.timeout("1 second"), Effect.exit), + abortDescendants(state.nativeSessionId).pipe( + Effect.timeout("1 second"), + Effect.exit, ), ), - { concurrency: 8, discard: true }, + { concurrency: 8 }, + ).pipe( + Effect.flatMap((outcomes) => { + const failed = outcomes + .flatMap(([abortExit, descendantsExit]) => [abortExit, descendantsExit]) + .find(Exit.isFailure); + return failed === undefined || failed._tag !== "Failure" + ? Effect.void + : Effect.failCause(failed.cause).pipe(Effect.orDie); + }), ), ), ); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index ff25a104d83f..f46f1084f24d 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -5996,6 +5996,102 @@ it.effect( }), ); +it.effect( + "ProviderSessionManagerV2 keeps the attach lock through the adapter call when a detach queues", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const issuing = yield* Deferred.make(); + const issueGate = yield* Deferred.make(); + const pauseIssue = yield* Ref.make(false); + const adapterCalls = yield* Ref.make(0); + const callsWithClaim = yield* Ref.make(0); + const allocatorSlot = yield* Ref.make(undefined); + const sessionIdSlot = yield* Ref.make(undefined); + const sharedThreadId = ThreadId.make("thread-detach-gap-shared"); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (input) => + Ref.get(pauseIssue).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(issuing, undefined).pipe( + Effect.andThen(Deferred.await(issueGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.issue(input)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + yield* Ref.set(allocatorSlot, yield* IdAllocatorV2); + const ownerThreadId = ThreadId.make("thread-detach-gap-owner"); + const owner = yield* makeThreadSessionFixture(ownerThreadId); + yield* makeThreadSessionFixture(sharedThreadId); + const providerSessionId = yield* owner.allocate; + yield* Ref.set(sessionIdSlot, providerSessionId); + const runtime = yield* owner.open(providerSessionId); + yield* Ref.set(pauseIssue, true); + // The shared thread's attach suspends inside credential issuance with + // the [session, thread] lock held and its provisional attachment + // already recorded, so the terminal detach queues on that lock. + const attaching = yield* runtime + .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(issuing); + const detaching = yield* manager + .detach({ providerSessionId, threadId: sharedThreadId, revokeMcpCredential: true }) + .pipe(Effect.exit, Effect.forkChild); + // The queued detach cannot strip the attachment between the attach's + // bookkeeping and the adapter call: one lock hold spans both, so the + // adapter runs while the credential claim still exists and the + // teardown only lands after the call completes. + yield* Effect.forEach(Array.from({ length: 12 }), () => Effect.yieldNow, { + discard: true, + }); + yield* Deferred.succeed(issueGate, undefined); + const attachExit = yield* Fiber.join(attaching); + assert.equal(attachExit._tag, "Success"); + assert.equal(yield* Ref.get(adapterCalls), 1); + assert.equal(yield* Ref.get(callsWithClaim), 1); + const detachExit = yield* Fiber.join(detaching); + assert.equal(detachExit._tag, "Success"); + // The queued detach then tears the attachment down: the thread's + // credential is revoked and its config slot cleared. + assert.isUndefined(McpProviderSession.readMcpProviderSession(sharedThreadId)); + }).pipe( + Effect.ensuring(Deferred.succeed(issueGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpRegistryLayer, + ensureThread: (threadInput) => + Effect.gen(function* () { + yield* Ref.update(adapterCalls, (count) => count + 1); + if (McpProviderSession.readMcpProviderSession(threadInput.threadId) !== undefined) { + yield* Ref.update(callsWithClaim, (count) => count + 1); + } + return makeProviderThread({ + idAllocator: (yield* Ref.get(allocatorSlot))!, + threadId: threadInput.threadId, + providerSessionId: (yield* Ref.get(sessionIdSlot))!, + now: yield* DateTime.now, + }); + }), + }), + ), + ); + }), +); + it.effect( "ProviderSessionManagerV2 serializes credential revocation against a new attachment's reservation", () => @@ -7767,16 +7863,27 @@ it.effect( const mcpConfigs = yield* Ref.make< ReadonlyArray >([]); - const poisonOpen = yield* Ref.make(true); - const poisonClose = yield* Ref.make(true); + const poisonOpen = yield* Ref.make(false); + const poisonClose = yield* Ref.make(false); yield* Effect.gen(function* () { const manager = yield* ProviderSessionManagerV2; const registry = yield* McpSessionRegistry.McpSessionRegistry; const threadId = ThreadId.make("thread-open-startup-failed-cleanup"); const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const peerSessionId = yield* allocate; const providerSessionId = yield* allocate; const replacementId = yield* allocate; + // A live peer on the same thread records the thread's credential: the + // failed startup below reserves the same credential, and the peer's + // terminal detach must keep seeing the failed startup's claim. + yield* open(peerSessionId); + const issued = (yield* Ref.get(mcpConfigs)).at(-1); + const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(token); + + yield* Ref.set(poisonOpen, true); + yield* Ref.set(poisonClose, true); // openSession fails after registering its scope finalizers, and the // scope close itself defects: the startup cleanup cannot complete, so // the provider process and its credential may still be owned. @@ -7800,13 +7907,17 @@ it.effect( ); assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); - // The unclaimed credential was still swept even though the scope close - // failed: the record stays only for the provider resources. - const issued = (yield* Ref.get(mcpConfigs)).at(-1); - const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); - assert.isDefined(token); - assert.isUndefined(yield* registry.resolve(token!)); - assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); + // The failed startup keeps its credential reservation: its provider + // process may still own the credential, so the live peer's terminal + // detach — which sweeps the thread's configured credential — must + // leave the credential valid even though the thread's config slot is + // honestly released. + yield* manager.detach({ + providerSessionId: peerSessionId, + threadId, + revokeMcpCredential: true, + }); + assert.isDefined(yield* registry.resolve(token!)); }).pipe( Effect.provide( makeTestLayer({ diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index c50ed554f707..8bb7b4473fb4 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -1798,269 +1798,280 @@ export const layerWithOptions = ( return updated; }); - const ensureThreadAttached = (input: { + const attachLockSection = (input: { readonly providerSessionId: ProviderSessionId; readonly threadId: ThreadId; readonly providerInstanceId: ProviderInstanceId; readonly expectedRuntime: ProviderAdapterV2SessionRuntime; - }) => - threadAttach.withLock( - threadAttachKey(input.providerSessionId, input.expectedRuntime, input.threadId), - Effect.suspend(() => { - // Set synchronously when prepare reserves a credential, so a prepare - // that never returns (failed or interrupted mid-resolve/issue) can - // still have its in-flight reservation unwound. - let pendingMcpCredentialId: string | undefined; - // Whether the post-prepare step introduced the entry's credential - // claim; a claim recorded before this attach (a plain detach - // retains it for the still-live process) is not ours to remove. - let recordOwned = false; - // A retained credential this attach's record overwrote. Once it is - // out of the entry it has no remaining cleanup path, so the unwind - // must cover it if the normal sweep below is interrupted first. - let supersededMcpCredentialId: string | undefined; - let reservationDropped = false; - const dropReservation = () => { - if (!reservationDropped && pendingMcpCredentialId !== undefined) { - reservationDropped = true; - dropMcpCredentialReservation(input.threadId, pendingMcpCredentialId); - } - }; - const expectedRuntime = input.expectedRuntime; - // Entry identity rides on the runtime reference: bookkeeping updates - // replace the entry record but never the runtime, while a reopened - // same-id session always carries a new one. - const entryStillOwned = (current: ReadonlyMap) => { - const key = sessionKey(input.providerSessionId); - const entry = current.get(key); - return entry !== undefined && entry.runtime === expectedRuntime - ? entry - : ((releasing.has(key) ? "closing" : "released") as "closing" | "released"); - }; - // The guard in rejectPendingThreadAttachment ran before this - // caller queued on the attach lock and suspended in credential - // preparation — a dying startup that still owns this thread's - // resources may have been marked meanwhile, so every ownership - // recheck below must see it too. - const attachOwnership = (current: ReadonlyMap) => { - const owned = entryStillOwned(current); - return owned !== "closing" && - owned !== "released" && - findBlockingStartup(input.providerSessionId, input.threadId) !== undefined - ? ("closing" as const) - : owned; - }; - const abandonPrepared = ( - outcome: "closing" | "released", - ): Effect.Effect<"closing" | "released", ProviderSessionActivityError> => - Effect.gen(function* () { - // "closing"/"released" return normally, so the onExit unwind - // below never runs for them — the provisional attachment and - // any recorded credential claim must be removed here or a live - // entry keeps a thread whose attach was rejected. - yield* removeThreadAttachment({ ...input, expectedRuntime }); - yield* releasePrepared(); - return outcome; - }); - // Unwind a failed attach's credential bookkeeping while the entry may - // still be owned: drop our reservation and the claim this attempt - // recorded, then revoke only when no other holder claims the - // credential. - const releasePrepared = () => - Effect.suspend(() => { - // When prepare never reserved — the attach was interrupted - // while queued on mcpPrepareLock — the thread's configured - // credential may still need a claim recheck once the - // provisional attachment is gone. - const mcpCredentialId = - pendingMcpCredentialId ?? - McpProviderSession.readMcpProviderSession(input.threadId)?.providerSessionId; - dropReservation(); - const removeRecord = - recordOwned && mcpCredentialId !== undefined - ? Ref.update(sessions, (current) => { - const key = sessionKey(input.providerSessionId); - const entry = current.get(key); - if ( - entry === undefined || - entry.runtime !== expectedRuntime || - entry.mcpCredentialIdByThread.get(input.threadId) !== mcpCredentialId - ) { - return current; - } - const mcpCredentialIdByThread = new Map(entry.mcpCredentialIdByThread); - mcpCredentialIdByThread.delete(input.threadId); - const updated = new Map(current); - updated.set(key, { ...entry, mcpCredentialIdByThread }); - return updated; - }) - : Effect.void; - return removeRecord.pipe( - Effect.andThen( - mcpCredentialId === undefined - ? Effect.void - : releaseUnclaimedMcpCredential(input.threadId, mcpCredentialId), - ), - Effect.andThen( - supersededMcpCredentialId === undefined - ? Effect.void - : releaseUnclaimedMcpCredential(input.threadId, supersededMcpCredentialId), + }): Effect.Effect< + "closing" | "released" | "attached" | "alreadyAttached", + ProviderSessionActivityError + > => + Effect.suspend(() => { + // Set synchronously when prepare reserves a credential, so a prepare + // that never returns (failed or interrupted mid-resolve/issue) can + // still have its in-flight reservation unwound. + let pendingMcpCredentialId: string | undefined; + // Whether the post-prepare step introduced the entry's credential + // claim; a claim recorded before this attach (a plain detach + // retains it for the still-live process) is not ours to remove. + let recordOwned = false; + // A retained credential this attach's record overwrote. Once it is + // out of the entry it has no remaining cleanup path, so the unwind + // must cover it if the normal sweep below is interrupted first. + let supersededMcpCredentialId: string | undefined; + let reservationDropped = false; + const dropReservation = () => { + if (!reservationDropped && pendingMcpCredentialId !== undefined) { + reservationDropped = true; + dropMcpCredentialReservation(input.threadId, pendingMcpCredentialId); + } + }; + const expectedRuntime = input.expectedRuntime; + // Entry identity rides on the runtime reference: bookkeeping updates + // replace the entry record but never the runtime, while a reopened + // same-id session always carries a new one. + const entryStillOwned = (current: ReadonlyMap) => { + const key = sessionKey(input.providerSessionId); + const entry = current.get(key); + return entry !== undefined && entry.runtime === expectedRuntime + ? entry + : ((releasing.has(key) ? "closing" : "released") as "closing" | "released"); + }; + // The guard in rejectPendingThreadAttachment ran before this + // caller queued on the attach lock and suspended in credential + // preparation — a dying startup that still owns this thread's + // resources may have been marked meanwhile, so every ownership + // recheck below must see it too. + const attachOwnership = (current: ReadonlyMap) => { + const owned = entryStillOwned(current); + return owned !== "closing" && + owned !== "released" && + findBlockingStartup(input.providerSessionId, input.threadId) !== undefined + ? ("closing" as const) + : owned; + }; + const abandonPrepared = ( + outcome: "closing" | "released", + ): Effect.Effect<"closing" | "released", ProviderSessionActivityError> => + Effect.gen(function* () { + // "closing"/"released" return normally, so the onExit unwind + // below never runs for them — the provisional attachment and + // any recorded credential claim must be removed here or a live + // entry keeps a thread whose attach was rejected. + yield* removeThreadAttachment({ ...input, expectedRuntime }); + yield* releasePrepared(); + return outcome; + }); + // Unwind a failed attach's credential bookkeeping while the entry may + // still be owned: drop our reservation and the claim this attempt + // recorded, then revoke only when no other holder claims the + // credential. + const releasePrepared = () => + Effect.suspend(() => { + // When prepare never reserved — the attach was interrupted + // while queued on mcpPrepareLock — the thread's configured + // credential may still need a claim recheck once the + // provisional attachment is gone. + const mcpCredentialId = + pendingMcpCredentialId ?? + McpProviderSession.readMcpProviderSession(input.threadId)?.providerSessionId; + dropReservation(); + const removeRecord = + recordOwned && mcpCredentialId !== undefined + ? Ref.update(sessions, (current) => { + const key = sessionKey(input.providerSessionId); + const entry = current.get(key); + if ( + entry === undefined || + entry.runtime !== expectedRuntime || + entry.mcpCredentialIdByThread.get(input.threadId) !== mcpCredentialId + ) { + return current; + } + const mcpCredentialIdByThread = new Map(entry.mcpCredentialIdByThread); + mcpCredentialIdByThread.delete(input.threadId); + const updated = new Map(current); + updated.set(key, { ...entry, mcpCredentialIdByThread }); + return updated; + }) + : Effect.void; + return removeRecord.pipe( + Effect.andThen( + mcpCredentialId === undefined + ? Effect.void + : releaseUnclaimedMcpCredential(input.threadId, mcpCredentialId), + ), + Effect.andThen( + supersededMcpCredentialId === undefined + ? Effect.void + : releaseUnclaimedMcpCredential(input.threadId, supersededMcpCredentialId), + ), + ); + }); + return Effect.gen(function* () { + const attach = yield* attachThread(input); + if (attach.outcome === "closing" || attach.outcome === "released") { + return attach.outcome; + } + const attached = attach.outcome; + if (attached === "attached") { + const prepared = yield* prepareMcpSession( + input.threadId, + input.providerInstanceId, + (mcpCredentialId) => { + pendingMcpCredentialId = mcpCredentialId; + }, + ); + // Re-check ownership after prepare suspends: a release may have + // claimed the session in between. Recording the credential and + // deciding happen in the same Ref.modify. + const postPrepare = yield* Ref.modify( + sessions, + ( + current, + ): readonly [ + ( + | "closing" + | "released" + | { + readonly entry: LiveSessionEntry; + readonly priorMcpCredentialId: string | undefined; + } ), - ); - }); - return Effect.gen(function* () { - const attach = yield* attachThread(input); - if (attach.outcome === "closing" || attach.outcome === "released") { - return attach.outcome; + Map, + ] => { + const key = sessionKey(input.providerSessionId); + const owned = attachOwnership(current); + if (owned === "closing" || owned === "released") { + return [owned, current]; + } + const entry = owned; + if (prepared.mcpCredentialId === undefined) { + return [{ entry, priorMcpCredentialId: undefined } as const, current]; + } + const priorMcpCredentialId = entry.mcpCredentialIdByThread.get(input.threadId); + recordOwned = priorMcpCredentialId !== prepared.mcpCredentialId; + supersededMcpCredentialId = + priorMcpCredentialId !== undefined && + priorMcpCredentialId !== prepared.mcpCredentialId + ? priorMcpCredentialId + : undefined; + const mcpCredentialIdByThread = new Map(entry.mcpCredentialIdByThread); + mcpCredentialIdByThread.set(input.threadId, prepared.mcpCredentialId); + const updated = new Map(current); + updated.set(key, { ...entry, mcpCredentialIdByThread }); + return [{ entry, priorMcpCredentialId } as const, updated]; + }, + ); + if (postPrepare === "closing" || postPrepare === "released") { + return yield* abandonPrepared(postPrepare); } - const attached = attach.outcome; - if (attached === "attached") { - const prepared = yield* prepareMcpSession( - input.threadId, - input.providerInstanceId, - (mcpCredentialId) => { - pendingMcpCredentialId = mcpCredentialId; - }, - ); - // Re-check ownership after prepare suspends: a release may have - // claimed the session in between. Recording the credential and - // deciding happen in the same Ref.modify. - const postPrepare = yield* Ref.modify( - sessions, - ( - current, - ): readonly [ - ( - | "closing" - | "released" - | { - readonly entry: LiveSessionEntry; - readonly priorMcpCredentialId: string | undefined; + const superseded = postPrepare.priorMcpCredentialId; + if ( + superseded !== undefined && + superseded !== prepared.mcpCredentialId && + superseded !== pendingMcpCredentialId + ) { + // A retained record from an earlier attach was just + // overwritten: revoke the superseded credential once nothing + // else claims it, or it lives on unclaimed. + yield* releaseUnclaimedMcpCredential(input.threadId, superseded); + } + // The release path reports terminal status under releaseStatus, + // so checking ownership and writing the attach event inside the + // same lock orders this event strictly before any stopped/error + // write: a release that claimed the session first is observed + // here and the late write is skipped instead of resurrecting the + // persisted live status. + const written = yield* releaseStatus + .withLock( + input.providerSessionId, + Ref.get(sessions).pipe( + Effect.flatMap( + ( + current, + ): Effect.Effect< + LiveSessionEntry | "closing" | "released", + ProviderSessionActivityError + > => { + const owned = attachOwnership(current); + if (owned === "closing" || owned === "released") { + return Effect.succeed(owned); } + return withActivityError( + input.providerSessionId, + writeProviderSessionEvents({ + runtime: owned.runtime, + threadIds: [input.threadId], + type: "provider-session.attached", + payload: owned.runtime.providerSession, + }), + ).pipe(Effect.as(owned)); + }, ), - Map, - ] => { - const key = sessionKey(input.providerSessionId); - const owned = attachOwnership(current); - if (owned === "closing" || owned === "released") { - return [owned, current]; - } - const entry = owned; - if (prepared.mcpCredentialId === undefined) { - return [{ entry, priorMcpCredentialId: undefined } as const, current]; - } - const priorMcpCredentialId = entry.mcpCredentialIdByThread.get(input.threadId); - recordOwned = priorMcpCredentialId !== prepared.mcpCredentialId; - supersededMcpCredentialId = - priorMcpCredentialId !== undefined && - priorMcpCredentialId !== prepared.mcpCredentialId - ? priorMcpCredentialId - : undefined; - const mcpCredentialIdByThread = new Map(entry.mcpCredentialIdByThread); - mcpCredentialIdByThread.set(input.threadId, prepared.mcpCredentialId); - const updated = new Map(current); - updated.set(key, { ...entry, mcpCredentialIdByThread }); - return [{ entry, priorMcpCredentialId } as const, updated]; - }, - ); - if (postPrepare === "closing" || postPrepare === "released") { - return yield* abandonPrepared(postPrepare); - } - const superseded = postPrepare.priorMcpCredentialId; - if ( - superseded !== undefined && - superseded !== prepared.mcpCredentialId && - superseded !== pendingMcpCredentialId - ) { - // A retained record from an earlier attach was just - // overwritten: revoke the superseded credential once nothing - // else claims it, or it lives on unclaimed. - yield* releaseUnclaimedMcpCredential(input.threadId, superseded); - } - // The release path reports terminal status under releaseStatus, - // so checking ownership and writing the attach event inside the - // same lock orders this event strictly before any stopped/error - // write: a release that claimed the session first is observed - // here and the late write is skipped instead of resurrecting the - // persisted live status. - const written = yield* releaseStatus - .withLock( - input.providerSessionId, + ), + ) + .pipe( + // A failed write skips the post-write recheck, so run it on + // the error path too: a release that landed mid-write must + // still reject the caller instead of returning a closed + // adapter. + Effect.catch((writeError) => Ref.get(sessions).pipe( - Effect.flatMap( - ( - current, - ): Effect.Effect< - LiveSessionEntry | "closing" | "released", - ProviderSessionActivityError - > => { - const owned = attachOwnership(current); - if (owned === "closing" || owned === "released") { - return Effect.succeed(owned); - } - return withActivityError( - input.providerSessionId, - writeProviderSessionEvents({ - runtime: owned.runtime, - threadIds: [input.threadId], - type: "provider-session.attached", - payload: owned.runtime.providerSession, - }), - ).pipe(Effect.as(owned)); - }, - ), - ), - ) - .pipe( - // A failed write skips the post-write recheck, so run it on - // the error path too: a release that landed mid-write must - // still reject the caller instead of returning a closed - // adapter. - Effect.catch((writeError) => - Ref.get(sessions).pipe( - Effect.flatMap((current) => { - const owned = attachOwnership(current); - return owned === "closing" || owned === "released" - ? Effect.succeed(owned) - : Effect.fail(writeError); - }), - ), + Effect.flatMap((current) => { + const owned = attachOwnership(current); + return owned === "closing" || owned === "released" + ? Effect.succeed(owned) + : Effect.fail(writeError); + }), ), - ); - if (written === "closing" || written === "released") { - return yield* abandonPrepared(written); - } - // Removal from `sessions` does not take the releaseStatus lock, - // so a release may still have claimed the session between the - // in-lock check and now. Re-check before reporting success. - const postWrite = yield* Ref.modify(sessions, (current) => [ - attachOwnership(current), - current, - ]); - if (postWrite === "closing" || postWrite === "released") { - return yield* abandonPrepared(postWrite); - } + ), + ); + if (written === "closing" || written === "released") { + return yield* abandonPrepared(written); } - return attached; - }).pipe( - // Unwind on any non-success exit: typed failures, defects, and - // interruption all must drop the provisional thread attachment — - // while it stays in the entry it claims the credential, which - // keeps a token alive that no live holder recorded — and then - // release the in-flight credential bookkeeping. - Effect.onExit((exit) => - Exit.isSuccess(exit) - ? Effect.void - : removeThreadAttachment({ ...input, expectedRuntime }).pipe( - Effect.andThen(releasePrepared()), - ), - ), - // The entry's own record (written above while the thread is - // attached) guards the credential from here on; the reservation - // is only needed until then. Ensuring covers defects/interrupts. - Effect.ensuring(Effect.sync(dropReservation)), - ); - }), + // Removal from `sessions` does not take the releaseStatus lock, + // so a release may still have claimed the session between the + // in-lock check and now. Re-check before reporting success. + const postWrite = yield* Ref.modify(sessions, (current) => [ + attachOwnership(current), + current, + ]); + if (postWrite === "closing" || postWrite === "released") { + return yield* abandonPrepared(postWrite); + } + } + return attached; + }).pipe( + // Unwind on any non-success exit: typed failures, defects, and + // interruption all must drop the provisional thread attachment — + // while it stays in the entry it claims the credential, which + // keeps a token alive that no live holder recorded — and then + // release the in-flight credential bookkeeping. + Effect.onExit((exit) => + Exit.isSuccess(exit) + ? Effect.void + : removeThreadAttachment({ ...input, expectedRuntime }).pipe( + Effect.andThen(releasePrepared()), + ), + ), + // The entry's own record (written above while the thread is + // attached) guards the credential from here on; the reservation + // is only needed until then. Ensuring covers defects/interrupts. + Effect.ensuring(Effect.sync(dropReservation)), + ); + }); + + const ensureThreadAttached = (input: { + readonly providerSessionId: ProviderSessionId; + readonly threadId: ThreadId; + readonly providerInstanceId: ProviderInstanceId; + readonly expectedRuntime: ProviderAdapterV2SessionRuntime; + }) => + threadAttach.withLock( + threadAttachKey(input.providerSessionId, input.expectedRuntime, input.threadId), + attachLockSection(input), ); const markBusy = ( @@ -2143,13 +2154,21 @@ export const layerWithOptions = ( // pending-release check runs twice: the gate rejects attachments whose // thread a pending release recorded, and attachThread atomically rejects // a session that releaseEntry claimed between the gate and the attach. - const attachThreadOrReject = (input: { - readonly providerSessionId: ProviderSessionId; - readonly threadId: ThreadId; - readonly providerInstanceId: ProviderInstanceId; - readonly driver: ProviderDriverKind; - readonly expectedRuntime: ProviderAdapterV2SessionRuntime; - }) => + const attachThreadOrReject = ( + input: { + readonly providerSessionId: ProviderSessionId; + readonly threadId: ThreadId; + readonly providerInstanceId: ProviderInstanceId; + readonly driver: ProviderDriverKind; + readonly expectedRuntime: ProviderAdapterV2SessionRuntime; + }, + // The follow-up runs inside the same [session, thread] lock hold as + // the attach bookkeeping: releasing in between let a terminal detach + // strip the attachment and revoke its credential claim before the + // adapter call was admitted, so the op could create provider resources + // the teardown already passed. + runAttached: (attached: "attached" | "alreadyAttached") => Effect.Effect, + ): Effect.Effect => Effect.andThen( rejectPendingThreadAttachment({ providerSessionId: input.providerSessionId, @@ -2157,57 +2176,67 @@ export const layerWithOptions = ( driver: input.driver, }), Effect.gen(function* () { - const attached = yield* ensureThreadAttached({ - providerSessionId: input.providerSessionId, - threadId: input.threadId, - providerInstanceId: input.providerInstanceId, - expectedRuntime: input.expectedRuntime, - }).pipe( - Effect.catch((error) => - Ref.get(sessions).pipe( - Effect.flatMap( - ( - current, - ): Effect.Effect<"closing" | "released", ProviderAdapterProtocolError> => { - const key = sessionKey(input.providerSessionId); - const entry = current.get(key); - if (entry === undefined || entry.runtime !== input.expectedRuntime) { - // The attach failed because the session was claimed - // mid-flight — report the lifecycle outcome. - return Effect.succeed( - releasing.has(key) ? ("closing" as const) : ("released" as const), - ); - } - // The session is still owned but the attach bookkeeping - // failed: proceeding would invoke the adapter for an - // untracked attachment missing its prepared credential. - return Effect.logWarning("orchestration-v2.driver-session.attach-failed", { - providerSessionId: input.providerSessionId, - error, - }).pipe( - Effect.andThen( - Effect.fail( - new ProviderAdapterProtocolError({ - driver: input.driver, - detail: "The provider session could not attach the thread.", - }), + const outcome = yield* threadAttach.withLock( + threadAttachKey(input.providerSessionId, input.expectedRuntime, input.threadId), + attachLockSection(input).pipe( + Effect.catch((error) => + Ref.get(sessions).pipe( + Effect.flatMap( + ( + current, + ): Effect.Effect<"closing" | "released", ProviderAdapterProtocolError> => { + const key = sessionKey(input.providerSessionId); + const entry = current.get(key); + if (entry === undefined || entry.runtime !== input.expectedRuntime) { + // The attach failed because the session was claimed + // mid-flight — report the lifecycle outcome. + return Effect.succeed( + releasing.has(key) ? ("closing" as const) : ("released" as const), + ); + } + // The session is still owned but the attach bookkeeping + // failed: proceeding would invoke the adapter for an + // untracked attachment missing its prepared credential. + return Effect.logWarning("orchestration-v2.driver-session.attach-failed", { + providerSessionId: input.providerSessionId, + error, + }).pipe( + Effect.andThen( + Effect.fail( + new ProviderAdapterProtocolError({ + driver: input.driver, + detail: "The provider session could not attach the thread.", + }), + ), ), - ), - ); - }, + ); + }, + ), ), ), + Effect.flatMap( + ( + attached, + ): Effect.Effect< + { readonly attached: "attached" | "alreadyAttached"; readonly value: A }, + E | ProviderAdapterProtocolError, + R + > => + attached === "closing" || attached === "released" + ? Effect.fail( + new ProviderAdapterProtocolError({ + driver: input.driver, + detail: + attached === "closing" + ? "A previous provider session has not finished cleanup." + : "The provider session is no longer running.", + }), + ) + : Effect.map(runAttached(attached), (value) => ({ attached, value })), + ), ), ); - if (attached === "closing" || attached === "released") { - return yield* new ProviderAdapterProtocolError({ - driver: input.driver, - detail: - attached === "closing" - ? "A previous provider session has not finished cleanup." - : "The provider session is no longer running.", - }); - } + return outcome.value; }), ); @@ -2247,6 +2276,7 @@ export const layerWithOptions = ( readonly driver: ProviderDriverKind; readonly operation: Effect.Effect; readonly drainTimeout: Duration.Duration | undefined; + readonly requiredThreadId?: ThreadId; }): Effect.Effect => Effect.uninterruptibleMask((_restore) => Effect.gen(function* () { @@ -2255,22 +2285,36 @@ export const layerWithOptions = ( done: Deferred.makeUnsafe(), drainTimeout: input.drainTimeout, }; - const admitted = yield* Ref.modify(sessions, (current) => { + const admission = yield* Ref.modify(sessions, (current) => { const entry = current.get(key); if (entry === undefined || entry.runtime !== input.expectedRuntime) { - return [false, current] as const; + return ["missing" as const, current] as const; + } + // A terminal detach that won the [session, thread] lock between + // the attach's bookkeeping and this admission removed the thread + // while the shared runtime stays live — refuse rather than + // invoke the adapter for a detached thread whose credential + // claim was already revoked. + if ( + input.requiredThreadId !== undefined && + !entry.attachedThreadIds.has(input.requiredThreadId) + ) { + return ["detached" as const, current] as const; } const ops = inflightAdapterOps.get(key) ?? new Set(); ops.add(record); inflightAdapterOps.set(key, ops); - return [true, current] as const; + return ["admitted" as const, current] as const; }); - if (!admitted) { + if (admission !== "admitted") { return yield* new ProviderAdapterProtocolError({ driver: input.driver, - detail: releasing.has(key) - ? "A previous provider session has not finished cleanup." - : "The provider session is no longer running.", + detail: + admission === "detached" + ? "The thread is no longer attached to the provider session." + : releasing.has(key) + ? "A previous provider session has not finished cleanup." + : "The provider session is no longer running.", }); } // The operation runs uninterruptibly inside the mask: adapters @@ -2347,43 +2391,42 @@ export const layerWithOptions = ( subscribeEvents.pipe(Effect.map((subscription) => subscription.events)), ), ensureThread: (input) => - attachThreadOrReject({ - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, - driver: runtime.driver, - expectedRuntime: runtime, - }).pipe( - Effect.andThen( - // The adapter call holds the attach lock so a detach cannot - // strip this thread's bookkeeping mid-call, and it runs under - // an admission record a release drains before closing the - // session scope — a resource the adapter creates (e.g. - // Cursor's late runner.open result) is never installed after - // the finalizer that would have closed it already ran. - threadAttach.withLock( - threadAttachKey(providerSessionId, runtime, input.threadId), - runResourceCreatingAdapterOp({ - providerSessionId, - expectedRuntime: runtime, - driver: runtime.driver, - operation: runtime.ensureThread(input), - drainTimeout: undefined, - }), - ), - ), - Effect.tap((providerThread) => - markProviderThreadLoaded({ + attachThreadOrReject( + { + providerSessionId, + threadId: input.threadId, + providerInstanceId: runtime.instanceId, + driver: runtime.driver, + expectedRuntime: runtime, + }, + // The adapter call holds the attach lock so a detach cannot + // strip this thread's bookkeeping mid-call, and it runs under + // an admission record a release drains before closing the + // session scope — a resource the adapter creates (e.g. + // Cursor's late runner.open result) is never installed after + // the finalizer that would have closed it already ran. + () => + runResourceCreatingAdapterOp({ providerSessionId, expectedRuntime: runtime, - threadId: input.threadId, - providerThreadKey: providerThreadLoadKey({ - providerThread, - modelSelection: input.modelSelection, - runtimePolicy: input.runtimePolicy, - }), - }), - ), + driver: runtime.driver, + operation: runtime.ensureThread(input), + drainTimeout: undefined, + requiredThreadId: input.threadId, + }).pipe( + Effect.tap((providerThread) => + markProviderThreadLoaded({ + providerSessionId, + expectedRuntime: runtime, + threadId: input.threadId, + providerThreadKey: providerThreadLoadKey({ + providerThread, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + }), + }), + ), + ), ), resumeThread: (input) => { const threadId = input.threadId ?? input.providerThread.appThreadId; @@ -2409,124 +2452,116 @@ export const layerWithOptions = ( : { modelSelection: input.modelSelection }), ...(input.runtimePolicy === undefined ? {} : { runtimePolicy: input.runtimePolicy }), }); - return attachThreadOrReject({ - providerSessionId, - threadId, - providerInstanceId: runtime.instanceId, - driver: runtime.driver, - expectedRuntime: runtime, - }).pipe( - Effect.andThen( + return attachThreadOrReject( + { + providerSessionId, + threadId, + providerInstanceId: runtime.instanceId, + driver: runtime.driver, + expectedRuntime: runtime, + }, + () => isProviderThreadLoaded({ providerSessionId, threadId, providerThreadKey, expectedRuntime: runtime, - }), - ), - Effect.flatMap((loaded) => - loaded - ? Effect.succeed(input.providerThread) - : threadAttach.withLock( - threadAttachKey(providerSessionId, runtime, threadId), - runResourceCreatingAdapterOp({ - providerSessionId, - expectedRuntime: runtime, - driver: runtime.driver, - operation: runtime.resumeThread(input), - drainTimeout: undefined, + }).pipe( + Effect.flatMap((loaded) => + loaded + ? Effect.succeed(input.providerThread) + : runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.resumeThread(input), + drainTimeout: undefined, + requiredThreadId: threadId, + }), + ), + Effect.tap((providerThread) => + markProviderThreadLoaded({ + providerSessionId, + expectedRuntime: runtime, + threadId, + providerThreadKey: providerThreadLoadKey({ + providerThread, + ...(input.modelSelection === undefined + ? {} + : { modelSelection: input.modelSelection }), + ...(input.runtimePolicy === undefined + ? {} + : { runtimePolicy: input.runtimePolicy }), }), - ), - ), - Effect.tap((providerThread) => - markProviderThreadLoaded({ - providerSessionId, - expectedRuntime: runtime, - threadId, - providerThreadKey: providerThreadLoadKey({ - providerThread, - ...(input.modelSelection === undefined - ? {} - : { modelSelection: input.modelSelection }), - ...(input.runtimePolicy === undefined - ? {} - : { runtimePolicy: input.runtimePolicy }), - }), - }), - ), + }), + ), + ), ); }, forkThread: (input) => - attachThreadOrReject({ - providerSessionId, - threadId: input.targetThreadId, - providerInstanceId: runtime.instanceId, - driver: runtime.driver, - expectedRuntime: runtime, - }).pipe( + attachThreadOrReject( + { + providerSessionId, + threadId: input.targetThreadId, + providerInstanceId: runtime.instanceId, + driver: runtime.driver, + expectedRuntime: runtime, + }, // Fork acquires provider-side state for the target thread // (ACP session/fork may restart the runtime) — the same // acquire-then-return shape as ensure/resume, so it drains // under the same admission record. - Effect.andThen( - threadAttach.withLock( - threadAttachKey(providerSessionId, runtime, input.targetThreadId), - runResourceCreatingAdapterOp({ - providerSessionId, - expectedRuntime: runtime, - driver: runtime.driver, - operation: runtime.forkThread(input), - drainTimeout: undefined, - }), - ), - ), - Effect.tap((providerThread) => - markProviderThreadLoaded({ + () => + runResourceCreatingAdapterOp({ providerSessionId, expectedRuntime: runtime, - threadId: input.targetThreadId, - providerThreadKey: providerThreadLoadKey({ - providerThread, - ...(input.modelSelection === undefined - ? {} - : { modelSelection: input.modelSelection }), - ...(input.runtimePolicy === undefined - ? {} - : { runtimePolicy: input.runtimePolicy }), - }), - }), - ), + driver: runtime.driver, + operation: runtime.forkThread(input), + drainTimeout: undefined, + requiredThreadId: input.targetThreadId, + }).pipe( + Effect.tap((providerThread) => + markProviderThreadLoaded({ + providerSessionId, + expectedRuntime: runtime, + threadId: input.targetThreadId, + providerThreadKey: providerThreadLoadKey({ + providerThread, + ...(input.modelSelection === undefined + ? {} + : { modelSelection: input.modelSelection }), + ...(input.runtimePolicy === undefined + ? {} + : { runtimePolicy: input.runtimePolicy }), + }), + }), + ), + ), ), startTurn: (input) => - Effect.andThen( - attachThreadOrReject({ - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, - driver: runtime.driver, - expectedRuntime: runtime, - }), - observeActivity(providerSessionId, markBusy(providerSessionId, runtime)).pipe( - // markBusy can suspend waiting on the idle fiber; a close may - // have claimed the session meanwhile — re-check before - // invoking the adapter. - Effect.andThen( - requireLiveRuntime({ + // markBusy runs before the attach rather than between it and the + // adapter call: it can suspend waiting on the idle fiber, and the + // attach lock must not be held across that wait (an open already + // holding threadLifecycle would deadlock on it). The attach's own + // gates re-check ownership after the suspend. + observeActivity(providerSessionId, markBusy(providerSessionId, runtime)).pipe( + Effect.andThen( + attachThreadOrReject( + { providerSessionId, - expectedRuntime: runtime, + threadId: input.threadId, + providerInstanceId: runtime.instanceId, driver: runtime.driver, - }), - ), - // The adapter call returns once the turn is started — most - // adapters fork the turn's lifetime into the session scope — - // so the admission record spans only the acquisition window - // (Cursor's openAgent/runner.open, ACP session activate+prompt - // submit). Only in-band turn calls get the bounded drain — - // abandoning an acquire-then-return call could strand a - // resource the scope close already finished checking for. - Effect.andThen( - threadAttach.withLock( - threadAttachKey(providerSessionId, runtime, input.threadId), + expectedRuntime: runtime, + }, + // The adapter call returns once the turn is started — most + // adapters fork the turn's lifetime into the session scope — + // so the admission record spans only the acquisition window + // (Cursor's openAgent/runner.open, ACP session activate+prompt + // submit). Only in-band turn calls get the bounded drain — + // abandoning an acquire-then-return call could strand a + // resource the scope close already finished checking for. + () => runResourceCreatingAdapterOp({ providerSessionId, expectedRuntime: runtime, @@ -2535,13 +2570,13 @@ export const layerWithOptions = ( drainTimeout: INBAND_TURN_CALL_DRIVERS.has(runtime.driver) ? Duration.millis(ADAPTER_OP_DRAIN_TIMEOUT_MS) : undefined, + requiredThreadId: input.threadId, }), - ), ), - Effect.catch((error) => - observeActivity(providerSessionId, markIdle(providerSessionId, runtime)).pipe( - Effect.andThen(Effect.fail(error)), - ), + ), + Effect.catch((error) => + observeActivity(providerSessionId, markIdle(providerSessionId, runtime)).pipe( + Effect.andThen(Effect.fail(error)), ), ), ), @@ -2610,28 +2645,22 @@ export const layerWithOptions = ( ? {} : { compactThread: (input: Parameters>[0]) => - Effect.andThen( - attachThreadOrReject({ - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, - driver: runtime.driver, - expectedRuntime: runtime, - }), - observeActivity(providerSessionId, markBusy(providerSessionId, runtime)).pipe( - Effect.andThen( - requireLiveRuntime({ + // Same markBusy-before-attach ordering as startTurn: the + // wait on the idle fiber must not run under the attach lock. + observeActivity(providerSessionId, markBusy(providerSessionId, runtime)).pipe( + Effect.andThen( + attachThreadOrReject( + { providerSessionId, - expectedRuntime: runtime, + threadId: input.threadId, + providerInstanceId: runtime.instanceId, driver: runtime.driver, - }), - ), - // Compaction delegates to the same acquire-then-start - // path as startTurn, so it shares the admission record - // and the same driver-keyed drain policy. - Effect.andThen( - threadAttach.withLock( - threadAttachKey(providerSessionId, runtime, input.threadId), + expectedRuntime: runtime, + }, + // Compaction delegates to the same acquire-then-start + // path as startTurn, so it shares the admission record + // and the same driver-keyed drain policy. + () => runResourceCreatingAdapterOp({ providerSessionId, expectedRuntime: runtime, @@ -2640,14 +2669,13 @@ export const layerWithOptions = ( drainTimeout: INBAND_TURN_CALL_DRIVERS.has(runtime.driver) ? Duration.millis(ADAPTER_OP_DRAIN_TIMEOUT_MS) : undefined, + requiredThreadId: input.threadId, }), - ), ), - Effect.catch((error) => - observeActivity( - providerSessionId, - markIdle(providerSessionId, runtime), - ).pipe(Effect.andThen(Effect.fail(error))), + ), + Effect.catch((error) => + observeActivity(providerSessionId, markIdle(providerSessionId, runtime)).pipe( + Effect.andThen(Effect.fail(error)), ), ), ), @@ -3379,14 +3407,19 @@ export const layerWithOptions = ( const closeExit = yield* Scope.close(sessionScope, Exit.void).pipe( Effect.exit, ); - // The reservation outlives the scope close: the - // provider process can still present the credential - // while it is shutting down, so a peer's terminal - // detach must keep seeing this claim until the close - // settles. - dropReservationNow(); + // The reservation outlives a successful scope + // close: the provider process can still present + // the credential while it is shutting down, so a + // peer's terminal detach must keep seeing this + // claim until the close settles. A failed close + // keeps the reservation — and skips the sweep — + // because the provider may still own the + // credential. + if (Exit.isSuccess(closeExit)) { + dropReservationNow(); + } const sweepExit = - pendingMcpCredentialId === undefined + Exit.isFailure(closeExit) || pendingMcpCredentialId === undefined ? Exit.void : yield* releaseUnclaimedMcpCredential( input.threadId, diff --git a/apps/server/src/provider/Layers/ProviderAuthService.test.ts b/apps/server/src/provider/Layers/ProviderAuthService.test.ts index 2a29c46c5dee..340c03eae2e2 100644 --- a/apps/server/src/provider/Layers/ProviderAuthService.test.ts +++ b/apps/server/src/provider/Layers/ProviderAuthService.test.ts @@ -4,13 +4,8 @@ import { ProviderInstanceId, ProviderSessionId, ProviderSetupError, - ProviderThreadId, - ThreadId, - type OrchestrationV2ProviderSession, - type OrchestrationV2ThreadShell, type ProviderAuthState, } from "@t3tools/contracts"; -import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; @@ -21,14 +16,9 @@ import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; import { - ProjectionStoreReadError, - ProjectionStoreV2, -} from "../../orchestration-v2/ProjectionStore.ts"; -import { + ProviderSessionCloseError, ProviderSessionManagerV2, - ProviderSessionReleaseError, } from "../../orchestration-v2/ProviderSessionManager.ts"; -import { AcpProviderCapabilitiesV2 } from "../../orchestration-v2/Adapters/AcpAdapterV2.ts"; import type { ProviderInstance } from "../ProviderDriver.ts"; import type { ProviderAuthController } from "../Services/ProviderAuthService.ts"; import { ProviderInstanceRegistry } from "../Services/ProviderInstanceRegistry.ts"; @@ -42,7 +32,6 @@ const owner = "paired-client-owner"; const otherOwner = "paired-client-other"; const flowId = "test-sign-in-flow"; const callbackUrl = "http://127.0.0.1:48123/?state=test-state&code=test-code"; -const now = "2026-09-02T00:00:00.000Z"; const idleAuthState: ProviderAuthState = { instanceId, phase: "idle", @@ -81,81 +70,25 @@ function makeInstance(input: { }; } -const nowUtc = DateTime.makeUnsafe(now); - -/** - * A thread bound to a provider instance with (or without) a live native - * thread. The auth service reads only the routing fields, so the rest of the - * shell row is left out. - */ -function makeThread( - thread: string, - input: { readonly providerInstanceId?: ProviderInstanceId; readonly active?: boolean } = {}, -): OrchestrationV2ThreadShell { - const shell: Pick< - OrchestrationV2ThreadShell, - "id" | "providerInstanceId" | "activeProviderThreadId" - > = { - id: ThreadId.make(thread), - providerInstanceId: input.providerInstanceId ?? instanceId, - activeProviderThreadId: - input.active === false ? null : ProviderThreadId.make(`${thread}:native`), - }; - return shell as unknown as OrchestrationV2ThreadShell; -} - -function makeSession( - session: string, - status: OrchestrationV2ProviderSession["status"] = "ready", - providerInstanceId = instanceId, -): OrchestrationV2ProviderSession { - return { - id: ProviderSessionId.make(session), - driver: driverKind, - providerInstanceId, - status, - cwd: "/workspace", - model: null, - capabilities: AcpProviderCapabilitiesV2, - createdAt: nowUtc, - updatedAt: nowUtc, - lastError: null, - }; -} - -function sessionThreads(list: ReadonlyArray) { - return { - threads: list.map((session) => makeThread(session.id)), - sessions: new Map(list.map((session) => [ThreadId.make(session.id), [session]])), - }; -} - const makeHarness = Effect.fn("ProviderAuthService.test.makeHarness")(function* ( input: { enabled?: boolean; - threads?: ReadonlyArray; - sessions?: ReadonlyMap>; - shellError?: boolean; - stopError?: boolean; + stopFailures?: number; logoutError?: ProviderSetupError; sharedCredentials?: boolean; sharedBusy?: boolean; sharedBusyEffect?: Effect.Effect; responds?: boolean; beforeLogout?: Effect.Effect; - beforeStop?: Effect.Effect; + beforeStop?: (id: ProviderInstanceId) => Effect.Effect; beforeListSessions?: Effect.Effect; onLookup?: Effect.Effect; - onListThreads?: (instances: ProviderInstance[]) => void; } = {}, ) { const actions: string[] = []; const registryChanges = yield* PubSub.unbounded(); - const threads = input.threads ?? []; - const sessions = new Map( - [...(input.sessions?.entries() ?? [])].map(([threadId, list]) => [threadId, [...list]]), - ); - const released: string[] = []; + const closedInstances: string[] = []; + let closeAttempts = 0; const idle = idleAuthState; let state = idle; let flowOwner: string | undefined; @@ -259,54 +192,31 @@ const makeHarness = Effect.fn("ProviderAuthService.test.makeHarness")(function* yield* input.onLookup ?? Effect.void; return found; }), - listInstances: Effect.succeed(instances), + listInstances: Effect.gen(function* () { + yield* input.beforeListSessions ?? Effect.void; + return instances; + }), subscribeChanges: PubSub.subscribe(registryChanges), }), - Layer.mock(ProjectionStoreV2)({ - getRecoveryThreadIds: () => - Effect.gen(function* () { - assert.isTrue(gateClosed); - actions.push("list-threads"); - input.onListThreads?.(instances); - yield* input.beforeListSessions ?? Effect.void; - return yield* input.shellError - ? Effect.fail( - new ProjectionStoreReadError({ - threadId: ThreadId.make("shell"), - cause: new Error("private database diagnostics"), - }), - ) - : Effect.succeed(threads.map((thread) => thread.id)); - }), - getThreadRecords: (threadId) => - Effect.sync(() => { - actions.push(`sessions:${threadId}`); - return { - providerSessions: sessions.get(threadId) ?? [], - } as never; - }), - }), Layer.mock(ProviderSessionManagerV2)({ - release: ({ providerSessionId, reason }) => + closeInstance: (id) => Effect.gen(function* () { assert.isTrue(gateClosed); - actions.push(`stop:${providerSessionId}`); - yield* input.beforeStop ?? Effect.void; - if (input.stopError) { + actions.push(`close-instance:${id}`); + yield* input.beforeStop?.(id) ?? Effect.void; + closeAttempts += 1; + if (closeAttempts <= (input.stopFailures ?? 0)) { return yield* Effect.fail( - new ProviderSessionReleaseError({ - providerSessionId, - reason, + new ProviderSessionCloseError({ + providerSessionId: ProviderSessionId.make( + `provider-session:provider-instance:${id}`, + ), cause: new Error("private process diagnostics"), }), ); } - released.push(providerSessionId); - for (const [threadId, list] of sessions) { - const remaining = list.filter((session) => session.id !== providerSessionId); - if (remaining.length === 0) sessions.delete(threadId); - else sessions.set(threadId, remaining); - } + closedInstances.push(id); + }), }), ), @@ -315,8 +225,7 @@ const makeHarness = Effect.fn("ProviderAuthService.test.makeHarness")(function* return { service, actions, - released, - sessions, + closedInstances, auth, addInstance: (instance: ProviderInstance) => instances.push(instance), replaceInstance: (replacement: ProviderInstance) => { @@ -386,7 +295,6 @@ const makeSubscriptionHarness = Effect.fn("ProviderAuthService.test.makeSubscrip return instance; }), }), - Layer.mock(ProjectionStoreV2)({}), Layer.mock(ProviderSessionManagerV2)({}), ), ), @@ -417,17 +325,12 @@ describe("ProviderAuthService", () => { "stops sessions sharing credentials and invalidates their processes before logout", () => Effect.gen(function* () { - const { service, actions, sessions } = yield* makeHarness({ + const { service, actions, closedInstances } = yield* makeHarness({ sharedCredentials: true, - ...sessionThreads([ - makeSession("target"), - makeSession("shared", "ready", otherInstanceId), - makeSession("unrelated", "ready", unsupportedInstanceId), - ]), }); yield* service.logout({ instanceId }); - assert.deepStrictEqual([...sessions.keys()], [ThreadId.make("unrelated")]); - assert.isBelow(actions.indexOf("stop:shared"), actions.indexOf("invalidate-shared")); + assert.deepStrictEqual(closedInstances, [instanceId, otherInstanceId]); + assert.isBelow(actions.indexOf(`close-instance:${otherInstanceId}`), actions.indexOf("invalidate-shared")); assert.isBelow(actions.indexOf("invalidate-shared"), actions.indexOf("native-logout")); }), ); @@ -463,23 +366,19 @@ describe("ProviderAuthService", () => { assert.include(error.detail, "does not accept"); }), ); - it.effect("stops routed sessions before sign-in, including for a disabled instance", () => + it.effect("closes the instance before sign-in, including for a disabled instance", () => Effect.gen(function* () { - const { service, actions, released } = yield* makeHarness({ + const { service, actions, closedInstances } = yield* makeHarness({ enabled: false, - threads: [makeThread("active")], - sessions: new Map([[ThreadId.make("active"), [makeSession("active-session")]]]), }); const state = yield* service.start({ instanceId }, owner); assert.strictEqual(state.instanceId, instanceId); assert.strictEqual(state.phase, "waiting"); - assert.deepStrictEqual(released, ["active-session"]); + assert.deepStrictEqual(closedInstances, [instanceId]); assert.deepStrictEqual(actions, [ "close-gate", - "list-threads", - "sessions:active", - "stop:active-session", + `close-instance:${instanceId}`, "start-sign-in", ]); }), @@ -633,71 +532,62 @@ describe("ProviderAuthService", () => { }), ); - it.effect("stops every live session on the instance before native logout", () => + it.effect("closes the instance before native logout", () => Effect.gen(function* () { - const { service, actions, released } = yield* makeHarness({ - threads: [ - makeThread("running"), - makeThread("starting"), - makeThread("idle", { active: false }), - makeThread("other-instance", { providerInstanceId: otherInstanceId }), - ], - sessions: new Map([ - [ - ThreadId.make("running"), - [ - makeSession("running-session", "running"), - makeSession("stopped-session", "stopped"), - makeSession("errored-session", "error"), - makeSession("foreign-session", "ready", otherInstanceId), - ], - ], - [ThreadId.make("starting"), [makeSession("starting-session", "starting")]], - [ThreadId.make("idle"), [makeSession("idle-session")]], - [ - ThreadId.make("other-instance"), - [makeSession("other-session", "ready", otherInstanceId)], - ], - ]), - }); + const { service, actions, closedInstances } = yield* makeHarness(); const state = yield* service.logout({ instanceId }); assert.strictEqual(state.phase, "idle"); assert.deepStrictEqual(actions, [ "close-gate", - "list-threads", - "sessions:running", - "stop:running-session", - "sessions:starting", - "stop:starting-session", - "sessions:idle", - "stop:idle-session", - "sessions:other-instance", + `close-instance:${instanceId}`, "native-logout", ]); - assert.deepStrictEqual(released, ["running-session", "starting-session", "idle-session"]); + assert.deepStrictEqual(closedInstances, [instanceId]); }), ); it.effect( - "releases a shared native session once after a thread's selected provider changes", + "retries instance cleanup on a later logout instead of signing out over owned resources", () => Effect.gen(function* () { - const shared = makeSession("shared-native-session"); - const { service, released } = yield* makeHarness({ - threads: [ - makeThread("changed-selection", { providerInstanceId: otherInstanceId }), - makeThread("same-session"), - ], - sessions: new Map([ - [ThreadId.make("changed-selection"), [shared]], - [ThreadId.make("same-session"), [shared]], - ]), - }); + const { service, actions, closedInstances } = yield* makeHarness({ stopFailures: 1 }); + const error = yield* Effect.flip(service.logout({ instanceId })); - yield* service.logout({ instanceId }); + assert.instanceOf(error, ProviderSetupError); + assert.strictEqual(error.operation, "stopSessions"); + assert.notInclude(error.detail, "private process diagnostics"); + assert.deepStrictEqual(closedInstances, []); + + const state = yield* service.logout({ instanceId }); + + assert.strictEqual(state.phase, "idle"); + assert.deepStrictEqual(closedInstances, [instanceId]); + assert.deepStrictEqual(actions, [ + "close-gate", + `close-instance:${instanceId}`, + "close-gate", + `close-instance:${instanceId}`, + "native-logout", + ]); + }), + ); + + it.effect( + "retries instance cleanup on a later sign-in instead of signing in over owned resources", + () => + Effect.gen(function* () { + const { service, closedInstances } = yield* makeHarness({ stopFailures: 1 }); + const error = yield* Effect.flip(service.start({ instanceId }, owner)); + + assert.instanceOf(error, ProviderSetupError); + assert.strictEqual(error.operation, "stopSessions"); + assert.deepStrictEqual(closedInstances, []); - assert.deepStrictEqual(released, [shared.id]); + const state = yield* service.start({ instanceId }, owner); + + assert.strictEqual(state.phase, "waiting"); + assert.deepStrictEqual(closedInstances, [instanceId]); }), ); @@ -716,7 +606,7 @@ describe("ProviderAuthService", () => { assert.strictEqual(result, handled); assert.deepStrictEqual( actions, - handled ? ["close-gate", "list-threads", "native-logout"] : [], + handled ? ["close-gate", `close-instance:${instanceId}`, "native-logout"] : [], ); }), ); @@ -737,25 +627,26 @@ describe("ProviderAuthService", () => { }), ); - it.effect("does not log out when the thread shell cannot be read", () => + it.effect("does not log out when the instance close fails", () => Effect.gen(function* () { - const { service, actions } = yield* makeHarness({ shellError: true }); + const { service, actions, closedInstances } = yield* makeHarness({ + stopFailures: Number.POSITIVE_INFINITY, + }); const error = yield* Effect.flip(service.logout({ instanceId })); assert.instanceOf(error, ProviderSetupError); assert.strictEqual(error.instanceId, instanceId); assert.strictEqual(error.operation, "stopSessions"); - assert.notInclude(error.detail, "private database diagnostics"); - assert.deepStrictEqual(actions, ["close-gate", "list-threads"]); + assert.notInclude(error.detail, "private process diagnostics"); + assert.deepStrictEqual(closedInstances, []); + assert.deepStrictEqual(actions, ["close-gate", `close-instance:${instanceId}`]); }), ); - it.effect("does not log out or consume the command when stopping a session fails", () => + it.effect("does not log out or consume the command when the instance close fails", () => Effect.gen(function* () { - const { service, actions, released } = yield* makeHarness({ - threads: [makeThread("active")], - sessions: new Map([[ThreadId.make("active"), [makeSession("active-session")]]]), - stopError: true, + const { service, actions, closedInstances } = yield* makeHarness({ + stopFailures: Number.POSITIVE_INFINITY, }); const error = yield* Effect.flip( service.tryHandlePromptCommand({ instanceId, text: "/logout", hasAttachments: false }), @@ -764,13 +655,8 @@ describe("ProviderAuthService", () => { assert.instanceOf(error, ProviderSetupError); assert.strictEqual(error.operation, "stopSessions"); assert.notInclude(error.detail, "private process diagnostics"); - assert.deepStrictEqual(released, []); - assert.deepStrictEqual(actions, [ - "close-gate", - "list-threads", - "sessions:active", - "stop:active-session", - ]); + assert.deepStrictEqual(closedInstances, []); + assert.deepStrictEqual(actions, ["close-gate", `close-instance:${instanceId}`]); }), ); @@ -846,10 +732,6 @@ it.effect.each(["start", "logout", "prompt"] as const)( Effect.andThen(Deferred.await(continueCheck)), Effect.as(false), ), - ...sessionThreads([ - makeSession("old-shared", "ready", otherInstanceId), - makeSession("replacement-shared", "ready", replacementPeerId), - ]), }); harness.addInstance( makeInstance({ @@ -896,8 +778,8 @@ it.effect.each(["start", "logout", "prompt"] as const)( yield* Fiber.join(running); assert.equal(replacementMutations, 0); assert.include(harness.actions, action === "start" ? "start-sign-in" : "native-logout"); - assert.isFalse(harness.sessions.has(ThreadId.make("old-shared"))); - assert.isTrue(harness.sessions.has(ThreadId.make("replacement-shared"))); + assert.include(harness.closedInstances, otherInstanceId); + assert.notInclude(harness.closedInstances, replacementPeerId); const blocked = yield* Effect.flip(harness.service.logout({ instanceId })); assert.include(blocked.detail, "shared sign-in"); assert.equal(replacementMutations, 0); @@ -915,10 +797,9 @@ it.effect.each([ const continueDrain = yield* Deferred.make(); const harness = yield* makeHarness({ sharedCredentials: true, - ...sessionThreads([makeSession("shared-draining", "ready", otherInstanceId)]), - beforeStop: Deferred.succeed(draining, undefined).pipe( + beforeStop: (id) => id === otherInstanceId ? Deferred.succeed(draining, undefined).pipe( Effect.andThen(Deferred.await(continueDrain)), - ), + ) : Effect.void, }); const logout = yield* harness.service.logout({ instanceId }).pipe(Effect.forkChild); yield* Deferred.await(draining); @@ -939,7 +820,7 @@ it.effect.each([ yield* Deferred.succeed(continueDrain, undefined); yield* Fiber.join(logout); assert.isFalse(replacementInvalidated); - assert.isFalse(harness.sessions.has(ThreadId.make("shared-draining"))); + assert.include(harness.closedInstances, otherInstanceId); assert.include(harness.actions, "native-logout"); }).pipe(Effect.scoped), ); @@ -955,12 +836,7 @@ it.effect.each(["selection", "drain"] as const)( ); const harness = yield* makeHarness({ sharedCredentials: true, - ...sessionThreads([ - makeSession("target-draining"), - makeSession("peer-replacement", "ready", otherInstanceId), - makeSession("peer-persisted", "running", otherInstanceId), - ]), - ...(phase === "selection" ? { beforeListSessions: block } : { beforeStop: block }), + ...(phase === "selection" ? { beforeListSessions: block } : { beforeStop: () => block }), }); const logout = yield* harness.service.logout({ instanceId }).pipe(Effect.forkChild); yield* Deferred.await(entered); @@ -977,10 +853,8 @@ it.effect.each(["selection", "drain"] as const)( ); yield* Deferred.succeed(proceed, undefined); yield* Fiber.join(logout); - assert.isTrue(harness.sessions.has(ThreadId.make("peer-replacement"))); - assert.notInclude(harness.actions, "stop:peer-replacement"); - assert.notInclude(harness.actions, "stop:peer-persisted"); - assert.isFalse(harness.sessions.has(ThreadId.make("target-draining"))); + assert.notInclude(harness.closedInstances, otherInstanceId); + assert.include(harness.closedInstances, instanceId); assert.include(harness.actions, "native-logout"); }).pipe(Effect.scoped), ); diff --git a/apps/server/src/provider/Layers/ProviderAuthService.ts b/apps/server/src/provider/Layers/ProviderAuthService.ts index 5e08c671adac..380e395eff0d 100644 --- a/apps/server/src/provider/Layers/ProviderAuthService.ts +++ b/apps/server/src/provider/Layers/ProviderAuthService.ts @@ -1,7 +1,6 @@ import { ProviderSetupError, type ProviderInstanceId, - type ProviderSessionId, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -9,13 +8,11 @@ import * as Stream from "effect/Stream"; import * as Semaphore from "effect/Semaphore"; import { ProviderSessionManagerV2 } from "../../orchestration-v2/ProviderSessionManager.ts"; -import { ProjectionStoreV2 } from "../../orchestration-v2/ProjectionStore.ts"; import * as ProviderAuthService from "../Services/ProviderAuthService.ts"; import { ProviderInstanceRegistry } from "../Services/ProviderInstanceRegistry.ts"; export const makeProviderAuthService = Effect.gen(function* () { const registry = yield* ProviderInstanceRegistry; - const projections = yield* ProjectionStoreV2; const providerSessions = yield* ProviderSessionManagerV2; const credentialChanges = yield* Semaphore.make(1); @@ -36,15 +33,13 @@ export const makeProviderAuthService = Effect.gen(function* () { return instance.auth; }); - // Native sessions may still belong to the previous provider after the - // selected model changes. Read session bindings, not the selected model, - // when invalidating credentials for sign-in or sign-out. + // The manager tracks resources that projections cannot: in-flight opens and + // pending cleanup still own credentials even when their session is in error. + // Close each current shared owner before invalidating its auth controller. const stopSessions = Effect.fn("ProviderAuthService.stopSessions")(function* ( instanceId: ProviderInstanceId, binding: ProviderAuthService.ProviderAuthController["credentialBinding"], ) { - const failure = (detail: string) => - new ProviderSetupError({ instanceId, operation: "stopSessions", detail }); const affectedIds = new Set([ instanceId, ...(binding === undefined @@ -57,49 +52,27 @@ export const makeProviderAuthService = Effect.gen(function* () { ) .map((instance) => instance.instanceId)), ]); - const threadIds = yield* projections - .getRecoveryThreadIds("runtime") - .pipe( - Effect.mapError(() => failure("Could not read the provider's active sessions. Try again.")), - ); - const released = new Set(); yield* Effect.forEach( - threadIds, - (threadId) => - projections.getThreadRecords(threadId, ["providerSessions"]).pipe( - Effect.flatMap((projection) => - Effect.forEach( - projection.providerSessions.filter( - (session) => - affectedIds.has(session.providerInstanceId) && - session.status !== "stopped" && - session.status !== "error" && - !released.has(session.id), - ), - (session) => - Effect.gen(function* () { - if (session.providerInstanceId !== instanceId) { - const current = yield* registry.getInstance(session.providerInstanceId); - if ( - !binding || - current?.auth?.credentialBinding?.key !== binding.key || - current.auth.credentialBinding.owner !== binding.owner - ) - return; - } - yield* providerSessions - .release({ - providerSessionId: session.id, - reason: "manual_shutdown", - detail: "Provider sign-in changed.", - }) - .pipe(Effect.tap(() => Effect.sync(() => released.add(session.id)))); - }), - { discard: true }, - ), - ), - Effect.mapError(() => - failure("Could not stop all sessions for this provider. Try again."), + affectedIds, + (affectedId) => + Effect.gen(function* () { + if (affectedId !== instanceId) { + const current = yield* registry.getInstance(affectedId); + if ( + !binding || + current?.auth?.credentialBinding?.key !== binding.key || + current.auth.credentialBinding.owner !== binding.owner + ) return; + } + yield* providerSessions.closeInstance(affectedId); + }).pipe( + Effect.mapError((cause) => + new ProviderSetupError({ + instanceId, + operation: "stopSessions", + detail: "Could not stop all sessions for this provider. Try again.", + cause, + }), ), ), { discard: true }, From 124b10a2600712ec782ced848fa7b14cd8a99ddf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:48:12 +1000 Subject: [PATCH 03/12] fix(server): keep cleanup ownership honest through interrupt and busy marks Astra xhigh follow-ups on the provider-cleanup contract: - Claude interrupt timeouts only drop queryContext when the native close succeeded; a failed close stays tracked so the session scope retries it instead of reporting a clean release over a live CLI. - startTurn and compactThread unwind their busy mark with onExit so interruption while queued on the attach lock cannot pin the session busy forever (catch only covered typed failures). - The attach/detach race regression now sweeps 60 deterministic scheduler offsets through the credential-issuance gate. - Close-propagation tests inject typed Effect.fail errors instead of defects, since Effect.ignore preserves defects either way. Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../Adapters/ClaudeAdapterV2.test.ts | 120 +++++++++- .../Adapters/ClaudeAdapterV2.ts | 12 +- .../Adapters/CursorAdapterV2.test.ts | 16 +- .../ProviderSessionManager.test.ts | 225 +++++++++++++++--- .../ProviderSessionManager.ts | 125 +++++----- 5 files changed, 405 insertions(+), 93 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index ef6196485e3e..3f67a97e5d8a 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -40,9 +40,11 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; +import { TestClock } from "effect/testing"; import { Tool } from "effect/unstable/ai"; import { formatClaudeResumeCompactionQuestion } from "@t3tools/shared/claudeCompaction"; @@ -7134,7 +7136,15 @@ describe("ClaudeAdapterV2 session cleanup", () => { offer: () => Effect.void, setModel: () => Effect.void, interrupt: Effect.void, - close: Effect.die("claude query close failed"), + // A typed runner failure is what `Effect.ignore` used to + // swallow — a defect would propagate through it either way, so + // only the error channel distinguishes the fixed behavior. + close: Effect.fail( + new ClaudeAgentSdkQueryRunnerError({ + method: "query.close", + cause: new Error("claude query close failed"), + }), + ), }), forkSession: () => Effect.die("unused"), assertComplete: Effect.void, @@ -7171,4 +7181,112 @@ describe("ClaudeAdapterV2 session cleanup", () => { assert.isTrue(Exit.isFailure(closeExit)); }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, idAllocatorLayer))), ); + + it.effect("keeps a failed interrupt close tracked so the session scope retries it", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-interrupt-close-", + }); + const scope = yield* Scope.make(); + const closeCalls = yield* Ref.make(0); + const interrupted = yield* Deferred.make(); + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + path: yield* Path.Path, + idAllocator: yield* IdAllocatorV2, + queryRunner: { + allocateSessionId: Effect.succeed("native-thread-claude-interrupt-close"), + open: () => + Effect.succeed({ + // Never ends, so interruptTurn's closed wait can only resolve + // through its own timeout path. + messages: Stream.never, + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Deferred.succeed(interrupted, undefined).pipe(Effect.asVoid), + close: Ref.update(closeCalls, (count) => count + 1).pipe( + Effect.andThen( + Effect.fail( + new ClaudeAgentSdkQueryRunnerError({ + method: "query.close", + cause: new Error("claude query close failed"), + }), + ), + ), + ), + }), + forkSession: () => Effect.die("unused"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-interrupt-close"); + const runtime = yield* adapter + .openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-interrupt-close"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }) + .pipe(Effect.provideService(Scope.Scope, scope)); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const events: Array = []; + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkIn(scope), + ); + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-claude-interrupt-close"), + text: "hello", + attachments: [], + }), + ); + for (let attempt = 0; attempt < 5000; attempt++) { + if (events.some((event) => event.type === "provider_turn.updated")) { + break; + } + yield* Effect.yieldNow; + } + const providerTurnId = events.find( + (event): event is Extract => + event.type === "provider_turn.updated", + )?.providerTurn.id; + assert.isDefined(providerTurnId); + + const interrupting = yield* runtime + .interruptTurn({ providerThread, providerTurnId: providerTurnId! }) + .pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(interrupted); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* TestClock.adjust("10 seconds"); + const interruptExit = yield* Fiber.join(interrupting); + // The failed close propagates through interruptTurn... + assert.equal(interruptExit._tag, "Failure"); + assert.equal(yield* Ref.get(closeCalls), 1); + + // ...and the query stays tracked, so the scope close retries the + // native close and surfaces the failure instead of reporting a clean + // release over a CLI that may still be alive. + const closeExit = yield* Scope.close(scope, Exit.void).pipe(Effect.exit); + assert.isTrue(Exit.isFailure(closeExit)); + assert.equal(yield* Ref.get(closeCalls), 2); + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, idAllocatorLayer))), + ); }); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index d93539778fff..3264bd70f53d 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -6583,9 +6583,15 @@ export function makeClaudeAdapterV2( providerThreadId: turnInput.providerThread.id, providerTurnId: turnInput.providerTurnId, }); - yield* Ref.update(queryContext, (current) => - current?.query === existing.query ? null : current, - ); + // A failed close means the query's CLI may still be alive: + // dropping its context here would let the session finalizer + // report a clean close over it. Keep the failed query tracked + // so the scope close retries the close and propagates. + if (Exit.isSuccess(closeExit)) { + yield* Ref.update(queryContext, (current) => + current?.query === existing.query ? null : current, + ); + } yield* finalizeActiveTurn({ context: currentTurn, status: "interrupted", diff --git a/apps/server/src/orchestration-v2/Adapters/CursorAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CursorAdapterV2.test.ts index 9033bb38bd77..945f07171b22 100644 --- a/apps/server/src/orchestration-v2/Adapters/CursorAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CursorAdapterV2.test.ts @@ -36,7 +36,11 @@ import { makeCursorAdapterV2, nestedToolCallFromEnvelope, } from "./CursorAdapterV2.ts"; -import { isCursorCancellationError, loggedCursorAgentOptions } from "./CursorAgentSdk.ts"; +import { + CursorAgentSdkRunnerError, + isCursorCancellationError, + loggedCursorAgentOptions, +} from "./CursorAgentSdk.ts"; const decodeCursorSettings = Schema.decodeEffect(CursorSettings); @@ -849,7 +853,15 @@ describe("CursorAdapterV2", () => { Effect.succeed({ agentId: "native-cursor-close", listMessages: Effect.succeed([]), - close: Effect.die("cursor session close failed"), + // A typed runner failure is what `Effect.ignore` used to + // swallow — a defect would propagate through it either way, so + // only the error channel distinguishes the fixed behavior. + close: Effect.fail( + new CursorAgentSdkRunnerError({ + method: "agent.close", + cause: new Error("cursor session close failed"), + }), + ), send: () => Effect.die("unused in close test"), }), }, diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index f46f1084f24d..bb98c280aebc 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -5998,17 +5998,153 @@ it.effect( it.effect( "ProviderSessionManagerV2 keeps the attach lock through the adapter call when a detach queues", + () => + Effect.gen(function* () { + // Every op offset past the credential-issuance gate gets its own fresh + // manager: the attach is suspended exactly there, and the queued detach + // is free to run into any release/reacquire gap the bookkeeping leaves + // open. The adapter call must always observe the thread's credential + // claim and the teardown must land only after the call — the ordering + // one continuous [session, thread] lock hold guarantees. + for (let offset = 1; offset <= 60; offset += 1) { + const state = yield* Ref.make(emptyState); + const issuing = yield* Deferred.make(); + const issueGate = yield* Deferred.make(); + const pauseIssue = yield* Ref.make(false); + const adapterCalls = yield* Ref.make(0); + const callsWithClaim = yield* Ref.make(0); + const allocatorSlot = yield* Ref.make(undefined); + const sessionIdSlot = yield* Ref.make(undefined); + const sharedThreadId = ThreadId.make(`thread-detach-gap-shared-${offset}`); + const mcpRegistryLayer = Layer.effect( + McpSessionRegistry.McpSessionRegistry, + Effect.gen(function* () { + const delegate = yield* McpSessionRegistry.McpSessionRegistry; + return McpSessionRegistry.McpSessionRegistry.of({ + ...delegate, + issue: (input) => + Ref.get(pauseIssue).pipe( + Effect.flatMap((pause) => + pause + ? Deferred.succeed(issuing, undefined).pipe( + Effect.andThen(Deferred.await(issueGate)), + ) + : Effect.void, + ), + Effect.andThen(delegate.issue(input)), + ), + }); + }), + ).pipe(Layer.provide(TestMcpRegistryLayer)); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + yield* Ref.set(allocatorSlot, yield* IdAllocatorV2); + const stepper = makeSteppingScheduler(); + const ownerThreadId = ThreadId.make(`thread-detach-gap-owner-${offset}`); + const owner = yield* makeThreadSessionFixture(ownerThreadId); + yield* makeThreadSessionFixture(sharedThreadId); + const providerSessionId = yield* owner.allocate; + yield* Ref.set(sessionIdSlot, providerSessionId); + const runtime = yield* owner.open(providerSessionId); + // Arm the gate only after the owner's open — its own issuance must + // pass straight through. + yield* Ref.set(pauseIssue, true); + + // The shared thread's attach suspends inside credential issuance + // with the [session, thread] lock held and its provisional + // attachment already recorded, so the terminal detach queues on + // that lock. + const attaching = yield* runtime + .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) + .pipe( + Effect.exit, + Effect.forkDetach, + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + ); + yield* stepper.step(attaching, 10_000); + assert.isTrue( + yield* Deferred.isDone(issuing), + `attach never reached credential issuance (offset ${offset})`, + ); + const gatedAt = stepper.counts.get(attaching) ?? 0; + + const detaching = yield* manager + .detach({ providerSessionId, threadId: sharedThreadId, revokeMcpCredential: true }) + .pipe( + Effect.exit, + Effect.forkDetach, + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + ); + yield* stepper.step(detaching, 10_000); + + // Resume issuance, then hold the attach at the swept op count: if + // bookkeeping released the lock before the adapter call reacquired + // it, the parked detach wakes first and strips the attachment and + // its credential claim before the call is admitted. + yield* Deferred.succeed(issueGate, undefined); + yield* stepper.step(attaching, gatedAt + offset); + yield* stepper.step(detaching, 10_000); + yield* stepper.drain; + + const attachExit = yield* Fiber.join(attaching); + assert.equal( + attachExit._tag, + "Success", + `attach failed at op offset ${offset}: ${ + attachExit._tag === "Failure" ? Cause.pretty(attachExit.cause) : "no cause" + }`, + ); + assert.equal(yield* Ref.get(adapterCalls), 1, `op count at offset ${offset}`); + assert.equal( + yield* Ref.get(callsWithClaim), + 1, + `adapter call ran after the detach stripped the credential claim (offset ${offset})`, + ); + const detachExit = yield* Fiber.join(detaching); + assert.equal(detachExit._tag, "Success", `detach outcome at offset ${offset}`); + // The queued detach then tears the attachment down: the thread's + // credential is revoked and its config slot cleared. + assert.isUndefined(McpProviderSession.readMcpProviderSession(sharedThreadId)); + }).pipe( + Effect.ensuring(Deferred.succeed(issueGate, undefined)), + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + mcpRegistryLayer, + ensureThread: (threadInput) => + Effect.gen(function* () { + yield* Ref.update(adapterCalls, (count) => count + 1); + if ( + McpProviderSession.readMcpProviderSession(threadInput.threadId) !== undefined + ) { + yield* Ref.update(callsWithClaim, (count) => count + 1); + } + return makeProviderThread({ + idAllocator: (yield* Ref.get(allocatorSlot))!, + threadId: threadInput.threadId, + providerSessionId: (yield* Ref.get(sessionIdSlot))!, + now: yield* DateTime.now, + }); + }), + }), + ), + ); + } + }), +); + +it.effect( + "ProviderSessionManagerV2 unwinds a queued startTurn's busy mark when the caller is interrupted", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); const issuing = yield* Deferred.make(); const issueGate = yield* Deferred.make(); const pauseIssue = yield* Ref.make(false); - const adapterCalls = yield* Ref.make(0); - const callsWithClaim = yield* Ref.make(0); + const sharedThreadId = ThreadId.make("thread-busy-unwind-shared"); const allocatorSlot = yield* Ref.make(undefined); const sessionIdSlot = yield* Ref.make(undefined); - const sharedThreadId = ThreadId.make("thread-detach-gap-shared"); const mcpRegistryLayer = Layer.effect( McpSessionRegistry.McpSessionRegistry, Effect.gen(function* () { @@ -6031,54 +6167,89 @@ it.effect( ).pipe(Layer.provide(TestMcpRegistryLayer)); yield* Effect.gen(function* () { const manager = yield* ProviderSessionManagerV2; - yield* Ref.set(allocatorSlot, yield* IdAllocatorV2); - const ownerThreadId = ThreadId.make("thread-detach-gap-owner"); + const idAllocator = yield* IdAllocatorV2; + const projectionStore = yield* ProjectionStoreV2; + const now = yield* DateTime.now; + const ownerThreadId = ThreadId.make("thread-busy-unwind-owner"); const owner = yield* makeThreadSessionFixture(ownerThreadId); yield* makeThreadSessionFixture(sharedThreadId); const providerSessionId = yield* owner.allocate; + yield* Ref.set(allocatorSlot, idAllocator); yield* Ref.set(sessionIdSlot, providerSessionId); const runtime = yield* owner.open(providerSessionId); yield* Ref.set(pauseIssue, true); - // The shared thread's attach suspends inside credential issuance with - // the [session, thread] lock held and its provisional attachment - // already recorded, so the terminal detach queues on that lock. + + // The shared thread's attach parks inside credential issuance with + // the [session, thread] lock held, so the queued startTurn marks the + // session busy and then waits on that lock. const attaching = yield* runtime .ensureThread({ threadId: sharedThreadId, modelSelection, runtimePolicy }) .pipe(Effect.exit, Effect.forkChild); yield* Deferred.await(issuing); - const detaching = yield* manager - .detach({ providerSessionId, threadId: sharedThreadId, revokeMcpCredential: true }) + const sharedAppThread = (yield* projectionStore.getThreadProjection(sharedThreadId)).thread; + const sharedRunId = idAllocator.derive.run({ threadId: sharedThreadId, ordinal: 1 }); + const turning = yield* runtime + .startTurn({ + appThread: sharedAppThread, + threadId: sharedThreadId, + runId: sharedRunId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId: sharedRunId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId: sharedRunId }), + providerThread: makeProviderThread({ + idAllocator, + threadId: sharedThreadId, + providerSessionId, + now, + }), + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ + threadId: sharedThreadId, + ordinal: 1, + }), + text: "queued behind the attach", + attachments: [], + }, + modelSelection, + runtimePolicy, + }) .pipe(Effect.exit, Effect.forkChild); - // The queued detach cannot strip the attachment between the attach's - // bookkeeping and the adapter call: one lock hold spans both, so the - // adapter runs while the credential claim still exists and the - // teardown only lands after the call completes. - yield* Effect.forEach(Array.from({ length: 12 }), () => Effect.yieldNow, { + // Let the turn reach the lock wait behind the in-flight attach. + yield* Effect.forEach(Array.from({ length: 20 }), () => Effect.yieldNow, { discard: true, }); + yield* Fiber.interrupt(turning); + const turnExit = yield* Fiber.await(turning); + assert.equal(turnExit._tag, "Failure"); + yield* Deferred.succeed(issueGate, undefined); const attachExit = yield* Fiber.join(attaching); assert.equal(attachExit._tag, "Success"); - assert.equal(yield* Ref.get(adapterCalls), 1); - assert.equal(yield* Ref.get(callsWithClaim), 1); - const detachExit = yield* Fiber.join(detaching); - assert.equal(detachExit._tag, "Success"); - // The queued detach then tears the attachment down: the thread's - // credential is revoked and its config slot cleared. - assert.isUndefined(McpProviderSession.readMcpProviderSession(sharedThreadId)); + + // markBusy incremented before the attach wait; without an unwind for + // interruption the count would pin the session busy forever and the + // idle release below would never fire. + yield* TestClock.adjust("1 second"); + yield* Effect.yieldNow; + yield* TestClock.adjust("1 second"); + yield* Effect.yieldNow; + assert.isTrue( + Option.isNone(yield* manager.get(providerSessionId)), + "interrupted queued startTurn left the session pinned busy", + ); + assert.equal((yield* Ref.get(state)).closeCount, 1); }).pipe( Effect.ensuring(Deferred.succeed(issueGate, undefined)), Effect.provide( makeTestLayer({ state, - idleTimeoutMs: 3_600_000, + idleTimeoutMs: 1_000, mcpRegistryLayer, ensureThread: (threadInput) => Effect.gen(function* () { - yield* Ref.update(adapterCalls, (count) => count + 1); - if (McpProviderSession.readMcpProviderSession(threadInput.threadId) !== undefined) { - yield* Ref.update(callsWithClaim, (count) => count + 1); - } return makeProviderThread({ idAllocator: (yield* Ref.get(allocatorSlot))!, threadId: threadInput.threadId, diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 8bb7b4473fb4..4cdbbcc2547f 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2543,42 +2543,45 @@ export const layerWithOptions = ( // adapter call: it can suspend waiting on the idle fiber, and the // attach lock must not be held across that wait (an open already // holding threadLifecycle would deadlock on it). The attach's own - // gates re-check ownership after the suspend. - observeActivity(providerSessionId, markBusy(providerSessionId, runtime)).pipe( - Effect.andThen( - attachThreadOrReject( - { - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, - driver: runtime.driver, - expectedRuntime: runtime, - }, - // The adapter call returns once the turn is started — most - // adapters fork the turn's lifetime into the session scope — - // so the admission record spans only the acquisition window - // (Cursor's openAgent/runner.open, ACP session activate+prompt - // submit). Only in-band turn calls get the bounded drain — - // abandoning an acquire-then-return call could strand a - // resource the scope close already finished checking for. - () => - runResourceCreatingAdapterOp({ + // gates re-check ownership after the suspend. The unwind covers + // every non-success exit — interruption while queued on the attach + // lock must not leave the session pinned busy forever. + Effect.onExit( + observeActivity(providerSessionId, markBusy(providerSessionId, runtime)).pipe( + Effect.andThen( + attachThreadOrReject( + { providerSessionId, - expectedRuntime: runtime, + threadId: input.threadId, + providerInstanceId: runtime.instanceId, driver: runtime.driver, - operation: runtime.startTurn(input), - drainTimeout: INBAND_TURN_CALL_DRIVERS.has(runtime.driver) - ? Duration.millis(ADAPTER_OP_DRAIN_TIMEOUT_MS) - : undefined, - requiredThreadId: input.threadId, - }), - ), - ), - Effect.catch((error) => - observeActivity(providerSessionId, markIdle(providerSessionId, runtime)).pipe( - Effect.andThen(Effect.fail(error)), + expectedRuntime: runtime, + }, + // The adapter call returns once the turn is started — most + // adapters fork the turn's lifetime into the session scope — + // so the admission record spans only the acquisition window + // (Cursor's openAgent/runner.open, ACP session activate+prompt + // submit). Only in-band turn calls get the bounded drain — + // abandoning an acquire-then-return call could strand a + // resource the scope close already finished checking for. + () => + runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.startTurn(input), + drainTimeout: INBAND_TURN_CALL_DRIVERS.has(runtime.driver) + ? Duration.millis(ADAPTER_OP_DRAIN_TIMEOUT_MS) + : undefined, + requiredThreadId: input.threadId, + }), + ), ), ), + (exit) => + Exit.isSuccess(exit) + ? Effect.void + : observeActivity(providerSessionId, markIdle(providerSessionId, runtime)), ), steerTurn: (input) => requireLiveRuntime({ @@ -2646,38 +2649,40 @@ export const layerWithOptions = ( : { compactThread: (input: Parameters>[0]) => // Same markBusy-before-attach ordering as startTurn: the - // wait on the idle fiber must not run under the attach lock. - observeActivity(providerSessionId, markBusy(providerSessionId, runtime)).pipe( - Effect.andThen( - attachThreadOrReject( - { - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, - driver: runtime.driver, - expectedRuntime: runtime, - }, - // Compaction delegates to the same acquire-then-start - // path as startTurn, so it shares the admission record - // and the same driver-keyed drain policy. - () => - runResourceCreatingAdapterOp({ + // wait on the idle fiber must not run under the attach lock, + // and the unwind covers interruption the same way. + Effect.onExit( + observeActivity(providerSessionId, markBusy(providerSessionId, runtime)).pipe( + Effect.andThen( + attachThreadOrReject( + { providerSessionId, - expectedRuntime: runtime, + threadId: input.threadId, + providerInstanceId: runtime.instanceId, driver: runtime.driver, - operation: runtime.compactThread!(input), - drainTimeout: INBAND_TURN_CALL_DRIVERS.has(runtime.driver) - ? Duration.millis(ADAPTER_OP_DRAIN_TIMEOUT_MS) - : undefined, - requiredThreadId: input.threadId, - }), - ), - ), - Effect.catch((error) => - observeActivity(providerSessionId, markIdle(providerSessionId, runtime)).pipe( - Effect.andThen(Effect.fail(error)), + expectedRuntime: runtime, + }, + // Compaction delegates to the same acquire-then-start + // path as startTurn, so it shares the admission record + // and the same driver-keyed drain policy. + () => + runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.compactThread!(input), + drainTimeout: INBAND_TURN_CALL_DRIVERS.has(runtime.driver) + ? Duration.millis(ADAPTER_OP_DRAIN_TIMEOUT_MS) + : undefined, + requiredThreadId: input.threadId, + }), + ), ), ), + (exit) => + Exit.isSuccess(exit) + ? Effect.void + : observeActivity(providerSessionId, markIdle(providerSessionId, runtime)), ), }), readThreadSnapshot: (input) => From c197fd512631ff79e3cc2752264abb2a923f0428 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 21:16:17 +1000 Subject: [PATCH 04/12] fix(server): couple busy-mark acquisition to its unwind and arm the attach-race sweep before the gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The onExit unwind for startTurn/compactThread decremented busyCount on any non-success exit, so an interrupt landing before markBusy's increment stole a different holder's mark and idled a live turn. markBusy now takes an acquisition flag set inside the same uninterruptible region as the increment, and the unwind only calls markIdle when this call actually took the mark. The attach-lock sweep aimed the stepping scheduler after opening the credential gate, but Deferred.succeed resumes waiters inline on the synchronous scheduler, so the target never engaged. The sweep now arms before the gate opens and reaches offset 520 — past the ~505-op bookkeeping-to-admission boundary where the unfused release/reacquire gap actually sat. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../ProviderSessionManager.test.ts | 130 ++++++++++- .../ProviderSessionManager.ts | 216 +++++++++++------- 2 files changed, 257 insertions(+), 89 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index bb98c280aebc..7d32fe3d0dd9 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -6005,8 +6005,11 @@ it.effect( // is free to run into any release/reacquire gap the bookkeeping leaves // open. The adapter call must always observe the thread's credential // claim and the teardown must land only after the call — the ordering - // one continuous [session, thread] lock hold guarantees. - for (let offset = 1; offset <= 60; offset += 1) { + // one continuous [session, thread] lock hold guarantees. The sweep + // reaches well past the bookkeeping-to-admission boundary (~op 505 on + // the unfused implementation — credential issuance dominates the + // ~440 ops between the park point and the lock release). + for (let offset = 1; offset <= 520; offset += 1) { const state = yield* Ref.make(emptyState); const issuing = yield* Deferred.make(); const issueGate = yield* Deferred.make(); @@ -6077,13 +6080,17 @@ it.effect( ); yield* stepper.step(detaching, 10_000); - // Resume issuance, then hold the attach at the swept op count: if - // bookkeeping released the lock before the adapter call reacquired - // it, the parked detach wakes first and strips the attachment and - // its credential claim before the call is admitted. + // Arm the hold *before* opening the gate: on this synchronous + // scheduler the waiter's resume runs inline inside + // Deferred.succeed, so a target installed afterwards never sees + // the resume. If bookkeeping released the lock before the adapter + // call reacquired it, the parked detach wakes first while the + // attach is held and strips the attachment and its credential + // claim before the call is admitted. + stepper.aim(attaching, gatedAt + offset); yield* Deferred.succeed(issueGate, undefined); - yield* stepper.step(attaching, gatedAt + offset); yield* stepper.step(detaching, 10_000); + yield* stepper.step(attaching, 10_000); yield* stepper.drain; const attachExit = yield* Fiber.join(attaching); @@ -6263,6 +6270,115 @@ it.effect( }), ); +it.effect( + "ProviderSessionManagerV2 does not steal a running turn's busy mark when an early interrupt lands", + () => + Effect.gen(function* () { + // The unwind must only release a mark this call actually took. Sweeping + // the armed op puts the interrupt at every point around the markBusy + // increment — before it, the interrupted call owns no mark and the + // running turn's count must keep the session busy. + const sawInterrupt = yield* Ref.make(false); + for (let offset = 1; offset <= 16; offset += 1) { + const state = yield* Ref.make(emptyState); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const idAllocator = yield* IdAllocatorV2; + const projectionStore = yield* ProjectionStoreV2; + const stepper = makeSteppingScheduler(); + const threadId = ThreadId.make(`thread-busy-steal-${offset}`); + const session = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* session.allocate; + const runtime = yield* session.open(providerSessionId); + const now = yield* DateTime.now; + const appThread = (yield* projectionStore.getThreadProjection(threadId)).thread; + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + const turnInput = (ordinal: number) => + Effect.gen(function* () { + const runId = idAllocator.derive.run({ threadId, ordinal }); + return { + appThread, + threadId, + runId, + runOrdinal: ordinal, + providerTurnOrdinal: ordinal, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal }), + text: "keep the session busy", + attachments: [], + }, + modelSelection, + runtimePolicy, + } satisfies ProviderAdapterV2TurnInput; + }); + + // Turn A takes a mark and no terminal turn event ever arrives, so + // the session holds busyCount=1 for the rest of the test. + yield* runtime.startTurn(yield* turnInput(1)); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + // Turn B is held at the swept op and then interrupted — before its + // own increment it owns no mark, so the unwind must leave turn A's + // count alone. + // The hold must be armed after the fork: while the forkDetach is + // evaluated under provideService the test fiber itself consults the + // stepper, and an armed hold would park it in `held` forever. + const turning = yield* runtime + .startTurn(yield* turnInput(2)) + .pipe( + Effect.exit, + Effect.forkDetach, + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + ); + stepper.holdSpawnedAt(offset); + yield* stepper.drain; + // interruptUnsafe only signals — Fiber.interrupt would wait for an + // exit the parked stepper cannot produce while the test fiber + // blocks. + yield* Effect.sync(() => turning.interruptUnsafe()); + stepper.resumeSpawned(); + yield* stepper.drain; + const turnExit = yield* Fiber.await(turning); + if (turnExit._tag === "Failure") { + yield* Ref.set(sawInterrupt, true); + } + + yield* TestClock.adjust("1 second"); + yield* Effect.yieldNow; + yield* TestClock.adjust("1 second"); + yield* Effect.yieldNow; + assert.isTrue( + Option.isSome(yield* manager.get(providerSessionId)), + `interrupted startTurn stole the running turn's busy mark (offset ${offset})`, + ); + assert.equal((yield* Ref.get(state)).closeCount, 0, `close at offset ${offset}`); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 1_000, + }), + ), + ); + } + assert.isTrue( + yield* Ref.get(sawInterrupt), + "no swept offset interrupted the startTurn mid-flight", + ); + }), +); + it.effect( "ProviderSessionManagerV2 serializes credential revocation against a new attachment's reservation", () => diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 4cdbbcc2547f..36be4631b18b 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2077,30 +2077,43 @@ export const layerWithOptions = ( const markBusy = ( providerSessionId: ProviderSessionId, expectedRuntime?: ProviderAdapterV2SessionRuntime, + acquired?: Ref.Ref, ) => withActivityError( providerSessionId, Effect.gen(function* () { const key = sessionKey(providerSessionId); const now = yield* Clock.currentTimeMillis; - const idleFiber = yield* Ref.modify(sessions, (current) => { - const entry = current.get(key); - if ( - entry === undefined || - (expectedRuntime !== undefined && entry.runtime !== expectedRuntime) - ) { - return [null, current] as const; - } - const updated = new Map(current); - updated.set(key, { - ...entry, - busyCount: entry.busyCount + 1, - idleFiber: null, - lastActivityAtMs: now, - pinnedSinceMs: null, - }); - return [entry.idleFiber, updated] as const; - }); + const [idleFiber] = yield* Effect.uninterruptible( + Effect.gen(function* () { + const outcome = yield* Ref.modify(sessions, (current) => { + const entry = current.get(key); + if ( + entry === undefined || + (expectedRuntime !== undefined && entry.runtime !== expectedRuntime) + ) { + return [[null, false] as const, current] as const; + } + const updated = new Map(current); + updated.set(key, { + ...entry, + busyCount: entry.busyCount + 1, + idleFiber: null, + lastActivityAtMs: now, + pinnedSinceMs: null, + }); + return [[entry.idleFiber, true] as const, updated] as const; + }); + // The acquisition flag is set in the same uninterruptible + // region as the increment so an onExit unwind can tell + // exactly whether this call took a mark — decrementing + // without it would steal another holder's busy count. + if (acquired !== undefined && outcome[1]) { + yield* Ref.set(acquired, true); + } + return outcome; + }), + ); yield* cancelIdleFiber(idleFiber); }), ); @@ -2545,43 +2558,63 @@ export const layerWithOptions = ( // holding threadLifecycle would deadlock on it). The attach's own // gates re-check ownership after the suspend. The unwind covers // every non-success exit — interruption while queued on the attach - // lock must not leave the session pinned busy forever. - Effect.onExit( - observeActivity(providerSessionId, markBusy(providerSessionId, runtime)).pipe( - Effect.andThen( - attachThreadOrReject( - { - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, - driver: runtime.driver, - expectedRuntime: runtime, - }, - // The adapter call returns once the turn is started — most - // adapters fork the turn's lifetime into the session scope — - // so the admission record spans only the acquisition window - // (Cursor's openAgent/runner.open, ACP session activate+prompt - // submit). Only in-band turn calls get the bounded drain — - // abandoning an acquire-then-return call could strand a - // resource the scope close already finished checking for. - () => - runResourceCreatingAdapterOp({ - providerSessionId, - expectedRuntime: runtime, - driver: runtime.driver, - operation: runtime.startTurn(input), - drainTimeout: INBAND_TURN_CALL_DRIVERS.has(runtime.driver) - ? Duration.millis(ADAPTER_OP_DRAIN_TIMEOUT_MS) - : undefined, - requiredThreadId: input.threadId, - }), + // lock must not leave the session pinned busy forever — but only + // when this call actually took the mark: decrementing without it + // would steal another holder's busy count and idle a live turn. + Ref.make(false).pipe( + Effect.flatMap((busyMarked) => + Effect.onExit( + observeActivity( + providerSessionId, + markBusy(providerSessionId, runtime, busyMarked), + ).pipe( + Effect.andThen( + attachThreadOrReject( + { + providerSessionId, + threadId: input.threadId, + providerInstanceId: runtime.instanceId, + driver: runtime.driver, + expectedRuntime: runtime, + }, + // The adapter call returns once the turn is started — + // most adapters fork the turn's lifetime into the + // session scope — so the admission record spans only + // the acquisition window (Cursor's openAgent/ + // runner.open, ACP session activate+prompt submit). + // Only in-band turn calls get the bounded drain — + // abandoning an acquire-then-return call could strand + // a resource the scope close already finished + // checking for. + () => + runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.startTurn(input), + drainTimeout: INBAND_TURN_CALL_DRIVERS.has(runtime.driver) + ? Duration.millis(ADAPTER_OP_DRAIN_TIMEOUT_MS) + : undefined, + requiredThreadId: input.threadId, + }), + ), + ), ), + (exit) => + Exit.isSuccess(exit) + ? Effect.void + : Ref.get(busyMarked).pipe( + Effect.flatMap((marked) => + marked + ? observeActivity( + providerSessionId, + markIdle(providerSessionId, runtime), + ) + : Effect.void, + ), + ), ), ), - (exit) => - Exit.isSuccess(exit) - ? Effect.void - : observeActivity(providerSessionId, markIdle(providerSessionId, runtime)), ), steerTurn: (input) => requireLiveRuntime({ @@ -2650,39 +2683,58 @@ export const layerWithOptions = ( compactThread: (input: Parameters>[0]) => // Same markBusy-before-attach ordering as startTurn: the // wait on the idle fiber must not run under the attach lock, - // and the unwind covers interruption the same way. - Effect.onExit( - observeActivity(providerSessionId, markBusy(providerSessionId, runtime)).pipe( - Effect.andThen( - attachThreadOrReject( - { - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, - driver: runtime.driver, - expectedRuntime: runtime, - }, - // Compaction delegates to the same acquire-then-start - // path as startTurn, so it shares the admission record - // and the same driver-keyed drain policy. - () => - runResourceCreatingAdapterOp({ - providerSessionId, - expectedRuntime: runtime, - driver: runtime.driver, - operation: runtime.compactThread!(input), - drainTimeout: INBAND_TURN_CALL_DRIVERS.has(runtime.driver) - ? Duration.millis(ADAPTER_OP_DRAIN_TIMEOUT_MS) - : undefined, - requiredThreadId: input.threadId, - }), + // and the unwind covers interruption the same way — gated on + // this call's own acquisition flag so a pre-increment + // interrupt cannot steal another holder's busy count. + Ref.make(false).pipe( + Effect.flatMap((busyMarked) => + Effect.onExit( + observeActivity( + providerSessionId, + markBusy(providerSessionId, runtime, busyMarked), + ).pipe( + Effect.andThen( + attachThreadOrReject( + { + providerSessionId, + threadId: input.threadId, + providerInstanceId: runtime.instanceId, + driver: runtime.driver, + expectedRuntime: runtime, + }, + // Compaction delegates to the same + // acquire-then-start path as startTurn, so it + // shares the admission record and the same + // driver-keyed drain policy. + () => + runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.compactThread!(input), + drainTimeout: INBAND_TURN_CALL_DRIVERS.has(runtime.driver) + ? Duration.millis(ADAPTER_OP_DRAIN_TIMEOUT_MS) + : undefined, + requiredThreadId: input.threadId, + }), + ), + ), ), + (exit) => + Exit.isSuccess(exit) + ? Effect.void + : Ref.get(busyMarked).pipe( + Effect.flatMap((marked) => + marked + ? observeActivity( + providerSessionId, + markIdle(providerSessionId, runtime), + ) + : Effect.void, + ), + ), ), ), - (exit) => - Exit.isSuccess(exit) - ? Effect.void - : observeActivity(providerSessionId, markIdle(providerSessionId, runtime)), ), }), readThreadSnapshot: (input) => From aa5bdea57fa9ee43db28557b5dd4a22d556af021 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 21:34:46 +1000 Subject: [PATCH 05/12] fix(server): annotate the markBusy modify outcome tuple for strict CI typecheck The two-branch modify callback inferred the union's first member as the result type, leaving outcome unknown under the repo-wide CI typecheck. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../ProviderSessionManager.ts | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 36be4631b18b..8647d3f249be 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2086,24 +2086,32 @@ export const layerWithOptions = ( const now = yield* Clock.currentTimeMillis; const [idleFiber] = yield* Effect.uninterruptible( Effect.gen(function* () { - const outcome = yield* Ref.modify(sessions, (current) => { - const entry = current.get(key); - if ( - entry === undefined || - (expectedRuntime !== undefined && entry.runtime !== expectedRuntime) - ) { - return [[null, false] as const, current] as const; - } - const updated = new Map(current); - updated.set(key, { - ...entry, - busyCount: entry.busyCount + 1, - idleFiber: null, - lastActivityAtMs: now, - pinnedSinceMs: null, - }); - return [[entry.idleFiber, true] as const, updated] as const; - }); + const outcome = yield* Ref.modify( + sessions, + ( + current, + ): readonly [ + readonly [Fiber.Fiber | null, boolean], + Map, + ] => { + const entry = current.get(key); + if ( + entry === undefined || + (expectedRuntime !== undefined && entry.runtime !== expectedRuntime) + ) { + return [[null, false], current]; + } + const updated = new Map(current); + updated.set(key, { + ...entry, + busyCount: entry.busyCount + 1, + idleFiber: null, + lastActivityAtMs: now, + pinnedSinceMs: null, + }); + return [[entry.idleFiber, true], updated]; + }, + ); // The acquisition flag is set in the same uninterruptible // region as the increment so an onExit unwind can tell // exactly whether this call took a mark — decrementing From 016d28ad7c12dd1ac5bf8bf38c3e6c6c34b79f92 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 22:21:46 +1000 Subject: [PATCH 06/12] fix(server): bound the effect-worker stop wait and reconcile the replay outbox before its worker restarts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stalled adapter acquisition runs inside an uninterruptible mask, so an unbounded Fiber.interrupt on the effect worker could pin shutdown before provider-session cleanup ever ran. The stop wait is now bounded by the same 30s bound used for cleanup, at both the shutdown finalizer and the relay-startup failure path. The replay harness restart path also forked the effect worker daemon with its layer before reconcileAfterProcessLoss ran, letting the daemon claim persisted rows that reconciliation then reset to pending. The restart path now builds the layer without the daemon, reconciles, then forks the worker — matching production ordering. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../ProviderSessionManager.ts | 5 +- ...estratorReplayRecovery.integration.test.ts | 152 ++++++++++++++++++ .../testkit/ProviderReplayHarness.ts | 42 +++-- .../Layers/ProviderAuthService.test.ts | 15 +- .../provider/Layers/ProviderAuthService.ts | 23 ++- apps/server/src/serverRuntimeStartup.test.ts | 34 ++++ apps/server/src/serverRuntimeStartup.ts | 18 ++- 7 files changed, 252 insertions(+), 37 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 8647d3f249be..5681f7fcac20 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -3798,7 +3798,10 @@ export const layerWithOptions = ( } if (currentEntry?.supportsMultipleProviderThreads === true) { const projection = yield* Effect.option( - projectionStore.getThreadRecords(input.threadId, ["providerThreads", "providerTurns"]), + projectionStore.getThreadRecords(input.threadId, [ + "providerThreads", + "providerTurns", + ]), ); if (Option.isSome(projection)) { const providerThreads = new Map( diff --git a/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts b/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts index d919bdaa38bb..e9cca5ea829d 100644 --- a/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts @@ -1,3 +1,4 @@ +import * as NodeSqlite from "node:sqlite"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; @@ -423,4 +424,155 @@ describe("orchestrator replay recovery", () => { ), ), ); + it.effect("reconciles persisted effects before the effect worker restarts", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = yield* readCursorTranscript(); + const runner = makeCursorAgentSdkReplayRunner(transcript); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const tempDir = yield* Effect.acquireRelease( + fs.makeTempDirectory({ + prefix: "t3-orchestration-v2-cursor-recovery-", + }), + (directory) => fs.remove(directory, { recursive: true, force: true }).pipe(Effect.orDie), + ); + yield* fs.makeDirectory(tempDir, { recursive: true }); + const dbPath = path.join(tempDir, "state.sqlite"); + const materialized = yield* materializeFixtureInput({ + scenario: "provider_thread_resume", + fixtureInput: { + steps: [ + { type: "message", text: PROVIDER_THREAD_RESUME_FIRST_PROMPT }, + { type: "message", text: PROVIDER_THREAD_RESUME_SECOND_PROMPT }, + ], + }, + driver: ProviderDriverKind.make("cursor"), + modelSelection: CURSOR_MODEL_SELECTION, + }); + const { phase1Commands, phase1Steps, phase2Commands, phase2Steps } = + splitAfterFirstIdle(materialized); + const options = { + databaseLayer: makeSqlitePersistenceLive(dbPath).pipe(Layer.provide(NodeServices.layer)), + }; + const harness = { + ...CursorOrchestratorReplayHarness, + makeProviderAdapterRegistryLayer: () => + makeCursorProviderAdapterRegistryReplayLayer(transcript, { + runner, + assertCompleteOnFinalize: false, + }), + }; + + yield* Effect.scoped( + runOrchestratorV2ProviderReplayScenario( + { + name: "provider_thread_resume/cursor:reconcile-first-runtime", + transcript, + commands: phase1Commands, + steps: phase1Steps, + projectionThreadIds: materialized.projectionThreadIds, + runtimePolicyOverride: { cwd: tempDir }, + runtimeRestart: true, + }, + harness, + options, + ), + ); + + // Persisted work left behind by the dead process: a pending row that + // must be claimed exactly once after reconciliation, and a running + // row that reconciliation must requeue before it can execute again. + // The deterministic runtime's clock starts at epoch, so persisted + // rows must be seeded at epoch to be immediately claimable. + const pendingEffectId = "restart-reconcile-pending"; + const runningEffectId = "restart-reconcile-running"; + const seededAt = "1970-01-01T00:00:00.000Z"; + const seed = new NodeSqlite.DatabaseSync(dbPath); + try { + const insert = seed.prepare( + `INSERT INTO orchestration_v2_effect_outbox ( + effect_id, command_id, thread_id, effect_type, payload_json, + status, attempt_count, available_at, created_at, updated_at + ) VALUES (?, ?, ?, 'terminal.cleanup', '{"type":"terminal.cleanup"}', ?, ?, ?, ?, ?)`, + ); + insert.run( + pendingEffectId, + "restart-reconcile-command-pending", + "restart-reconcile-thread-pending", + "pending", + 0, + seededAt, + seededAt, + seededAt, + ); + insert.run( + runningEffectId, + "restart-reconcile-command-running", + "restart-reconcile-thread-running", + "running", + 1, + seededAt, + seededAt, + seededAt, + ); + } finally { + seed.close(); + } + + yield* Effect.scoped( + runOrchestratorV2ProviderReplayScenario( + { + name: "provider_thread_resume/cursor:reconcile-second-runtime", + transcript, + commands: phase2Commands, + steps: phase2Steps, + projectionThreadIds: materialized.projectionThreadIds, + runtimePolicyOverride: { cwd: tempDir }, + runtimeRestart: true, + }, + harness, + options, + ), + ); + + const check = new NodeSqlite.DatabaseSync(dbPath); + try { + const rows = check + .prepare( + `SELECT effect_id, status, attempt_count, last_error + FROM orchestration_v2_effect_outbox + WHERE effect_id IN (?, ?)`, + ) + .all(pendingEffectId, runningEffectId) as unknown as ReadonlyArray<{ + readonly effect_id: string; + readonly status: string; + readonly attempt_count: number; + readonly last_error: string | null; + }>; + const pendingRow = rows.find((row) => row.effect_id === pendingEffectId); + const runningRow = rows.find((row) => row.effect_id === runningEffectId); + // A pre-reconciliation claim would surface as a second attempt + // after reconciliation requeues the claimed row. + assert.deepStrictEqual(pendingRow, { + effect_id: pendingEffectId, + status: "succeeded", + attempt_count: 1, + last_error: null, + }); + assert.deepStrictEqual(runningRow, { + effect_id: runningEffectId, + status: "succeeded", + attempt_count: 2, + last_error: null, + }); + } finally { + check.close(); + } + }).pipe( + provideDeterministicTestRuntime, + Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer)), + ), + ), + ); }); diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 57a60fbebc8b..09fdef829113 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -220,26 +220,38 @@ export function runOrchestratorV2ProviderReplayScenario< : [], ) ?? [], ); - const layer = makeOrchestratorV2ProviderReplayLayer(scenario, harness, { - ...options, - replayGate, - }); const run = runOrchestratorV2Scenario(scenario, { replayGate }); if (scenario.runtimeRestart !== true) { + const layer = makeOrchestratorV2ProviderReplayLayer(scenario, harness, { + ...options, + replayGate, + }); return run.pipe(Effect.provide(layer)); } - return Effect.gen(function* () { - yield* Effect.flatMap(EffectOutbox.EffectOutboxV2, (outbox) => - outbox.reconcileAfterProcessLoss.pipe(Effect.orDie), - ); - const result = yield* run; - yield* Effect.flatMap( - ProviderSessionManager.ProviderSessionManagerV2, - (sessions) => sessions.shutdown, - ); - return result; - }).pipe(Effect.provide(layer)); + // Production recovers the outbox before starting the effect worker, so the + // restart path builds the layer without the daemon: reconcile first, then + // fork the worker — a daemon already claiming work during reconcile could + // have a freshly-claimed effect reset to pending and run twice. + const restartLayer = makeOrchestratorV2ProviderReplayLayer(scenario, harness, { + ...options, + replayGate, + runEffectWorker: false, + }); + return Effect.scoped( + Effect.gen(function* () { + yield* Effect.flatMap(EffectOutbox.EffectOutboxV2, (outbox) => + outbox.reconcileAfterProcessLoss.pipe(Effect.orDie), + ); + yield* runEffectWorkerDaemon.pipe(Effect.forkScoped); + const result = yield* run; + yield* Effect.flatMap( + ProviderSessionManager.ProviderSessionManagerV2, + (sessions) => sessions.shutdown, + ); + return result; + }), + ).pipe(Effect.provide(restartLayer)); } export function makeOrchestratorV2ProviderReplayLayer< diff --git a/apps/server/src/provider/Layers/ProviderAuthService.test.ts b/apps/server/src/provider/Layers/ProviderAuthService.test.ts index 340c03eae2e2..4eceb39a2617 100644 --- a/apps/server/src/provider/Layers/ProviderAuthService.test.ts +++ b/apps/server/src/provider/Layers/ProviderAuthService.test.ts @@ -216,7 +216,6 @@ const makeHarness = Effect.fn("ProviderAuthService.test.makeHarness")(function* ); } closedInstances.push(id); - }), }), ), @@ -330,7 +329,10 @@ describe("ProviderAuthService", () => { }); yield* service.logout({ instanceId }); assert.deepStrictEqual(closedInstances, [instanceId, otherInstanceId]); - assert.isBelow(actions.indexOf(`close-instance:${otherInstanceId}`), actions.indexOf("invalidate-shared")); + assert.isBelow( + actions.indexOf(`close-instance:${otherInstanceId}`), + actions.indexOf("invalidate-shared"), + ); assert.isBelow(actions.indexOf("invalidate-shared"), actions.indexOf("native-logout")); }), ); @@ -797,9 +799,12 @@ it.effect.each([ const continueDrain = yield* Deferred.make(); const harness = yield* makeHarness({ sharedCredentials: true, - beforeStop: (id) => id === otherInstanceId ? Deferred.succeed(draining, undefined).pipe( - Effect.andThen(Deferred.await(continueDrain)), - ) : Effect.void, + beforeStop: (id) => + id === otherInstanceId + ? Deferred.succeed(draining, undefined).pipe( + Effect.andThen(Deferred.await(continueDrain)), + ) + : Effect.void, }); const logout = yield* harness.service.logout({ instanceId }).pipe(Effect.forkChild); yield* Deferred.await(draining); diff --git a/apps/server/src/provider/Layers/ProviderAuthService.ts b/apps/server/src/provider/Layers/ProviderAuthService.ts index 380e395eff0d..cfc4fab8008e 100644 --- a/apps/server/src/provider/Layers/ProviderAuthService.ts +++ b/apps/server/src/provider/Layers/ProviderAuthService.ts @@ -1,7 +1,4 @@ -import { - ProviderSetupError, - type ProviderInstanceId, -} from "@t3tools/contracts"; +import { ProviderSetupError, type ProviderInstanceId } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Stream from "effect/Stream"; @@ -62,17 +59,19 @@ export const makeProviderAuthService = Effect.gen(function* () { !binding || current?.auth?.credentialBinding?.key !== binding.key || current.auth.credentialBinding.owner !== binding.owner - ) return; + ) + return; } yield* providerSessions.closeInstance(affectedId); }).pipe( - Effect.mapError((cause) => - new ProviderSetupError({ - instanceId, - operation: "stopSessions", - detail: "Could not stop all sessions for this provider. Try again.", - cause, - }), + Effect.mapError( + (cause) => + new ProviderSetupError({ + instanceId, + operation: "stopSessions", + detail: "Could not stop all sessions for this provider. Try again.", + cause, + }), ), ), { discard: true }, diff --git a/apps/server/src/serverRuntimeStartup.test.ts b/apps/server/src/serverRuntimeStartup.test.ts index a89d010fab75..7e92c8180877 100644 --- a/apps/server/src/serverRuntimeStartup.test.ts +++ b/apps/server/src/serverRuntimeStartup.test.ts @@ -10,6 +10,8 @@ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Ref from "effect/Ref"; +import * as Scope from "effect/Scope"; +import { TestClock } from "effect/testing"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; @@ -64,6 +66,38 @@ it.effect("interrupts the effect worker when awareness relay startup fails", () ), ); +it.effect("bounds the worker stop wait when a stalled worker cannot honour the interrupt", () => + Effect.scoped( + Effect.gen(function* () { + const workerFiberRef = yield* Ref.make | null>(null); + + // The worker sits in an uninterruptible region forever, like a stalled + // adapter acquisition, so its interrupt signal can never be honoured. + // The stop wait must still resolve so teardown can continue. The worker + // is scoped: hand the call a scope that is never closed so the + // uninterruptible fiber cannot pin test teardown. + const leakedScope = yield* Scope.make(); + const call = yield* ServerRuntimeStartup.startEffectWorkerWithRelay({ + runWorker: Effect.never.pipe(Effect.uninterruptible), + startRelay: Effect.yieldNow.pipe( + Effect.andThen(Effect.die("awareness relay startup failed")), + ), + workerFiberRef, + }).pipe(Effect.exit, Effect.provideService(Scope.Scope, leakedScope), Effect.forkDetach); + + // Drive the clock until the bounded stop wait fires and the call exits; + // an unbounded wait on the stalled worker never would. + while (call.pollUnsafe() === undefined) { + yield* Effect.yieldNow; + yield* TestClock.adjust("31 seconds"); + } + + const exit = yield* Fiber.join(call); + assert.isTrue(Exit.isFailure(exit)); + }), + ), +); + it.effect("queues commands until startup signals readiness", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index 941bcefa2b95..76f7b877a688 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -98,6 +98,18 @@ const settleQueuedCommand = (deferred: Deferred.Deferred, exit: Exit ? Deferred.succeed(deferred, exit.value) : Deferred.failCause(deferred, exit.cause); +const EFFECT_WORKER_INTERRUPT_TIMEOUT_MS = 30_000; + +// Adapter resource acquisitions run inside an uninterruptible mask, so a +// stalled worker can never honour the interrupt. The interrupt signal is still +// delivered; only the wait is bounded — an overdue worker dies with the +// runtime while teardown moves on to bounded session cleanup. +const interruptEffectWorker = (workerFiber: Fiber.Fiber) => + Fiber.interrupt(workerFiber).pipe( + Effect.timeoutOption(EFFECT_WORKER_INTERRUPT_TIMEOUT_MS), + Effect.ignore, + ); + export const makeCommandGate = Effect.gen(function* () { const commandReady = yield* Deferred.make(); const commandQueue = yield* Queue.unbounded(); @@ -371,9 +383,7 @@ export const startEffectWorkerWithRelay = Effect.fn( } return Ref.getAndSet(input.workerFiberRef, null).pipe( Effect.flatMap((ownedWorkerFiber) => - ownedWorkerFiber === null - ? Effect.void - : Fiber.interrupt(ownedWorkerFiber).pipe(Effect.asVoid), + ownedWorkerFiber === null ? Effect.void : interruptEffectWorker(ownedWorkerFiber), ), ); }), @@ -437,7 +447,7 @@ const make = (options?: StartupOptions) => ); const workerFiber = yield* Ref.getAndSet(effectWorkerFiber, null); if (workerFiber !== null) { - yield* Fiber.interrupt(workerFiber).pipe(Effect.ignore); + yield* interruptEffectWorker(workerFiber); } yield* providerRuntimeRecovery.prepareForShutdown.pipe( Effect.ensuring(providerSessions.shutdown), From 46eac0c831bcb8186c2e24edf478d55791710ca8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 22:35:56 +1000 Subject: [PATCH 07/12] fix(server): detach the effect worker so its scope finalizer cannot wedge teardown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit forkScoped registers its own unbounded Fiber.interrupt finalizer, and it lands after the shutdown teardown finalizer, so it runs first on scope close — a stalled worker still pinned teardown before the bounded wait could run. The worker is now forked detached and bound by a bounded-interrupt finalizer that clears workerFiberRef, keeping a single 30s bound before providerSessions.shutdown. The replay restart path also now honours a caller's runEffectWorker: false instead of forking the daemon unconditionally after reconciliation. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...estratorReplayRecovery.integration.test.ts | 57 +++++++++++++++++++ .../testkit/ProviderReplayHarness.ts | 4 +- apps/server/src/serverActivation.ts | 12 +++- apps/server/src/serverRuntimeStartup.test.ts | 29 ++++++++++ apps/server/src/serverRuntimeStartup.ts | 13 ++++- 5 files changed, 110 insertions(+), 5 deletions(-) diff --git a/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts b/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts index e9cca5ea829d..eb4261b7362d 100644 --- a/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts @@ -569,6 +569,63 @@ describe("orchestrator replay recovery", () => { } finally { check.close(); } + + // A restart with the worker disabled must not start the daemon after + // reconciliation — persisted work stays parked for manual inspection. + const parkedEffectId = "restart-reconcile-parked"; + const seedParked = new NodeSqlite.DatabaseSync(dbPath); + try { + seedParked + .prepare( + `INSERT INTO orchestration_v2_effect_outbox ( + effect_id, command_id, thread_id, effect_type, payload_json, + status, attempt_count, available_at, created_at, updated_at + ) VALUES (?, ?, ?, 'terminal.cleanup', '{"type":"terminal.cleanup"}', 'pending', 0, ?, ?, ?)`, + ) + .run( + parkedEffectId, + "restart-reconcile-command-parked", + "restart-reconcile-thread-parked", + seededAt, + seededAt, + seededAt, + ); + } finally { + seedParked.close(); + } + + yield* Effect.scoped( + runOrchestratorV2ProviderReplayScenario( + { + name: "provider_thread_resume/cursor:reconcile-worker-disabled", + transcript, + commands: [], + steps: [], + projectionThreadIds: materialized.projectionThreadIds, + runtimePolicyOverride: { cwd: tempDir }, + runtimeRestart: true, + }, + harness, + { ...options, runEffectWorker: false }, + ), + ); + + const checkParked = new NodeSqlite.DatabaseSync(dbPath); + try { + const parkedRow = checkParked + .prepare( + `SELECT status, attempt_count + FROM orchestration_v2_effect_outbox + WHERE effect_id = ?`, + ) + .get(parkedEffectId) as unknown as { + readonly status: string; + readonly attempt_count: number; + }; + assert.deepStrictEqual(parkedRow, { status: "pending", attempt_count: 0 }); + } finally { + checkParked.close(); + } }).pipe( provideDeterministicTestRuntime, Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer)), diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 09fdef829113..196538fb1bee 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -243,7 +243,9 @@ export function runOrchestratorV2ProviderReplayScenario< yield* Effect.flatMap(EffectOutbox.EffectOutboxV2, (outbox) => outbox.reconcileAfterProcessLoss.pipe(Effect.orDie), ); - yield* runEffectWorkerDaemon.pipe(Effect.forkScoped); + if (options.runEffectWorker !== false) { + yield* runEffectWorkerDaemon.pipe(Effect.forkScoped); + } const result = yield* run; yield* Effect.flatMap( ProviderSessionManager.ProviderSessionManagerV2, diff --git a/apps/server/src/serverActivation.ts b/apps/server/src/serverActivation.ts index 21326973538d..e5cd707bf007 100644 --- a/apps/server/src/serverActivation.ts +++ b/apps/server/src/serverActivation.ts @@ -9,17 +9,23 @@ export class ServerActivation extends Context.Reference | un { defaultValue: () => undefined }, ) {} -/** Forks a long-running root before commit, returning it after it reaches the activation boundary. */ +/** + * Forks a long-running root before commit, returning it after it reaches the activation boundary. + * `detached` leaves the fiber unbound from the scope — the caller owns its shutdown — + * for workers whose scope-finalizer interrupt wait must stay bounded. + */ export const forkParkedFiber = ( effect: Effect.Effect, + options?: { readonly detached?: boolean }, ): Effect.Effect, never, Scope.Scope | R> => Effect.gen(function* () { const activation = yield* ServerActivation; + const fork = options?.detached === true ? Effect.forkDetach : Effect.forkScoped; if (activation === undefined) { - return yield* Effect.forkScoped(effect); + return yield* fork(effect); } const parked = yield* Deferred.make(); - const fiber = yield* Effect.forkScoped( + const fiber = yield* fork( Deferred.succeed(parked, undefined).pipe(Effect.andThen(activation), Effect.andThen(effect)), ); yield* Deferred.await(parked); diff --git a/apps/server/src/serverRuntimeStartup.test.ts b/apps/server/src/serverRuntimeStartup.test.ts index 7e92c8180877..8c9d59b86d0c 100644 --- a/apps/server/src/serverRuntimeStartup.test.ts +++ b/apps/server/src/serverRuntimeStartup.test.ts @@ -98,6 +98,35 @@ it.effect("bounds the worker stop wait when a stalled worker cannot honour the i ), ); +it.effect("scope close reaches teardown past a stalled uninterruptible worker", () => + Effect.gen(function* () { + const scope = yield* Scope.make(); + const workerFiberRef = yield* Ref.make | null>(null); + // Stands in for providerSessions.shutdown: registered before the worker is + // forked, so it runs after the worker's own scope finalizer on close. + const teardownRan = yield* Ref.make(false); + yield* Effect.gen(function* () { + yield* Effect.addFinalizer(() => Ref.set(teardownRan, true)); + yield* ServerRuntimeStartup.startEffectWorkerWithRelay({ + runWorker: Effect.never.pipe(Effect.uninterruptible), + startRelay: Effect.void, + workerFiberRef, + }); + }).pipe(Effect.provideService(Scope.Scope, scope)); + + // A scope-bound forkScoped worker would wait on the stalled worker's exit + // forever here; the detached worker's bounded-interrupt finalizer lets the + // close reach the teardown finalizer. + const closing = yield* Scope.close(scope, Exit.void).pipe(Effect.forkDetach); + while (closing.pollUnsafe() === undefined) { + yield* Effect.yieldNow; + yield* TestClock.adjust("31 seconds"); + } + yield* Fiber.join(closing); + assert.isTrue(yield* Ref.get(teardownRan)); + }), +); + it.effect("queues commands until startup signals readiness", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index 76f7b877a688..331d8d713f2b 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -374,8 +374,19 @@ export const startEffectWorkerWithRelay = Effect.fn( readonly startRelay: Effect.Effect; readonly workerFiberRef: Ref.Ref | null>; }) { - const workerFiber = yield* forkParkedFiber(input.runWorker); + // The worker is detached from the scope and bound by this bounded-interrupt + // finalizer instead: forkScoped's own finalizer waits on the worker's exit + // unboundedly, and it is registered after the teardown finalizer, so a + // stalled worker would pin scope close before providerSessions.shutdown. + const workerFiber = yield* forkParkedFiber(input.runWorker, { detached: true }); yield* Ref.set(input.workerFiberRef, workerFiber); + yield* Effect.addFinalizer(() => + Ref.getAndSet(input.workerFiberRef, null).pipe( + Effect.flatMap((ownedWorkerFiber) => + ownedWorkerFiber === null ? Effect.void : interruptEffectWorker(ownedWorkerFiber), + ), + ), + ); yield* input.startRelay.pipe( Effect.onExit((exit) => { if (Exit.isSuccess(exit)) { From f6895ec3f0a9299bf19d117a61f7484f30aac245 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 22:52:25 +1000 Subject: [PATCH 08/12] fix(server): make the detached effect-worker handoff atomic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A detached worker was forked before workerFiberRef recorded it and its bounded-interrupt finalizer was registered, so an interrupt landing between the fork and the record orphaned a worker parked at activation — free to run against a closed runtime once activation released. The fork, ownership record, and finalizer registration now run uninterruptibly, so every spawned worker is either fully owned or never created. The regression sweeps hold offsets around the handoff and asserts the worker can never run after scope close plus activation release; it fails on the non-atomic version. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- apps/server/src/serverRuntimeStartup.test.ts | 102 +++++++++++++++++++ apps/server/src/serverRuntimeStartup.ts | 29 ++++-- 2 files changed, 120 insertions(+), 11 deletions(-) diff --git a/apps/server/src/serverRuntimeStartup.test.ts b/apps/server/src/serverRuntimeStartup.test.ts index 8c9d59b86d0c..3b4b1a006006 100644 --- a/apps/server/src/serverRuntimeStartup.test.ts +++ b/apps/server/src/serverRuntimeStartup.test.ts @@ -10,14 +10,69 @@ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Ref from "effect/Ref"; +import * as Scheduler from "effect/Scheduler"; import * as Scope from "effect/Scope"; import { TestClock } from "effect/testing"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; +import * as ServerActivation from "./serverActivation.ts"; import * as ServerConfig from "./config.ts"; import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; +// Stepping scheduler (mirrors ProviderSessionManager.test.ts): counts op +// boundaries per fiber and can hold the first spawned fiber reaching an armed +// offset, parking its resume task until `resumeSpawned` requeues it. Used to +// land an interrupt inside an otherwise unschedulable fork/record gap. +function makeSteppingScheduler() { + const tasks: Array<() => void> = []; + const counts = new Map, number>(); + const targets = new Map, number>(); + const held = new Map, () => void>(); + let spawnedTarget: number | undefined; + let capture: Fiber.Fiber | undefined; + const scheduler: Scheduler.Scheduler = { + executionMode: "sync", + shouldYield: (fiber) => { + const count = (counts.get(fiber) ?? 0) + 1; + counts.set(fiber, count); + if (targets.has(fiber)) return count === targets.get(fiber); + if (spawnedTarget === undefined || count !== spawnedTarget) return false; + spawnedTarget = undefined; + capture = fiber; + return true; + }, + makeDispatcher: () => ({ + scheduleTask: (task) => { + if (capture !== undefined) { + held.set(capture, task); + capture = undefined; + return; + } + tasks.push(task); + }, + flush: () => { + while (tasks.length > 0) tasks.shift()!(); + }, + }), + }; + const drain = Effect.gen(function* () { + for (let round = 0; round < 64; round++) { + while (tasks.length > 0) tasks.shift()!(); + yield* Effect.yieldNow; + if (tasks.length === 0) return; + } + }); + const holdSpawnedAt = (ops: number) => { + spawnedTarget = ops; + }; + const resumeSpawned = () => { + for (const task of held.values()) tasks.push(task); + held.clear(); + }; + return { scheduler, drain, holdSpawnedAt, resumeSpawned }; +} + it("uses the canonical Codex model for auto-bootstrap", () => { assert.deepEqual(ServerRuntimeStartup.getAutoBootstrapThreadModelSelection(), { instanceId: ProviderInstanceId.make("codex"), @@ -127,6 +182,53 @@ it.effect("scope close reaches teardown past a stalled uninterruptible worker", }), ); +it.effect("does not orphan a worker interrupted while it parks at activation", () => + Effect.gen(function* () { + // Sweep the hold offset so the interrupt lands at every scheduling point + // around the detached fork and the ownership record. Offsets that hold the + // start fiber between the fork and `Ref.set` would orphan the worker on a + // non-atomic sequence — the worker is released at the activation gate and + // would then run against the closed scope. + for (let offset = 1; offset <= 96; offset++) { + const stepper = makeSteppingScheduler(); + const scope = yield* Scope.make(); + const workerFiberRef = yield* Ref.make | null>(null); + const gate = yield* Deferred.make(); + const workerRan = yield* Ref.make(false); + + const start = yield* ServerRuntimeStartup.startEffectWorkerWithRelay({ + runWorker: Ref.set(workerRan, true), + startRelay: Effect.never, + workerFiberRef, + }).pipe( + Effect.provideService(Scope.Scope, scope), + Effect.provideService(ServerActivation.ServerActivation, Deferred.await(gate)), + Effect.provideService(Scheduler.Scheduler, stepper.scheduler), + Effect.forkDetach, + ); + // Armed after the fork so the test fiber itself is never captured. + stepper.holdSpawnedAt(offset); + yield* stepper.drain; + + yield* Effect.sync(() => start.interruptUnsafe()); + stepper.resumeSpawned(); + yield* stepper.drain; + + const closing = yield* Scope.close(scope, Exit.void).pipe(Effect.forkDetach); + while (closing.pollUnsafe() === undefined) { + yield* stepper.drain; + yield* Effect.yieldNow; + } + yield* Fiber.join(closing); + + yield* Deferred.succeed(gate, undefined); + yield* stepper.drain; + yield* Effect.yieldNow; + assert.isFalse(yield* Ref.get(workerRan)); + } + }), +); + it.effect("queues commands until startup signals readiness", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index 331d8d713f2b..49cb955e8a46 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -374,18 +374,25 @@ export const startEffectWorkerWithRelay = Effect.fn( readonly startRelay: Effect.Effect; readonly workerFiberRef: Ref.Ref | null>; }) { - // The worker is detached from the scope and bound by this bounded-interrupt - // finalizer instead: forkScoped's own finalizer waits on the worker's exit - // unboundedly, and it is registered after the teardown finalizer, so a + // The worker is detached from the scope and bound by the bounded-interrupt + // finalizer below instead: forkScoped's own finalizer waits on the worker's + // exit unboundedly, and it is registered after the teardown finalizer, so a // stalled worker would pin scope close before providerSessions.shutdown. - const workerFiber = yield* forkParkedFiber(input.runWorker, { detached: true }); - yield* Ref.set(input.workerFiberRef, workerFiber); - yield* Effect.addFinalizer(() => - Ref.getAndSet(input.workerFiberRef, null).pipe( - Effect.flatMap((ownedWorkerFiber) => - ownedWorkerFiber === null ? Effect.void : interruptEffectWorker(ownedWorkerFiber), - ), - ), + // Fork, ownership record, and finalizer registration run uninterruptibly — + // an interrupt between them would leave a detached worker nobody owns, + // parked at activation and free to run against a closed runtime. + yield* Effect.uninterruptible( + Effect.gen(function* () { + const workerFiber = yield* forkParkedFiber(input.runWorker, { detached: true }); + yield* Ref.set(input.workerFiberRef, workerFiber); + yield* Effect.addFinalizer(() => + Ref.getAndSet(input.workerFiberRef, null).pipe( + Effect.flatMap((ownedWorkerFiber) => + ownedWorkerFiber === null ? Effect.void : interruptEffectWorker(ownedWorkerFiber), + ), + ), + ); + }), ); yield* input.startRelay.pipe( Effect.onExit((exit) => { From e7c941f1144c58f37610bef9a2283b85b6ea7365 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 20 Sep 2026 09:05:00 +1000 Subject: [PATCH 09/12] fix(server): guard injectHistory against stale and releasing runtimes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit decorateRuntime carried injectHistory through the spread unguarded, so a runtime a release or replacement already retired could still invoke native thread/inject_items — e.g. a Codex history injection racing a close during ContextHandoffDelivery's pending-write suspension could land on a dying session yet still be durably marked injected. Decorate it like snapshot/rollback: live-runtime checks around the activity touch, then admission into inflightAdapterOps so a release drains the call before scope close and the ownership recheck refuses stale or replaced runtimes, plus the detached-thread refusal when the provider thread carries an appThreadId. Regression coverage proves a released or same-id-replaced runtime rejects injectHistory before the adapter runs (invocation counter stays 0) while a live runtime is admitted. Model: SWE-2 Max (Devin/Cognition) via Cursor harness in T3 Code. --- .../ProviderSessionManager.test.ts | 190 +++++++++++++++++- .../ProviderSessionManager.ts | 61 +++++- 2 files changed, 246 insertions(+), 5 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 7d32fe3d0dd9..ab092ebf16b7 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -270,6 +270,9 @@ function makeProviderAdapter( readonly ensureThread?: ( input: Parameters[0], ) => ReturnType; + readonly injectHistory?: ( + input: Parameters>[0], + ) => ReturnType>; } = {}, ): ProviderAdapterV2Shape { const instanceId = options.instanceId ?? ProviderInstanceId.make("codex"); @@ -375,6 +378,10 @@ function makeProviderAdapter( readThreadSnapshot: () => unimplemented("readThreadSnapshot unused in test"), rollbackThread: () => unimplemented("rollbackThread unused in test"), forkThread: () => unimplemented("forkThread unused in test"), + injectHistory: (injectInput) => + options.injectHistory === undefined + ? unimplemented("injectHistory unused in test") + : options.injectHistory(injectInput), } satisfies ProviderAdapterV2SessionRuntime; }), }; @@ -411,6 +418,9 @@ function makeTestLayer(input: { readonly ensureThread?: ( input: Parameters[0], ) => ReturnType; + readonly injectHistory?: ( + input: Parameters>[0], + ) => ReturnType>; readonly serverSettingsLayer?: ReturnType; readonly projectServiceLayer?: Layer.Layer; readonly fileSystemLayer?: Layer.Layer; @@ -439,6 +449,7 @@ function makeTestLayer(input: { ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }), ...(input.resumeThread === undefined ? {} : { resumeThread: input.resumeThread }), ...(input.ensureThread === undefined ? {} : { ensureThread: input.ensureThread }), + ...(input.injectHistory === undefined ? {} : { injectHistory: input.injectHistory }), }), ...(input.extraAdapters ?? []), ]; @@ -3982,6 +3993,7 @@ it.effect( it.effect("ProviderSessionManagerV2 rejects runtime thread attachments on a released session", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); + const injectCount = yield* Ref.make(0); yield* Effect.gen(function* () { const manager = yield* ProviderSessionManagerV2; const threadId = ThreadId.make("thread-released-attach"); @@ -4002,10 +4014,167 @@ it.effect("ProviderSessionManagerV2 rejects runtime thread attachments on a rele error._tag === "ProviderAdapterProtocolError" ? error.detail : "", "no longer running", ); - }).pipe(Effect.provide(makeTestLayer({ state, idleTimeoutMs: 3_600_000 }))); + // History injection mutates native thread state the same way: the + // released runtime's handle rejects before reaching the adapter — + // injectCount staying 0 proves the adapter call never ran. + const idAllocator = yield* IdAllocatorV2; + const injectError = yield* runtime.injectHistory!({ + providerThread: makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now: yield* DateTime.now, + }), + context: "handoff", + messages: [], + }).pipe(Effect.flip); + assert.equal(injectError._tag, "ProviderAdapterProtocolError"); + assert.equal(yield* Ref.get(injectCount), 0); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + injectHistory: () => Ref.update(injectCount, (count) => count + 1).pipe(Effect.as(true)), + }), + ), + ); + }), +); + +it.effect("ProviderSessionManagerV2 admits history injection on a live session", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const injectCount = yield* Ref.make(0); + yield* Effect.gen(function* () { + const threadId = ThreadId.make("thread-live-inject"); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* allocate; + const runtime = yield* open(providerSessionId); + const idAllocator = yield* IdAllocatorV2; + const injected = yield* runtime.injectHistory!({ + providerThread: makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now: yield* DateTime.now, + }), + context: "handoff", + messages: [], + }); + assert.isTrue(injected); + assert.equal(yield* Ref.get(injectCount), 1); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + injectHistory: () => Ref.update(injectCount, (count) => count + 1).pipe(Effect.as(true)), + }), + ), + ); }), ); +it.effect( + "ProviderSessionManagerV2 does not let an unanswered history injection block session close", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const injectStarted = yield* Deferred.make(); + const neverAnswered = yield* Deferred.make(); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const threadId = ThreadId.make("thread-inject-unanswered"); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* allocate; + const runtime = yield* open(providerSessionId); + const idAllocator = yield* IdAllocatorV2; + const injecting = yield* runtime.injectHistory!({ + providerThread: makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now: yield* DateTime.now, + }), + context: "handoff", + messages: [], + }).pipe(Effect.forkChild); + yield* Deferred.await(injectStarted); + const closing = yield* manager.close(providerSessionId).pipe(Effect.forkChild); + yield* TestClock.adjust("20 seconds"); + yield* Fiber.join(closing); + // Teardown abandoned the unanswered call at the drain bound instead of + // waiting on it — this test-forked inject fiber stays parked until the + // request settles, but no longer pins cleanup. + assert.equal((yield* Ref.get(state)).closeCount, 1); + yield* Deferred.succeed(neverAnswered, undefined); + yield* Fiber.join(injecting); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + injectHistory: () => + Deferred.succeed(injectStarted, undefined).pipe( + Effect.andThen(Deferred.await(neverAnswered)), + Effect.as(true), + ), + }), + ), + ); + }), +); + +it.effect( + "ProviderSessionManagerV2 lets caller interruption settle an unanswered history injection", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const injectStarted = yield* Deferred.make(); + const neverAnswered = yield* Deferred.make(); + yield* Effect.gen(function* () { + const manager = yield* ProviderSessionManagerV2; + const threadId = ThreadId.make("thread-inject-interrupted"); + const { allocate, open } = yield* makeThreadSessionFixture(threadId); + const providerSessionId = yield* allocate; + const runtime = yield* open(providerSessionId); + const idAllocator = yield* IdAllocatorV2; + const injecting = yield* runtime.injectHistory!({ + providerThread: makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now: yield* DateTime.now, + }), + context: "handoff", + messages: [], + }).pipe(Effect.forkChild); + yield* Deferred.await(injectStarted); + // The request never settles: worker cancellation (EffectWorker's + // raceFirst waits on the losing execution's interruption) must still + // reach the caller, and the masked ensuring must settle the admission + // record so close below does not wait on an abandoned record. + yield* Fiber.interrupt(injecting); + assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(injecting))); + yield* manager.close(providerSessionId); + assert.equal((yield* Ref.get(state)).closeCount, 1); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + injectHistory: () => + Deferred.succeed(injectStarted, undefined).pipe( + Effect.andThen(Deferred.await(neverAnswered)), + Effect.as(true), + ), + }), + ), + ); + }), +); + it.effect( "ProviderSessionManagerV2 rejects a runtime attachment whose session closed mid-prepare", () => @@ -4752,6 +4921,7 @@ it.effect( () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); + const injectCount = yield* Ref.make(0); yield* Effect.gen(function* () { const manager = yield* ProviderSessionManagerV2; const ownerThreadId = ThreadId.make("thread-stale-runtime-owner"); @@ -4812,10 +4982,26 @@ it.effect( .resumeThread({ providerThread: { ...providerThread, appThreadId: null } }) .pipe(Effect.flip); assert.equal(orphanResumeError._tag, "ProviderAdapterProtocolError"); + const injectError = yield* runtimeA.injectHistory!({ + providerThread, + context: "handoff", + messages: [], + }).pipe(Effect.flip); + assert.equal(injectError._tag, "ProviderAdapterProtocolError"); + assert.equal(yield* Ref.get(injectCount), 0); assert.equal((yield* Ref.get(state)).resumeCount, 0); assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); assert.equal((yield* Ref.get(state)).closeCount, 1); - }).pipe(Effect.provide(makeTestLayer({ state, idleTimeoutMs: 3_600_000 }))); + }).pipe( + Effect.provide( + makeTestLayer({ + state, + idleTimeoutMs: 3_600_000, + injectHistory: () => + Ref.update(injectCount, (count) => count + 1).pipe(Effect.as(true)), + }), + ), + ); }), ); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 5681f7fcac20..86d708714097 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2298,8 +2298,12 @@ export const layerWithOptions = ( readonly operation: Effect.Effect; readonly drainTimeout: Duration.Duration | undefined; readonly requiredThreadId?: ThreadId; + // Request/response mutations create no resource that ownership must + // track: an unanswered one must yield to caller interruption, not wait + // for a settle that may never arrive. + readonly interruptibleOperation?: boolean; }): Effect.Effect => - Effect.uninterruptibleMask((_restore) => + Effect.uninterruptibleMask((restore) => Effect.gen(function* () { const key = sessionKey(input.providerSessionId); const record = { @@ -2338,14 +2342,20 @@ export const layerWithOptions = ( : "The provider session is no longer running.", }); } - // The operation runs uninterruptibly inside the mask: adapters + // Acquisitions run uninterruptibly inside the mask: adapters // acquire native resources through Effect.tryPromise (Cursor's // Agent.create/resume), whose interruption detaches the promise // rather than cancelling it. Signalling `done` at interrupt time // would let the drain complete while a late result still lands // with no cleanup owner — an interruptible caller instead waits // for the acquisition to settle into tracked ownership. - return yield* input.operation.pipe( + // interruptibleOperation opts out for calls whose only outcome is + // settled-or-ambiguous (history injection): interruption reaches + // the request, and the uninterruptible ensuring still settles the + // admission record so neither the drain nor the caller wedges. + return yield* ( + input.interruptibleOperation === true ? restore(input.operation) : input.operation + ).pipe( Effect.ensuring( Effect.sync(() => { const ops = inflightAdapterOps.get(key); @@ -2821,6 +2831,51 @@ export const layerWithOptions = ( Effect.andThen(runtime.uploadFeedback!(input)), ), }), + ...(runtime.injectHistory === undefined + ? {} + : { + injectHistory: (input: Parameters>[0]) => + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }).pipe( + Effect.andThen( + observeActivity(providerSessionId, touchActivity(providerSessionId, runtime)), + ), + Effect.andThen( + requireLiveRuntime({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + }), + ), + // Injection mutates an already-attached native thread + // without a turn's busy pin, so it admits like snapshot and + // rollback: a release drains the call before scope close + // and the ownership recheck refuses a runtime a detach or + // replacement already retired. + Effect.andThen( + runResourceCreatingAdapterOp({ + providerSessionId, + expectedRuntime: runtime, + driver: runtime.driver, + operation: runtime.injectHistory!(input), + ...(input.providerThread.appThreadId === null + ? {} + : { requiredThreadId: input.providerThread.appThreadId }), + // A request/response mutation, not an acquisition: an + // unanswered request must not pin teardown — the drain + // bound abandons the wait and scope close reclaims the + // call — and must not pin the caller either, so the + // request itself restores interruption while the + // admission and its settle stay masked. + drainTimeout: Duration.millis(ADAPTER_OP_DRAIN_TIMEOUT_MS), + interruptibleOperation: true, + }), + ), + ), + }), }; }; From 9ec24fe4ecea32cfdb9d8482c0c48e4cc0139b7e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:39:11 +1000 Subject: [PATCH 10/12] test(server): cover shared credential cleanup after restack --- .../Adapters/ClaudeAdapterV2.test.ts | 2 ++ .../Layers/ProviderAuthService.test.ts | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 3f67a97e5d8a..f20dbbf6b462 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -7147,6 +7147,7 @@ describe("ClaudeAdapterV2 session cleanup", () => { ), }), forkSession: () => Effect.die("unused"), + subagentLaunchToolUseId: () => Effect.succeed(null), assertComplete: Effect.void, }, }); @@ -7221,6 +7222,7 @@ describe("ClaudeAdapterV2 session cleanup", () => { ), }), forkSession: () => Effect.die("unused"), + subagentLaunchToolUseId: () => Effect.succeed(null), assertComplete: Effect.void, }, }); diff --git a/apps/server/src/provider/Layers/ProviderAuthService.test.ts b/apps/server/src/provider/Layers/ProviderAuthService.test.ts index 4eceb39a2617..8279370b497b 100644 --- a/apps/server/src/provider/Layers/ProviderAuthService.test.ts +++ b/apps/server/src/provider/Layers/ProviderAuthService.test.ts @@ -74,6 +74,7 @@ const makeHarness = Effect.fn("ProviderAuthService.test.makeHarness")(function* input: { enabled?: boolean; stopFailures?: number; + failingInstance?: ProviderInstanceId; logoutError?: ProviderSetupError; sharedCredentials?: boolean; sharedBusy?: boolean; @@ -205,7 +206,7 @@ const makeHarness = Effect.fn("ProviderAuthService.test.makeHarness")(function* actions.push(`close-instance:${id}`); yield* input.beforeStop?.(id) ?? Effect.void; closeAttempts += 1; - if (closeAttempts <= (input.stopFailures ?? 0)) { + if (closeAttempts <= (input.stopFailures ?? 0) || id === input.failingInstance) { return yield* Effect.fail( new ProviderSessionCloseError({ providerSessionId: ProviderSessionId.make( @@ -336,6 +337,21 @@ describe("ProviderAuthService", () => { assert.isBelow(actions.indexOf("invalidate-shared"), actions.indexOf("native-logout")); }), ); + it.effect("does not invalidate shared credentials when a peer fails to close", () => + Effect.gen(function* () { + const { service, actions, closedInstances } = yield* makeHarness({ + sharedCredentials: true, + failingInstance: otherInstanceId, + }); + const error = yield* service.logout({ instanceId }).pipe(Effect.flip); + assert.strictEqual(error.operation, "stopSessions"); + assert.instanceOf(error.cause, ProviderSessionCloseError); + assert.deepStrictEqual(closedInstances, [instanceId]); + assert.notInclude(actions, "invalidate-shared"); + assert.notInclude(actions, "native-logout"); + }), + ); + it.effect("rejects overlapping changes to a shared sign-in", () => Effect.gen(function* () { const { service, actions } = yield* makeHarness({ From 25c191714ecf5101f15f7961d594eb3bbc76559c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:19:52 +1000 Subject: [PATCH 11/12] test(server): batch the exhaustive attachment scheduler sweep --- .../src/orchestration-v2/ProviderSessionManager.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index ab092ebf16b7..805ca8dc9e64 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -6182,9 +6182,9 @@ it.effect( }), ); -it.effect( - "ProviderSessionManagerV2 keeps the attach lock through the adapter call when a detach queues", - () => +it.effect.each([1, 131, 261, 391])( + "ProviderSessionManagerV2 keeps the attach lock through the adapter call when a detach queues (offsets starting at %i)", + (firstOffset) => Effect.gen(function* () { // Every op offset past the credential-issuance gate gets its own fresh // manager: the attach is suspended exactly there, and the queued detach @@ -6195,7 +6195,7 @@ it.effect( // reaches well past the bookkeeping-to-admission boundary (~op 505 on // the unfused implementation — credential issuance dominates the // ~440 ops between the park point and the lock release). - for (let offset = 1; offset <= 520; offset += 1) { + for (let offset = firstOffset; offset < firstOffset + 130; offset += 1) { const state = yield* Ref.make(emptyState); const issuing = yield* Deferred.make(); const issueGate = yield* Deferred.make(); From 89c4265334ff0c2f9924454ad1dc35f0dc74e287 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:41:44 +1000 Subject: [PATCH 12/12] fix(server): preserve typed detach release errors --- .../orchestration-v2/ProviderSessionManager.test.ts | 7 ++++--- .../src/orchestration-v2/ProviderSessionManager.ts | 12 ++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 805ca8dc9e64..78a7f180f51f 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -9734,10 +9734,11 @@ it.effect("ProviderSessionManagerV2 bounds the detach wait on a stalled in-fligh const detaching = yield* manager .detach({ providerSessionId, threadId: attachThreadId, revokeMcpCredential: true }) - .pipe(Effect.exit, Effect.forkChild); + .pipe(Effect.flip, Effect.forkChild); yield* TestClock.adjust("30 seconds"); - const detachExit = yield* Fiber.join(detaching); - assert.equal(detachExit._tag, "Failure"); + const detachError = yield* Fiber.join(detaching); + assert.ok(detachError._tag === "ProviderSessionReleaseError"); + assert.equal(detachError.reason, "runtime_error"); yield* Deferred.succeed(issueGate, undefined); assert.isTrue(Exit.isSuccess(yield* Fiber.join(attaching))); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 86d708714097..f488a363d84c 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -4170,15 +4170,19 @@ export const layerWithOptions = ( } yield* scheduleIdleRelease(input.providerSessionId, detached.value.entry.runtime); }).pipe( - Effect.catchCause((cause) => - Effect.fail( + Effect.catchCause((cause) => { + const error = Cause.findErrorOption(cause); + if (Option.isSome(error) && isProviderSessionReleaseError(error.value)) { + return Effect.fail(error.value); + } + return Effect.fail( new ProviderSessionReleaseError({ providerSessionId: input.providerSessionId, reason: "manual_shutdown", cause, }), - ), - ), + ); + }), ), ).pipe( Effect.onExit(() =>