From 03ac9e154aabb18f9ec540e3adf9db3f1154b579 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 25 Sep 2026 14:04:34 -0700 Subject: [PATCH 1/4] perf(server): shutdown no longer rewrites every stopped session row stopAll runs on every quit, update, and restart. It read the whole provider_session_runtime table twice and upserted every row, including thousands of rows that were already stopped. Each upsert is a SELECT plus an UPSERT, and it also reset lastSeenAt on all of them. Now it reads the table once and skips rows that are already stopped with no active turn. The stopped_all analytics sessionCount now counts only the sessions this shutdown stops, not every row ever written. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../provider/Layers/ProviderService.test.ts | 72 +++++++++++++++++++ .../src/provider/Layers/ProviderService.ts | 21 +++++- 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index 547935a51e3c..0dc624774594 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -648,6 +648,78 @@ it.effect("ProviderServiceLive catches stopAll failures during shutdown", () => }), ); +it.effect("ProviderServiceLive shutdown leaves settled session rows untouched", () => + Effect.gen(function* () { + const recordedAnalytics = makeRecordingAnalytics(); + const codex = makeFakeCodexAdapter(); + const persistence = yield* Layer.build( + ProviderSessionDirectoryLive.pipe( + Layer.provide(ProviderSessionRuntime.layer.pipe(Layer.provide(SqlitePersistenceMemory))), + ), + ); + const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory.pipe( + Effect.provide(persistence), + ); + const seed = (threadId: ThreadId, status: "running" | "stopped", activeTurnId: TurnId | null) => + directory.upsert({ + threadId, + provider: CODEX_DRIVER, + providerInstanceId: codexInstanceId, + status, + runtimePayload: { cwd: "/repo", activeTurnId }, + }); + const readBindings = directory + .listBindings() + .pipe( + Effect.map((bindings) => new Map(bindings.map((binding) => [binding.threadId, binding]))), + ); + const settledId = asThreadId("shutdown-settled"); + const liveId = asThreadId("shutdown-live"); + const stoppedWithTurnId = asThreadId("shutdown-stopped-with-turn"); + yield* seed(settledId, "stopped", null); + yield* seed(liveId, "running", asTurnId("live-turn")); + yield* seed(stoppedWithTurnId, "stopped", asTurnId("stale-turn")); + const settledBefore = (yield* readBindings).get(settledId); + assert(settledBefore !== undefined); + + const scope = yield* Scope.make(); + yield* Layer.build( + makeProviderServiceLive().pipe( + Layer.provide(NodeServices.layer), + Layer.provide(Layer.succeed(ProviderSessionDirectory.ProviderSessionDirectory, directory)), + Layer.provide( + Layer.succeed( + ProviderAdapterRegistry.ProviderAdapterRegistry, + makeStaticInstanceRegistry([[codexInstanceId, codex.adapter]]), + ), + ), + Layer.provide(defaultServerSettingsLayer), + Layer.provide(serverConfigTestLayer), + Layer.provide(recordedAnalytics.layer), + Layer.provide( + Layer.succeed( + ProviderEventLoggers.ProviderEventLoggers, + ProviderEventLoggers.NoOpProviderEventLoggers, + ), + ), + ), + ).pipe(Scope.provide(scope)); + yield* TestClock.adjust("1 minute"); + yield* Scope.close(scope, Exit.void); + + const byThread = yield* readBindings; + assert.deepStrictEqual(byThread.get(settledId), settledBefore); + for (const threadId of [liveId, stoppedWithTurnId]) { + const binding = byThread.get(threadId); + assert.equal(binding?.status, "stopped"); + assert.propertyVal(binding?.runtimePayload, "activeTurnId", null); + assert.propertyVal(binding?.runtimePayload, "lastRuntimeEvent", "provider.stopAll"); + } + const [stoppedAll] = recordedAnalytics.eventsByName("provider.sessions.stopped_all"); + assert.equal(stoppedAll?.properties?.sessionCount, 2); + }).pipe(Effect.provide(NodeServices.layer)), +); + it.effect("ProviderServiceLive flushes deferred completions during shutdown", () => Effect.gen(function* () { const recordedAnalytics = makeRecordingAnalytics(); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 5e5052d5ef23..462b8fb36b4d 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -427,6 +427,17 @@ function readPersistedCwd( return trimmed.length > 0 ? trimmed : undefined; } +/** Stopped rows with no active turn are settled; shutdown leaves them untouched. */ +function isSettledBinding(binding: ProviderSessionDirectory.ProviderRuntimeBinding): boolean { + const payload = binding.runtimePayload; + const hasActiveTurn = + payload !== null && + typeof payload === "object" && + "activeTurnId" in payload && + payload.activeTurnId != null; + return binding.status === "stopped" && !hasActiveTurn; +} + const dieOnMissingBindingInstanceId = ( operation: string, payload: { @@ -2331,7 +2342,6 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( return [completed, state] as const; }); yield* recordCompletedTurnProperties(properties); - const threadIds = yield* directory.listThreadIds(); const currentAdapters = yield* getAdapterEntries; const activeSessions = yield* Effect.forEach(currentAdapters, ([instanceId, adapter]) => adapter.listSessions().pipe( @@ -2362,7 +2372,12 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( yield* Effect.forEach(currentAdapters, ([, adapter]) => adapter.stopAll()).pipe(Effect.asVoid); yield* McpSessionRegistry.revokeAllActiveMcpCredentials(); McpProviderSession.clearAllMcpProviderSessions(); - const bindings = yield* directory.listBindings().pipe(Effect.orElseSucceed(() => [])); + // Stopped rows stay for their resume cursors, so long-lived installs hold + // thousands. Only rewrite the ones this shutdown actually stops. + const bindings = yield* directory.listBindings().pipe( + Effect.map((all) => all.filter((binding) => !isSettledBinding(binding))), + Effect.orElseSucceed(() => []), + ); yield* Effect.forEach(bindings, (binding) => Effect.gen(function* () { const providerInstanceId = dieOnMissingBindingInstanceId( @@ -2383,7 +2398,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }), ).pipe(Effect.asVoid); yield* analytics.record("provider.sessions.stopped_all", { - sessionCount: threadIds.length, + sessionCount: bindings.length, }); yield* analytics.flush; }); From e3e84b208fa7b24de0edb2423d0ed3c0c1b02f8c Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 25 Sep 2026 14:08:25 -0700 Subject: [PATCH 2/4] refactor(server): match payload guard shape in isSettledBinding Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/provider/Layers/ProviderService.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 462b8fb36b4d..0a82ce6ee179 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -429,13 +429,10 @@ function readPersistedCwd( /** Stopped rows with no active turn are settled; shutdown leaves them untouched. */ function isSettledBinding(binding: ProviderSessionDirectory.ProviderRuntimeBinding): boolean { + if (binding.status !== "stopped") return false; const payload = binding.runtimePayload; - const hasActiveTurn = - payload !== null && - typeof payload === "object" && - "activeTurnId" in payload && - payload.activeTurnId != null; - return binding.status === "stopped" && !hasActiveTurn; + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return true; + return !("activeTurnId" in payload) || payload.activeTurnId == null; } const dieOnMissingBindingInstanceId = ( From 6fcda84a11f325d4e8de673d6a1e93fafbb7061a Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 25 Sep 2026 18:07:25 -0700 Subject: [PATCH 3/4] test(server): name the running shutdown row for what it is Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/provider/Layers/ProviderService.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index 0dc624774594..ec96b64f3b44 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -674,10 +674,10 @@ it.effect("ProviderServiceLive shutdown leaves settled session rows untouched", Effect.map((bindings) => new Map(bindings.map((binding) => [binding.threadId, binding]))), ); const settledId = asThreadId("shutdown-settled"); - const liveId = asThreadId("shutdown-live"); + const runningId = asThreadId("shutdown-running"); const stoppedWithTurnId = asThreadId("shutdown-stopped-with-turn"); yield* seed(settledId, "stopped", null); - yield* seed(liveId, "running", asTurnId("live-turn")); + yield* seed(runningId, "running", asTurnId("running-turn")); yield* seed(stoppedWithTurnId, "stopped", asTurnId("stale-turn")); const settledBefore = (yield* readBindings).get(settledId); assert(settledBefore !== undefined); @@ -709,7 +709,7 @@ it.effect("ProviderServiceLive shutdown leaves settled session rows untouched", const byThread = yield* readBindings; assert.deepStrictEqual(byThread.get(settledId), settledBefore); - for (const threadId of [liveId, stoppedWithTurnId]) { + for (const threadId of [runningId, stoppedWithTurnId]) { const binding = byThread.get(threadId); assert.equal(binding?.status, "stopped"); assert.propertyVal(binding?.runtimePayload, "activeTurnId", null); From f53ac44ca4730b6a8d144d1a0632704bdff9d0e1 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 25 Sep 2026 19:35:41 -0700 Subject: [PATCH 4/4] fix(server): report shutdown stops as stoppedSessionCount sessionCount on provider.sessions.stopped_all used to count every row in provider_session_runtime. It now counts only the sessions this shutdown stops. A new property name keeps the old and new values out of one series, the same way surface was added next to the legacy clientType. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/provider/Layers/ProviderService.test.ts | 2 +- apps/server/src/provider/Layers/ProviderService.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index ec96b64f3b44..242d0b55aa2f 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -716,7 +716,7 @@ it.effect("ProviderServiceLive shutdown leaves settled session rows untouched", assert.propertyVal(binding?.runtimePayload, "lastRuntimeEvent", "provider.stopAll"); } const [stoppedAll] = recordedAnalytics.eventsByName("provider.sessions.stopped_all"); - assert.equal(stoppedAll?.properties?.sessionCount, 2); + assert.equal(stoppedAll?.properties?.stoppedSessionCount, 2); }).pipe(Effect.provide(NodeServices.layer)), ); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 0a82ce6ee179..12f8869da486 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -2394,8 +2394,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( }); }), ).pipe(Effect.asVoid); + // Not `sessionCount`: that older property counted every row, so a new name + // keeps the two meanings in separate series. yield* analytics.record("provider.sessions.stopped_all", { - sessionCount: bindings.length, + stoppedSessionCount: bindings.length, }); yield* analytics.flush; });