diff --git a/packages/core/src/location-activity.ts b/packages/core/src/location-activity.ts index 0425e9acab04..f1385d5a9892 100644 --- a/packages/core/src/location-activity.ts +++ b/packages/core/src/location-activity.ts @@ -1,9 +1,11 @@ export * as LocationActivity from "./location-activity.js" -import { Clock, Context, Duration, Effect, Layer, RcMap, Schema } from "effect" +import { Clock, Context, Duration, Effect, Layer, Option, RcMap, Schema } from "effect" import { Bus } from "./bus.js" +import { Form } from "./form.js" import { Location } from "./location.js" import { LocationServiceMap } from "./location-service-map.js" +import { Permission } from "./permission.js" import { SessionEvent } from "./session/event.js" import { SessionExecution } from "./session/execution.js" import { SessionStore } from "./session/store.js" @@ -30,6 +32,25 @@ export function layer(options: { readonly timeToLive?: Duration.Input; readonly entries.set(key(ref), { ref, expiresAt: clock.currentTimeMillisUnsafe() + timeToLive }) }) + // A pending form or permission request is a person being asked to answer, and + // waiting for them produces no durable activity. Protect the whole location + // rather than its waiting owners: the session holding the question is often a + // child of another session that is idle only because it awaits that answer. + const awaitingReply = (ref: Location.Ref) => + Effect.gen(function* () { + const context = yield* locations.contextEffectOption(ref) + if (Option.isNone(context)) return false + const forms = Context.getOption(context.value, Form.Service) + if (Option.isSome(forms) && (yield* forms.value.list()).length > 0) return true + const permissions = Context.getOption(context.value, Permission.Service) + return Option.isSome(permissions) && (yield* permissions.value.list()).length > 0 + }).pipe( + // Borrow only long enough to read the pending sets, and let a detached or + // failed graph evict: it can no longer be holding anyone's answer. + Effect.scoped, + Effect.catchCause(() => Effect.succeed(false)), + ) + const unsubscribe = yield* bus.listen((event) => { if (!isSessionEvent(event)) return Effect.void const location = event.location @@ -55,6 +76,9 @@ export function layer(options: { readonly timeToLive?: Duration.Input; readonly expired, (entry) => Effect.gen(function* () { + // Hold the deadline open for the next interval instead of interrupting + // the wait. An explicit stop still cancels the question immediately. + if (yield* awaitingReply(entry.ref)) return yield* touch(entry.ref) const owners = active.flatMap((session) => session && key(session.location) === key(entry.ref) ? [session] : [], ) diff --git a/packages/core/test/location-activity.test.ts b/packages/core/test/location-activity.test.ts index 4ca873afbc01..4713f2bb7948 100644 --- a/packages/core/test/location-activity.test.ts +++ b/packages/core/test/location-activity.test.ts @@ -10,6 +10,7 @@ import { Form } from "@opencode/core/form" import { Location } from "@opencode/core/location" import { LocationActivity } from "@opencode/core/location-activity" import { LocationServiceMap, type LocationServices } from "@opencode/core/location-services" +import { Permission } from "@opencode/core/permission" import { Project } from "@opencode/core/project" import { ProjectTable } from "@opencode/core/project/sql" import { AbsolutePath } from "@opencode/core/schema" @@ -22,17 +23,33 @@ import { SessionStore } from "@opencode/core/session/store" import { Workspace } from "@opencode/core/workspace" import { testEffect } from "./lib/effect" -// Keep real execution ownership, location caching, forms, and eviction. The fixture -// runner waits on a form instead of making a model request before asking a question. +const ref = LocationServiceMap.canonical({ directory: AbsolutePath.make("/project") }) + +// The fixture runner waits on human input instead of making a model request, so a +// pending question or permission produces no durable activity while it is outstanding. +// A "hung" session never waits on a human and stands in for stalled model work. +function waiting(sessionID: Session.ID, forms: Form.Interface, permissions: Permission.Interface) { + if (sessionID.includes("hung")) return Effect.never + if (sessionID.includes("permission")) + return permissions + .assert({ id: Permission.ID.create(), sessionID, action: "edit", resources: ["src/index.ts"] }) + .pipe(Effect.orDie) + return forms.ask({ sessionID, title: "Questions", fields: [{ key: "runtime", type: "string" }] }).pipe(Effect.orDie) +} + +// Keep real execution ownership, location caching, forms, and eviction. Permission +// ownership is per location like forms, but the real service also needs agents, saved +// rules, and plugin hooks, so this fixture keeps only its pending-request behavior. const locations = Layer.effect( LocationServiceMap.Service, Effect.gen(function* () { const bus = yield* Bus.Service const map = yield* LayerMap.make( - (ref: Location.Ref) => - // The fixture only exercises these three Location services. + (ref: Location.Ref) => { + const pending = new Map }>() + // The fixture only exercises these four Location services. // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion - Layer.merge( + return Layer.merge( Layer.succeed( Location.Service, Location.Service.of({ @@ -45,27 +62,52 @@ const locations = Layer.effect( SessionRunner.Service, Effect.gen(function* () { const forms = yield* Form.Service + const permissions = yield* Permission.Service return SessionRunner.Service.of({ drain: ({ sessionID }) => - forms - .ask({ - sessionID, - title: "Questions", - fields: [{ key: "runtime", type: "string" }], - }) - .pipe( - Effect.orDie, - Effect.as(SessionRunner.DrainResult.Complete()), - Effect.onInterrupt(() => Effect.sleep("5 minutes")), - ), + waiting(sessionID, forms, permissions).pipe( + Effect.as(SessionRunner.DrainResult.Complete()), + Effect.onInterrupt(() => Effect.sleep("5 minutes")), + ), }) }), ), ).pipe( - Layer.provideMerge(Form.layer), + Layer.provideMerge( + Layer.merge( + Form.layer, + Layer.mock(Permission.Service, { + close: Effect.void, + assert: (input) => + Effect.gen(function* () { + const request = Permission.Request.make({ + id: input.id ?? Permission.ID.create(), + sessionID: input.sessionID, + action: input.action, + resources: input.resources, + }) + const deferred = yield* Deferred.make() + pending.set(request.id, { request, deferred }) + yield* bus.publish(Permission.Event.Asked, request) + yield* Deferred.await(deferred).pipe( + Effect.onInterrupt(() => Effect.sync(() => pending.delete(request.id))), + ) + }), + reply: (input) => + Effect.suspend(() => { + const item = pending.get(input.requestID) + if (!item) return Effect.void + pending.delete(input.requestID) + return Deferred.succeed(item.deferred, undefined).pipe(Effect.asVoid) + }), + list: () => Effect.sync(() => Array.from(pending.values(), (item) => item.request)), + }), + ), + ), Layer.provide(Layer.succeed(Bus.Service, bus)), Layer.fresh, - ) as unknown as Layer.Layer, + ) as unknown as Layer.Layer + }, { idleTimeToLive: Duration.infinity }, ) return { @@ -100,122 +142,244 @@ const it = testEffect( ), ) -describe("LocationActivity eviction", () => { - for (const [count, admission] of [ - [1, "none"], - [2, "none"], - [1, "other"], - [1, "same"], - ] as const) { - const newWork = admission !== "none" - it.effect( - `interrupts ${count} waiting executions before eviction (${admission} session admitted during cleanup)`, - () => - Effect.gen(function* () { - const db = (yield* Database.Service).db - const bus = yield* Bus.Service - const map = yield* LocationServiceMap.Service - const execution = yield* SessionExecution.Service - const store = yield* SessionStore.Service - const sessionIDs = Array.from({ length: count }, (_, index) => - Session.ID.make(`ses_waiting_question_${index}`), - ) - const newcomer = admission === "same" ? sessionIDs[0] : Session.ID.make("ses_new_question") - const ref = LocationServiceMap.canonical({ directory: AbsolutePath.make("/project") }) - const idle = Location.Ref.make({ directory: ref.directory, workspaceID: Workspace.ID.make("wrk_idle") }) - yield* db - .insert(ProjectTable) - .values({ id: Project.ID.global, worktree: ref.directory, sandboxes: [] }) - .run() - .pipe(Effect.orDie) - yield* db - .insert(SessionTable) - .values( - Array.from(new Set([...sessionIDs, newcomer]), (sessionID) => ({ - id: sessionID, - project_id: Project.ID.global, - slug: "question", - directory: ref.directory, - title: "Waiting question", - version: "test", - })), - ) - .run() - .pipe(Effect.orDie) - - const created = yield* Deferred.make() - const newCreated = yield* Deferred.make() - const pending: Form.Info[] = [] - const interrupted: SessionEvent.Execution.Interrupted["data"][] = [] - const unsubscribe = yield* bus.listen((event) => - Effect.gen(function* () { - if (event.type === SessionEvent.Execution.Interrupted.type) { - interrupted.push(Schema.decodeUnknownSync(SessionEvent.Execution.Interrupted.data)(event.data)) - } - if (event.type !== Form.Event.Created.type) return - pending.push(Schema.decodeUnknownSync(Form.Event.Created.data)(event.data).form) - if (pending.length === count) yield* Deferred.succeed(created, undefined) - if (pending.length > count) yield* Deferred.succeed(newCreated, undefined) - }), - ) - yield* Effect.addFinalizer(() => unsubscribe) - const running = yield* Effect.forEach(sessionIDs, (sessionID) => - execution.resume(sessionID).pipe(Effect.exit, Effect.forkScoped), - ) - yield* Effect.addFinalizer(() => - Effect.forEach([...sessionIDs, newcomer], (sessionID) => execution.interrupt(sessionID)).pipe( - Effect.andThen(TestClock.adjust("5 minutes")), - ), - ) - yield* Deferred.await(created) - const context = yield* map.contextEffect(ref).pipe(Effect.scoped) - const forms = Context.get(context, Form.Service) - expect((yield* store.listSuspended()).toSorted()).toEqual(sessionIDs.toSorted()) - yield* Location.Service.pipe(Effect.provide(map.get(idle)), Effect.scoped) - - // Human input produces no durable activity while the question is pending. - yield* TestClock.adjust("1 minute") - yield* TestClock.adjust("62 minutes") - // Interruption has cancelled each question, but slow cleanup still owns the graph. - expect(Array.from(yield* execution.active).toSorted()).toEqual(sessionIDs.toSorted()) - expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([ref]) - expect(yield* forms.list()).toEqual([]) - for (const form of pending) expect(yield* forms.state(form.id)).toEqual({ status: "cancelled" }) - - if (newWork) { - yield* execution.wake(newcomer) - if (admission === "other") yield* Deferred.await(newCreated) - } - yield* TestClock.adjust("5 minutes") - if (newWork) yield* Deferred.await(newCreated) - const results = yield* Effect.forEach(running, Fiber.join) - expect(results.every((exit) => exit._tag === "Failure")).toBe(true) - expect(Array.from(yield* execution.active)).toEqual(newWork ? [newcomer] : []) - expect(yield* store.listSuspended()).toEqual(newWork ? [newcomer] : []) - expect(interrupted.toSorted((a, b) => a.sessionID.localeCompare(b.sessionID))).toEqual( - sessionIDs.map((sessionID) => ({ sessionID, reason: "inactivity" })), - ) - expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual(newWork ? [ref] : []) - if (newWork) { - expect(yield* forms.list({ sessionID: newcomer })).toEqual([pending[count]]) - if (admission === "same") { - const later = LocationServiceMap.canonical({ directory: AbsolutePath.make("/later") }) - yield* Location.Service.pipe(Effect.provide(map.get(later)), Effect.scoped) - yield* TestClock.adjust("30 minutes") - // Keep fresh work active while a different graph reaches its own deadline. - yield* bus.publish(SessionEvent.Execution.Started, { sessionID: newcomer }, { location: ref }) - yield* TestClock.adjust("32 minutes") - expect(Array.from(yield* execution.active)).toEqual([newcomer]) - expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([ref]) - } - yield* execution.interrupt(newcomer) - yield* TestClock.adjust("5 minutes") - yield* execution.awaitIdle(newcomer) - yield* TestClock.adjust("62 minutes") - expect(yield* store.listSuspended()).toEqual([]) - expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([]) - } - }), +// `later` sessions are seeded but not started, so a test can admit one mid-cleanup. +function harness(sessionIDs: ReadonlyArray, later: ReadonlyArray = []) { + return Effect.gen(function* () { + const db = (yield* Database.Service).db + const bus = yield* Bus.Service + const execution = yield* SessionExecution.Service + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: ref.directory, sandboxes: [] }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values( + [...sessionIDs, ...later].map((sessionID) => ({ + id: sessionID, + project_id: Project.ID.global, + slug: "question", + directory: ref.directory, + title: "Waiting question", + version: "test", + })), + ) + .run() + .pipe(Effect.orDie) + + const forms: Form.Info[] = [] + const requests: Permission.Request[] = [] + const interrupted: SessionEvent.Execution.Interrupted["data"][] = [] + const waited = yield* Deferred.make() + const unsubscribe = yield* bus.listen((event) => + Effect.gen(function* () { + if (event.type === SessionEvent.Execution.Interrupted.type) { + interrupted.push(Schema.decodeUnknownSync(SessionEvent.Execution.Interrupted.data)(event.data)) + } + if (event.type === Form.Event.Created.type) { + forms.push(Schema.decodeUnknownSync(Form.Event.Created.data)(event.data).form) + yield* Deferred.succeed(waited, undefined).pipe(Effect.asVoid) + } + if (event.type === Permission.Event.Asked.type) { + requests.push(Schema.decodeUnknownSync(Permission.Request)(event.data)) + yield* Deferred.succeed(waited, undefined).pipe(Effect.asVoid) + } + }), + ) + yield* Effect.addFinalizer(() => unsubscribe) + // Release anything still parked on human input, then let slow cleanup settle. + yield* Effect.addFinalizer(() => + Effect.forEach([...sessionIDs, ...later], (sessionID) => execution.interrupt(sessionID)).pipe( + Effect.andThen(TestClock.adjust("5 minutes")), + ), + ) + const running = yield* Effect.forEach(sessionIDs, (sessionID) => + execution.resume(sessionID).pipe(Effect.exit, Effect.forkScoped), + ) + return { forms, requests, interrupted, waited, running } + }) +} + +const reaped = (events: ReadonlyArray) => + events.filter((event) => event.reason === "inactivity").map((event) => event.sessionID) + +describe("LocationActivity human waits", () => { + it.effect("keeps the location cached while a question stays pending", () => + Effect.gen(function* () { + const sessionID = Session.ID.make("ses_waiting_form") + const map = yield* LocationServiceMap.Service + const execution = yield* SessionExecution.Service + const store = yield* SessionStore.Service + const watch = yield* harness([sessionID]) + yield* Deferred.await(watch.waited) + const forms = Context.get(yield* map.contextEffect(ref).pipe(Effect.scoped), Form.Service) + expect(yield* store.listSuspended()).toEqual([sessionID]) + + // Human input produces no durable activity, so twelve hours of sweeps pass + // with nothing to refresh the location's deadline. + yield* Effect.forEach(Array.from({ length: 24 }), () => TestClock.adjust("30 minutes"), { discard: true }) + expect(reaped(watch.interrupted)).toEqual([]) + expect(Array.from(yield* execution.active)).toEqual([sessionID]) + expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([ref]) + expect((yield* forms.list()).map((form) => form.id)).toEqual([watch.forms[0].id]) + expect(yield* forms.state(watch.forms[0].id)).toEqual({ status: "pending" }) + + // The original question still accepts its answer and the execution resumes. + yield* forms.reply({ id: watch.forms[0].id, answer: { runtime: "bun" } }) + expect((yield* Fiber.join(watch.running[0]))._tag).toEqual("Success") + expect(yield* forms.state(watch.forms[0].id)).toEqual({ status: "answered", answer: { runtime: "bun" } }) + expect(Array.from(yield* execution.active)).toEqual([]) + + // Nothing waits on a human now, so the idle location reaches its deadline. + yield* TestClock.adjust("62 minutes") + expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([]) + expect(reaped(watch.interrupted)).toEqual([]) + }), + ) + + it.effect("keeps the location cached while a permission request stays pending", () => + Effect.gen(function* () { + const sessionID = Session.ID.make("ses_waiting_permission") + const map = yield* LocationServiceMap.Service + const execution = yield* SessionExecution.Service + const watch = yield* harness([sessionID]) + yield* Deferred.await(watch.waited) + const permissions = Context.get(yield* map.contextEffect(ref).pipe(Effect.scoped), Permission.Service) + + yield* Effect.forEach(Array.from({ length: 24 }), () => TestClock.adjust("30 minutes"), { discard: true }) + expect(reaped(watch.interrupted)).toEqual([]) + expect(Array.from(yield* execution.active)).toEqual([sessionID]) + expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([ref]) + expect(yield* permissions.list()).toEqual([watch.requests[0]]) + + yield* permissions.reply({ requestID: watch.requests[0].id, reply: "once" }) + expect((yield* Fiber.join(watch.running[0]))._tag).toEqual("Success") + expect(yield* permissions.list()).toEqual([]) + expect(Array.from(yield* execution.active)).toEqual([]) + + yield* TestClock.adjust("62 minutes") + expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([]) + expect(reaped(watch.interrupted)).toEqual([]) + }), + ) + + it.effect("still stops a pending question when the user interrupts it", () => + Effect.gen(function* () { + const sessionID = Session.ID.make("ses_cancelled_form") + const map = yield* LocationServiceMap.Service + const execution = yield* SessionExecution.Service + const store = yield* SessionStore.Service + const watch = yield* harness([sessionID]) + yield* Deferred.await(watch.waited) + const forms = Context.get(yield* map.contextEffect(ref).pipe(Effect.scoped), Form.Service) + yield* TestClock.adjust("3 hours") + + yield* execution.interrupt(sessionID) + yield* TestClock.adjust("5 minutes") + expect((yield* Fiber.join(watch.running[0]))._tag).toEqual("Failure") + expect(yield* forms.state(watch.forms[0].id)).toEqual({ status: "cancelled" }) + expect(Array.from(yield* execution.active)).toEqual([]) + expect(yield* store.listSuspended()).toEqual([]) + expect(reaped(watch.interrupted)).toEqual([]) + + yield* TestClock.adjust("62 minutes") + expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([]) + }), + ) + + for (const count of [1, 2, 3] as const) { + it.effect(`still reaps ${count} executions that are not waiting on a human`, () => + Effect.gen(function* () { + const sessionIDs = Array.from({ length: count }, (_, index) => Session.ID.make(`ses_hung_execution_${index}`)) + const map = yield* LocationServiceMap.Service + const execution = yield* SessionExecution.Service + const store = yield* SessionStore.Service + const idle = Location.Ref.make({ directory: ref.directory, workspaceID: Workspace.ID.make("wrk_idle") }) + const watch = yield* harness(sessionIDs) + yield* Location.Service.pipe(Effect.provide(map.get(idle)), Effect.scoped) + yield* TestClock.adjust("1 minute") + expect(Array.from(yield* execution.active).toSorted()).toEqual(sessionIDs.toSorted()) + + yield* TestClock.adjust("62 minutes") + // Interruption has stopped every drain, but slow cleanup still owns the graph. + expect(Array.from(yield* execution.active).toSorted()).toEqual(sessionIDs.toSorted()) + expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([ref]) + yield* TestClock.adjust("5 minutes") + const results = yield* Effect.forEach(watch.running, Fiber.join) + expect(results.every((exit) => exit._tag === "Failure")).toBe(true) + expect(reaped(watch.interrupted).toSorted()).toEqual(sessionIDs.toSorted()) + expect(yield* store.listSuspended()).toEqual([]) + expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([]) + }), ) } + + it.effect("retains the cached graph for work admitted while a reaped owner cleans up", () => + Effect.gen(function* () { + const reaping = Session.ID.make("ses_hung_reaped") + const admitted = Session.ID.make("ses_hung_admitted") + const map = yield* LocationServiceMap.Service + const execution = yield* SessionExecution.Service + const store = yield* SessionStore.Service + const watch = yield* harness([reaping], [admitted]) + yield* TestClock.adjust("1 minute") + const graph = yield* map.contextEffect(ref).pipe(Effect.scoped) + expect(Array.from(yield* execution.active)).toEqual([reaping]) + + yield* TestClock.adjust("62 minutes") + // Interruption has stopped the drain, but slow cleanup still owns the graph. + expect(Array.from(yield* execution.active)).toEqual([reaping]) + yield* execution.wake(admitted) + yield* TestClock.adjust("1 minute") + expect(Array.from(yield* execution.active).toSorted()).toEqual([admitted, reaping].toSorted()) + + yield* TestClock.adjust("4 minutes") + expect((yield* Fiber.join(watch.running[0]))._tag).toEqual("Failure") + expect(reaped(watch.interrupted)).toEqual([reaping]) + // The newcomer now owns the graph the reaped session borrowed, so eviction is + // abandoned and the same services stay cached rather than being rebuilt. + expect(Array.from(yield* execution.active)).toEqual([admitted]) + expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([ref]) + expect(yield* map.contextEffect(ref).pipe(Effect.scoped)).toBe(graph) + + // Abandoning that eviction only renews the deadline; the newcomer waits on no + // human, so it reaches its own deadline like any other stalled execution. + yield* TestClock.adjust("62 minutes") + yield* TestClock.adjust("5 minutes") + expect(reaped(watch.interrupted).toSorted()).toEqual([admitted, reaping].toSorted()) + expect(yield* store.listSuspended()).toEqual([]) + expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([]) + }), + ) + + it.effect("protects a co-owner of a directory where another session waits", () => + Effect.gen(function* () { + const child = Session.ID.make("ses_form_child") + const parent = Session.ID.make("ses_hung_parent") + const map = yield* LocationServiceMap.Service + const execution = yield* SessionExecution.Service + const watch = yield* harness([child, parent]) + yield* Deferred.await(watch.waited) + const forms = Context.get(yield* map.contextEffect(ref).pipe(Effect.scoped), Form.Service) + yield* TestClock.adjust("1 minute") + expect(Array.from(yield* execution.active).toSorted()).toEqual([child, parent].toSorted()) + + // The parent publishes no activity of its own, so per-owner filtering would + // reap it while its child is still blocked on the answer. + yield* TestClock.adjust("3 hours") + expect(reaped(watch.interrupted)).toEqual([]) + expect(Array.from(yield* execution.active).toSorted()).toEqual([child, parent].toSorted()) + expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([ref]) + + // Answering lifts the protection; the still-hung parent is reaped as before. + yield* forms.reply({ id: watch.forms[0].id, answer: { runtime: "bun" } }) + expect((yield* Fiber.join(watch.running[0]))._tag).toEqual("Success") + yield* TestClock.adjust("62 minutes") + yield* TestClock.adjust("5 minutes") + expect(reaped(watch.interrupted)).toEqual([parent]) + expect(Array.from(yield* RcMap.keys(map.rcMap))).toEqual([]) + }), + ) })