From abed95437246c9c92dbd6c7a7a22decf1e06700f Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Fri, 2 Oct 2026 19:34:23 -0600 Subject: [PATCH] refactor: remove in-product Relay orchestration Pylon's bridge to the Relay Orchestrator Claude plugin was a stand-in while upstream's orchestrator was unavailable. Upstream shipped orchestrator v2, so this removes the bridge rather than carrying two delegation implementations. Deletes RelayWorkerBridge, RelayCli and relayMcpConfig, and unhooks their optional-service wiring from server.ts, ws.ts, OrchestrationReactor and ProviderRuntimeIngestion. The Claude and Codex adapters no longer inject a relay MCP server. Drops the relay-only wire fields (source, relaySequence, relayPriorUsage). cancellable and watchable stay: they are generic capability flags that Prime and native agents also use. Historical events keep decoding because Struct ignores excess properties, covered by a new fold regression test. Collapses the logic those fields carried: - ProjectionSnapshotQuery drops the relay lifecycle pinning CTEs. - ThreadBackgroundLiveness loses its relay set; no work outlives its session, so clearThreadLiveness drops the thread entry outright. - subagentRuntime loses the attempt/sequence fence and cumulative usage accounting; canCancelSessionAgent loses its detached-controls parameter. - planBackgroundAgentStop reduced algebraically to canInterruptParent once detached workers were gone, so ChatView uses that directly. The managed relay tunnels (packages/shared/src/relay*, infra/relay, apps/server/src/relay, deploy-relay.yml) are a different subsystem and are untouched. Migration 063 stays: it is registered and already applied. --- .../src/features/threads/ThreadComposer.tsx | 18 +- .../features/threads/sessionAgentMenu.test.ts | 13 - .../src/features/threads/sessionAgentMenu.ts | 14 +- apps/mobile/src/lib/threadActivity.test.ts | 128 -- apps/mobile/src/lib/threadActivity.ts | 24 +- .../src/state/use-thread-composer-state.ts | 1 - .../Layers/OrchestrationReactor.ts | 3 - .../Layers/ProjectionSnapshotQuery.test.ts | 126 -- .../Layers/ProjectionSnapshotQuery.ts | 19 - .../Layers/ProviderRuntimeIngestion.ts | 10 - .../server/src/orchestration/RelayCli.test.ts | 114 - apps/server/src/orchestration/RelayCli.ts | 57 - .../orchestration/RelayWorkerBridge.test.ts | 1822 ---------------- .../src/orchestration/RelayWorkerBridge.ts | 1858 ----------------- .../ThreadBackgroundLiveness.test.ts | 13 +- .../orchestration/ThreadBackgroundLiveness.ts | 30 +- .../src/provider/Layers/ClaudeAdapter.test.ts | 21 - .../src/provider/Layers/ClaudeAdapter.ts | 21 +- .../src/provider/Layers/CodexAdapter.test.ts | 37 - .../src/provider/Layers/CodexAdapter.ts | 21 +- apps/server/src/provider/relayMcpConfig.ts | 19 - apps/server/src/server.ts | 2 - apps/server/src/testUtils/shardWeights.json | 2 - apps/server/src/ws.ts | 42 +- apps/web/src/components/AgentsPanel.test.tsx | 144 -- apps/web/src/components/AgentsPanel.tsx | 20 +- apps/web/src/components/ChatView.logic.ts | 37 - apps/web/src/components/ChatView.tsx | 77 +- apps/web/src/session-logic.test.ts | 130 -- apps/web/src/session-logic.ts | 24 +- docs/user/thread-sidebar.md | 14 - .../src/state/sessionAgentLiveActivity.ts | 1 - .../src/state/subagentRuntime.test.ts | 335 +-- .../src/state/subagentRuntime.ts | 143 +- packages/contracts/src/providerRuntime.ts | 6 - 35 files changed, 125 insertions(+), 5221 deletions(-) delete mode 100644 apps/server/src/orchestration/RelayCli.test.ts delete mode 100644 apps/server/src/orchestration/RelayCli.ts delete mode 100644 apps/server/src/orchestration/RelayWorkerBridge.test.ts delete mode 100644 apps/server/src/orchestration/RelayWorkerBridge.ts delete mode 100644 apps/server/src/provider/relayMcpConfig.ts diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index c922e99b0b..dea16dae33 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -891,9 +891,8 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer props.selectedThread.session?.runtimeMode === "full-access" && supportsSessionAgentCancel(activeSessionProviderStatus); const canCancelAgent = useCallback( - (agent: RuntimeSubagent) => - canCancelSessionAgent(agent, canCancelSessionAgents, props.connectionState === "connected"), - [canCancelSessionAgents, props.connectionState], + (agent: RuntimeSubagent) => canCancelSessionAgent(agent, canCancelSessionAgents), + [canCancelSessionAgents], ); const canMessageSessionAgents = sessionAgentReady && @@ -939,7 +938,6 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer agents: props.sessionAgents, provider: activeSessionProviderStatus, canCancel: canCancelSessionAgents, - detachedControlsAvailable: props.connectionState === "connected", canMessage: canMessageSessionAgents, cancellingAgentIds, onCancel: props.onCancelSessionAgent, @@ -950,7 +948,6 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer agents: props.sessionAgents, provider: activeSessionProviderStatus, canCancel: canCancelSessionAgents, - detachedControlsAvailable: props.connectionState === "connected", canMessage: canMessageSessionAgents, cancellingAgentIds, onCancel: props.onCancelSessionAgent, @@ -984,7 +981,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer if ( control.scopeKey !== expectedScopeKey || current === undefined || - !canCancelSessionAgent(current, control.canCancel, control.detachedControlsAvailable) || + !canCancelSessionAgent(current, control.canCancel) || control.cancellingAgentIds.has(agentId) ) { return; @@ -1076,12 +1073,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer const agent = control.agents.find((candidate) => candidate.id === action.agentId); if (!agent || !isActiveSubagentStatus(agent.status)) return; if (action.kind === "live-activity") { - if ( - !canWatchSessionAgentActivity || - agent.watchable === false || - agent.source === "relay" || - agent.kind === "workflow" - ) + if (!canWatchSessionAgentActivity || agent.watchable === false || agent.kind === "workflow") return; setLiveActivitySelection({ agentId: agent.id, scopeKey: control.scopeKey }); return; @@ -1095,7 +1087,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer return; } if ( - !canCancelSessionAgent(agent, control.canCancel, control.detachedControlsAvailable) || + !canCancelSessionAgent(agent, control.canCancel) || control.cancellingAgentIds.has(agent.id) ) return; diff --git a/apps/mobile/src/features/threads/sessionAgentMenu.test.ts b/apps/mobile/src/features/threads/sessionAgentMenu.test.ts index 5c65b52704..e010bdc5be 100644 --- a/apps/mobile/src/features/threads/sessionAgentMenu.test.ts +++ b/apps/mobile/src/features/threads/sessionAgentMenu.test.ts @@ -117,17 +117,4 @@ describe("session agent menu", () => { }); expect(settled).toEqual([]); }); - - it("keeps detached worker stop available after the parent session and hides unsupported actions", () => { - const actions = buildSessionAgentMenuActions({ - scopeKey: "remote:thread-1", - agents: [agent({ id: "relay:job-1", source: "relay", cancellable: true, watchable: false })], - canMessage: true, - canCancel: false, - canCancelAgent: (candidate) => candidate.source === "relay" && candidate.cancellable === true, - canWatchLiveActivity: true, - cancellingAgentIds: new Set(), - }); - expect(actions.map((action) => action.title)).toEqual(["Stop Nested reviewer"]); - }); }); diff --git a/apps/mobile/src/features/threads/sessionAgentMenu.ts b/apps/mobile/src/features/threads/sessionAgentMenu.ts index e6398e24c1..b5c159628b 100644 --- a/apps/mobile/src/features/threads/sessionAgentMenu.ts +++ b/apps/mobile/src/features/threads/sessionAgentMenu.ts @@ -29,12 +29,7 @@ export function buildSessionAgentMenuActions(input: { return input.agents.flatMap((agent) => { if (!isActiveSubagentStatus(agent.status)) return []; const actions: SessionAgentMenuAction[] = []; - if ( - input.canWatchLiveActivity && - agent.watchable !== false && - agent.source !== "relay" && - agent.kind !== "workflow" - ) { + if (input.canWatchLiveActivity && agent.watchable !== false && agent.kind !== "workflow") { actions.push({ id: `${LIVE_ACTIVITY_PREFIX}${encodeURIComponent(input.scopeKey)}:${encodeURIComponent(agent.id)}`, title: `Live activity · ${agent.title}`, @@ -42,12 +37,7 @@ export function buildSessionAgentMenuActions(input: { image: "eye", }); } - if ( - input.canMessage && - agent.source !== "relay" && - agent.messageable && - agent.kind !== "workflow" - ) { + if (input.canMessage && agent.messageable && agent.kind !== "workflow") { actions.push({ id: `${MESSAGE_PREFIX}${encodeURIComponent(input.scopeKey)}:${encodeURIComponent(agent.id)}`, title: `Message ${agent.title}`, diff --git a/apps/mobile/src/lib/threadActivity.test.ts b/apps/mobile/src/lib/threadActivity.test.ts index 90c0d3eb89..ce89e0304c 100644 --- a/apps/mobile/src/lib/threadActivity.test.ts +++ b/apps/mobile/src/lib/threadActivity.test.ts @@ -290,33 +290,6 @@ function makeThread( }; } -describe("Relay ownership receipts", () => { - it("keeps internal bindings out of the mobile conversation feed", () => { - const thread = makeThread({ - id: ThreadId.make("thread-relay"), - projectId: ProjectId.make("project-relay"), - title: "Relay proof", - activities: [ - makeActivity({ - id: EventId.make("relay-binding"), - kind: "relay.binding", - summary: "Relay binding", - createdAt: "2026-04-01T00:00:00.000Z", - payload: { id: "job-1", environmentId: "env-1", threadId: "thread-relay" }, - }), - makeActivity({ - id: EventId.make("relay-activation"), - kind: "relay.activation", - summary: "Relay activation", - createdAt: "2026-04-01T00:00:01.000Z", - payload: { id: "job-1", attempt: 2, toolCallId: "tool-2" }, - }), - ], - }); - expect(buildThreadFeed(thread)).toEqual([]); - }); -}); - describe("buildThreadFeed", () => { it("reuses unchanged feed and presentation rows during an assistant text update", () => { const completedTurnId = TurnId.make("completed-turn"); @@ -3667,70 +3640,6 @@ describe("quiet timeline: nested agents", () => { }, ); - it("places recovered Relay cards beside their original dispatch after history pagination", () => { - const oldTime = "2026-09-24T10:00:00.000Z"; - const restartTime = "2026-09-27T11:00:00.000Z"; - const receipt = makeActivity({ - id: EventId.make("original-dispatch"), - kind: "tool.completed", - summary: "Relay dispatch", - turnId: TurnId.make("old-turn"), - createdAt: oldTime, - sequence: 1, - payload: { toolCallId: "dispatch", status: "completed", itemType: "mcp_tool_call" }, - }); - const start = makeActivity({ - id: EventId.make("recovered-start"), - kind: "task.started", - summary: "Relay worker started", - turnId: TurnId.make("old-turn"), - createdAt: restartTime, - sequence: 100, - payload: { - taskId: "relay:job-old", - agentKind: "agent", - source: "relay", - toolUseId: "dispatch", - timelineBypass: true, - }, - }); - const recent = makeActivity({ - id: EventId.make("recent-work"), - kind: "tool.completed", - summary: "Recent work", - turnId: TurnId.make("new-turn"), - createdAt: "2026-09-27T10:00:00.000Z", - sequence: 2, - payload: { toolCallId: "new-dispatch", status: "completed" }, - }); - const rows = (activities: ReadonlyArray) => - buildThreadFeed( - makeThread({ - id: ThreadId.make("relay-thread"), - projectId: ProjectId.make("project-1"), - title: "Relay recovery", - activities, - }), - ).flatMap((entry) => (entry.type === "activity-group" ? entry.activities : [])); - expect(rows([start, recent]).find((entry) => entry.workEntry.agentSpawn)?.createdAt).toBe( - restartTime, - ); - const recoveredRows = rows([receipt, recent, start]); - expect(recoveredRows.find((entry) => entry.workEntry.agentSpawn)?.createdAt).toBe(oldTime); - expect(recoveredRows.findIndex((entry) => entry.workEntry.agentSpawn)).toBeLessThan( - recoveredRows.findIndex((entry) => entry.id === "recent-work"), - ); - expect(start.createdAt).toBe(restartTime); - const native = { - ...start, - id: EventId.make("native-start"), - payload: { taskId: "native", agentKind: "agent", toolUseId: "dispatch" }, - }; - expect(rows([receipt, native]).find((entry) => entry.workEntry.agentSpawn)?.createdAt).toBe( - restartTime, - ); - }); - it("folds bypassed Claude workflow members into the coordinator's batch and settles them with it", () => { const turnId = TurnId.make("turn-workflow"); const at = (seconds: number) => `2026-04-01T00:00:${String(seconds).padStart(2, "0")}.000Z`; @@ -3809,43 +3718,6 @@ describe("quiet timeline: nested agents", () => { expect(rows[0]?.getFullDetail()).toBe("Reviewer 0 · completed\nReviewer 1 · completed"); }); - it("groups Relay panel members with their coordinator in one mobile spawn card", () => { - const panel = "relay-panel:panel-22222222-2222-4222-8222-222222222222"; - const turnId = TurnId.make("turn-relay-panel"); - const activities = [panel, `${panel}:member:0`, `${panel}:member:1`].map((taskId, index) => - makeActivity({ - id: EventId.make(`relay-panel-${index}`), - kind: "task.started", - summary: index === 0 ? "Relay panel started" : "Relay worker started", - createdAt: `2026-04-01T00:00:0${index}.000Z`, - turnId, - payload: { - taskId, - taskType: index === 0 ? "local_workflow" : "subagent", - workflowName: "Relay panel", - agentKind: "agent", - source: "relay", - status: "running", - timelineBypass: true, - }, - }), - ); - const thread = makeThread({ - id: ThreadId.make("thread-relay-panel"), - projectId: ProjectId.make("project-1"), - title: "Relay panel", - activities, - }); - const rows = buildThreadFeed(thread).flatMap((entry) => - entry.type === "activity-group" ? entry.activities : [], - ); - expect(rows).toHaveLength(1); - expect(rows[0]?.workEntry.agentSpawn).toMatchObject({ - workflowId: panel, - agentTaskIds: [panel, `${panel}:member:0`, `${panel}:member:1`], - }); - }); - it("summarizes a spawn card from the newest member report and the batch outcome", () => { type Member = NonNullable["agents"][number]; const member = (title: string, status: Member["status"], detail: string, seconds: number) => diff --git a/apps/mobile/src/lib/threadActivity.ts b/apps/mobile/src/lib/threadActivity.ts index 98207a7e23..19b349596a 100644 --- a/apps/mobile/src/lib/threadActivity.ts +++ b/apps/mobile/src/lib/threadActivity.ts @@ -436,23 +436,11 @@ function deriveWorkLogEntries( activities: ReadonlyArray, ): DerivedWorkLogEntry[] { const ordered = Arr.sort(activities, activityOrder); - const toolOrigins = new Map(); - for (const activity of ordered) { - if (activity.kind !== "tool.started" && activity.kind !== "tool.completed") continue; - const payload = asRecord(activity.payload); - const toolCallId = - asTrimmedString(payload?.toolCallId) ?? asTrimmedString(asRecord(payload?.data)?.toolCallId); - if (!toolCallId) continue; - const key = `${activity.turnId ?? ""}:${toolCallId}`; - if (!toolOrigins.has(key)) toolOrigins.set(key, activity.createdAt); - } // Resolved across the thread: a task that names a background type stays // background even when a later bare terminal row carries the agent stamp. const backgroundTaskIds = collectBackgroundTaskIds(ordered); const entries: DerivedWorkLogEntry[] = []; for (const activity of foldUserInputActivities(ordered)) { - // Ownership receipts feed Relay recovery and control routing, not the transcript. - if (activity.kind === "relay.binding" || activity.kind === "relay.activation") continue; // The setup card owns its snapshot, including failed and cancelled outcomes. if ( isWorktreeSetupActivity(activity.kind) && @@ -488,15 +476,7 @@ function deriveWorkLogEntries( if (isNoContentRuntimeWarning(activity)) continue; if (isPlanBoundaryToolActivity(activity)) continue; if (isAgentInternalActivity(activity, backgroundTaskIds)) continue; - const entry = toDerivedWorkLogEntry(activity, backgroundTaskIds); - const payload = asRecord(activity.payload); - const origin = - payload?.source === "relay" && typeof payload.toolUseId === "string" - ? toolOrigins.get(`${activity.turnId ?? ""}:${payload.toolUseId}`) - : undefined; - // Match web: saved Relay rows backfilled at restart belong beside their - // original dispatch, even when that receipt arrives in a later history page. - entries.push(origin && origin < entry.createdAt ? { ...entry, createdAt: origin } : entry); + entries.push(toDerivedWorkLogEntry(activity, backgroundTaskIds)); } return collapseDerivedWorkLogEntries(entries); } @@ -699,8 +679,6 @@ function toDerivedWorkLogEntry( */ function agentSpawnGroupKey(entry: DerivedWorkLogEntry): string { const taskId = entry.taskId ?? ""; - const relayMemberSlot = taskId.startsWith("relay-panel:") ? taskId.indexOf(":member:") : -1; - if (relayMemberSlot !== -1) return `wf:${taskId.slice(0, relayMemberSlot)}`; const workflowSlot = taskId.indexOf(":wf:"); if (workflowSlot !== -1) return `wf:${taskId.slice(0, workflowSlot)}`; if (entry.isWorkflowCoordinator) return `wf:${taskId}`; diff --git a/apps/mobile/src/state/use-thread-composer-state.ts b/apps/mobile/src/state/use-thread-composer-state.ts index 34b6c41fe5..070ff7093a 100644 --- a/apps/mobile/src/state/use-thread-composer-state.ts +++ b/apps/mobile/src/state/use-thread-composer-state.ts @@ -1189,7 +1189,6 @@ export function useThreadComposerState() { session?.runtimeMode === "full-access" && (session.status === "ready" || session.status === "running") && supportsSessionAgentCancel(provider), - selectedEnvironmentRuntime?.connectionState === "connected", ) ) { return false; diff --git a/apps/server/src/orchestration/Layers/OrchestrationReactor.ts b/apps/server/src/orchestration/Layers/OrchestrationReactor.ts index 605cb89e79..fb21ac8a65 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationReactor.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationReactor.ts @@ -14,7 +14,6 @@ import * as PullRequestSyncReactor from "../PullRequestSyncReactor.ts"; import * as ProjectSettingsReactor from "../ProjectSettingsReactor.ts"; import * as ThreadPullRequestReactor from "../ThreadPullRequestReactor.ts"; import * as AgentAwarenessRelay from "../../relay/AgentAwarenessRelay.ts"; -import { RelayWorkerBridge } from "../RelayWorkerBridge.ts"; import * as Option from "effect/Option"; export const makeOrchestrationReactor = Effect.gen(function* () { @@ -27,7 +26,6 @@ export const makeOrchestrationReactor = Effect.gen(function* () { const threadPullRequestReactor = yield* ThreadPullRequestReactor.ThreadPullRequestReactor; const projectSettingsReactor = yield* ProjectSettingsReactor.ProjectSettingsReactor; const agentAwarenessRelay = yield* AgentAwarenessRelay.AgentAwarenessRelay; - const relayWorkerBridge = Option.getOrUndefined(yield* Effect.serviceOption(RelayWorkerBridge)); const start: OrchestrationReactorShape["start"] = Effect.fn("start")(function* () { yield* providerRuntimeIngestion.start(); @@ -39,7 +37,6 @@ export const makeOrchestrationReactor = Effect.gen(function* () { yield* threadSettlementReactor.start(); yield* pullRequestSyncReactor.start(); yield* agentAwarenessRelay.start(); - if (relayWorkerBridge) yield* relayWorkerBridge.start; }); return { diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts index adac3d4cd8..38d84e3f1c 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts @@ -3122,132 +3122,6 @@ projectionSnapshotLayer("ProjectionSnapshotQuery windowed thread detail", (it) = }), ); - it.effect("projects native background work separately from detached Relay workers", () => - Effect.gen(function* () { - yield* seedFanOutThread(); - const query = yield* ProjectionSnapshotQuery; - const liveness = yield* ThreadBackgroundLiveness.ThreadBackgroundLivenessService; - const sql = yield* SqlClient.SqlClient; - liveness.recordTaskLiveness({ - threadId: threadW, - taskId: "relay:job-11111111-1111-4111-8111-111111111111", - taskType: "subagent", - source: "relay", - status: "running", - kind: "started", - }); - const relayShell = Option.getOrThrow(yield* query.getThreadShellById(threadW)); - assert.equal(relayShell.backgroundLiveness, "working"); - assert.equal(relayShell.nativeBackgroundWork, false); - assert.equal( - (yield* query.getShellSnapshot()).threads.find((thread) => thread.id === threadW) - ?.nativeBackgroundWork, - false, - ); - - liveness.recordTaskLiveness({ - threadId: threadW, - taskId: "native-agent", - taskType: "subagent", - status: "running", - kind: "started", - }); - assert.equal( - (yield* query.getShellSnapshot()).threads.find((thread) => thread.id === threadW) - ?.nativeBackgroundWork, - true, - ); - yield* sql`UPDATE projection_threads SET archived_at = '2026-03-02T00:00:00.000Z' WHERE thread_id = ${threadW}`; - assert.equal( - (yield* query.getArchivedShellSnapshot()).threads.find((thread) => thread.id === threadW) - ?.nativeBackgroundWork, - true, - ); - }), - ); - - it.effect("pins a detached Relay agent beyond the 500 activity window until it settles", () => - Effect.gen(function* () { - yield* seedFanOutThread(); - const snapshotQuery = yield* ProjectionSnapshotQuery; - const sql = yield* SqlClient.SqlClient; - yield* sql`DELETE FROM projection_thread_activities`; - yield* sql` - INSERT INTO projection_thread_activities ( - activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at - ) VALUES - ( - 'relay-prior-complete', 'thread-w', 'turn-1', 'info', 'task.completed', - 'Relay worker completed', - '{"taskId":"relay:job-11111111-1111-4111-8111-111111111111","taskType":"subagent","agentKind":"agent","source":"relay","attempt":1,"relaySequence":2,"status":"completed","typedUsage":{"totalTokens":150,"inputTokens":100}}', - 1, '2026-03-01T00:00:00.000Z' - ), - ( - 'relay-old-progress', 'thread-w', 'turn-1', 'info', 'task.progress', - 'Relay worker', - '{"taskId":"relay:job-11111111-1111-4111-8111-111111111111","taskType":"subagent","agentKind":"agent","source":"relay","attempt":2,"relaySequence":1,"status":"running","typedUsage":{"totalTokens":40,"inputTokens":25},"relayPriorUsage":{"totalTokens":150,"inputTokens":100}}', - 2, '2026-03-01T00:00:01.000Z' - ) - `; - yield* sql` - WITH RECURSIVE activity_rows(sequence) AS ( - SELECT 3 UNION ALL SELECT sequence + 1 FROM activity_rows WHERE sequence < 503 - ) - INSERT INTO projection_thread_activities ( - activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at - ) - SELECT printf('newer-%04d', sequence), 'thread-w', 'turn-5', 'info', - 'tool.completed', 'Newer parent activity', '{}', sequence, - '2026-03-01T00:04:00.000Z' - FROM activity_rows - `; - - const raw = yield* snapshotQuery.getThreadDetailById(threadW); - assert.equal(raw._tag, "Some"); - if (raw._tag === "Some") { - assert.equal(raw.value.activities.length, 501); - assert.ok( - raw.value.activities.some((activity) => activity.id === asEventId("relay-old-progress")), - ); - assert.ok( - !raw.value.activities.some( - (activity) => activity.id === asEventId("relay-prior-complete"), - ), - ); - } - const client = yield* snapshotQuery.getThreadDetailSnapshot(threadW, { turnLimit: 1 }); - assert.equal(client._tag, "Some"); - if (client._tag === "Some") { - assert.ok( - client.value.thread.activities.some( - (activity) => activity.id === asEventId("relay-old-progress"), - ), - ); - assert.ok( - !client.value.thread.activities.some( - (activity) => activity.id === asEventId("relay-prior-complete"), - ), - ); - } - - yield* sql` - UPDATE projection_thread_activities SET kind = 'task.completed', - payload_json = '{"taskId":"relay:job-11111111-1111-4111-8111-111111111111","taskType":"subagent","agentKind":"agent","source":"relay","attempt":2,"relaySequence":2,"status":"completed","typedUsage":{"totalTokens":60},"relayPriorUsage":{"totalTokens":150,"inputTokens":100}}' - WHERE activity_id = 'relay-old-progress' - `; - const settled = yield* snapshotQuery.getThreadDetailById(threadW); - assert.equal(settled._tag, "Some"); - if (settled._tag === "Some") { - assert.equal(settled.value.activities.length, 500); - assert.ok( - !settled.value.activities.some( - (activity) => activity.id === asEventId("relay-old-progress"), - ), - ); - } - }), - ); - it.effect("bounds activity hydration and preserves unresolved requests", () => Effect.gen(function* () { yield* seedFanOutThread(); diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index 007a39945a..c8fde348cf 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -2285,22 +2285,6 @@ pending_approval_requests AS ( activity.activity_id DESC LIMIT 1 ), - relay_lifecycle AS ( - SELECT activity.activity_id, activity.kind, - ROW_NUMBER() OVER ( - PARTITION BY json_extract(activity.payload_json, '$.taskId') - ORDER BY activity.sequence DESC, activity.created_at DESC, activity.activity_id DESC - ) AS task_order - FROM projection_thread_activities AS activity - WHERE activity.thread_id = ${threadId} - AND activity.kind IN ('task.started', 'task.progress', 'task.updated', 'task.completed') - AND json_extract(activity.payload_json, '$.source') = 'relay' - AND json_type(activity.payload_json, '$.taskId') = 'text' - ), - active_relay_lifecycle AS ( - SELECT activity_id FROM relay_lifecycle - WHERE task_order = 1 AND kind != 'task.completed' - ), pinned_activity_ids AS ( SELECT activity_id FROM pending_approval_activities @@ -2313,9 +2297,6 @@ pending_approval_requests AS ( UNION ALL SELECT activity_id FROM latest_plan_activity - UNION ALL - SELECT activity_id - FROM active_relay_lifecycle ) `; diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index 6362e44f44..10edac8ad8 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -61,7 +61,6 @@ import { ProjectionThreadProposedPlanRepository } from "../../persistence/Servic import { ProjectionThreadProposedPlanRepositoryLive } from "../../persistence/Layers/ProjectionThreadProposedPlans.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; import { ThreadBackgroundLivenessService } from "../ThreadBackgroundLiveness.ts"; -import { RelayWorkerBridge } from "../RelayWorkerBridge.ts"; import { ThreadPlanProgressService } from "../ThreadPlanProgress.ts"; import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; import { @@ -530,10 +529,8 @@ function taskLinkageActivityFields(payload: Record): Record - fs.writeFileString(registry, JSON.stringify({ version: 2, plugins })); - yield* writeRegistry({ - "relay-orchestrator@relay-local": [ - { scope: "user", installPath: plugin }, - { scope: "project", installPath: path.join(home, "old-project-plugin") }, - ], - }); - return { fs, path, home, plugin, cli, registry, writeRegistry }; -}); - -it.layer(NodeServices.layer)("Relay CLI discovery", (it) => { - it.effect("finds the registered user plugin in the server's Claude home", () => - Effect.gen(function* () { - const f = yield* fixture; - expect(yield* resolveRelayCliPath({ CLAUDE_CONFIG_DIR: f.home })).toBe(f.cli); - }).pipe(Effect.scoped), - ); - - it.effect("keeps explicit overrides authoritative, even when invalid or empty", () => - Effect.gen(function* () { - const f = yield* fixture; - const override = f.path.join(f.home, "custom-relay.mjs"); - for (const [value, expected] of [ - [override, override], - ["relative/relay.mjs", undefined], - ["", undefined], - ] as const) { - expect( - yield* resolveRelayCliPath({ CLAUDE_CONFIG_DIR: f.home, PYLON_RELAY_CLI: value }), - ).toBe(expected); - } - }).pipe(Effect.scoped), - ); - - it.effect("does not let unrelated plugin metadata disable Relay discovery", () => - Effect.gen(function* () { - const f = yield* fixture; - for (const unrelated of [null, { legacy: true }, [{ scope: "user" }]]) { - yield* f.writeRegistry({ - "relay-orchestrator@relay-local": [{ scope: "user", installPath: f.plugin }], - "unrelated@other": unrelated, - }); - expect(yield* resolveRelayCliPath({ CLAUDE_CONFIG_DIR: f.home })).toBe(f.cli); - } - }).pipe(Effect.scoped), - ); - - it.effect("does not guess an installation when another Relay entry is malformed", () => - Effect.gen(function* () { - const f = yield* fixture; - for (const malformed of [null, { legacy: true }, [{ scope: "user" }]]) { - yield* f.writeRegistry({ - "relay-orchestrator@relay-local": [{ scope: "user", installPath: f.plugin }], - "relay-orchestrator@other": malformed, - }); - expect(yield* resolveRelayCliPath({ CLAUDE_CONFIG_DIR: f.home })).toBeUndefined(); - } - }).pipe(Effect.scoped), - ); - - it.effect("ignores project-only, unrelated, relative, and ambiguous installations", () => - Effect.gen(function* () { - const f = yield* fixture; - for (const plugins of [ - { "relay-orchestrator@relay-local": [{ scope: "project", installPath: f.plugin }] }, - { "other-relay-orchestrator@relay-local": [{ scope: "user", installPath: f.plugin }] }, - { "relay-orchestrator@relay-local": [{ scope: "user", installPath: "relative" }] }, - { - "relay-orchestrator@relay-local": [{ scope: "user", installPath: f.plugin }], - "relay-orchestrator@other": [ - { scope: "user", installPath: f.path.join(f.home, "other") }, - ], - }, - ]) { - yield* f.writeRegistry(plugins); - expect(yield* resolveRelayCliPath({ CLAUDE_CONFIG_DIR: f.home })).toBeUndefined(); - } - }).pipe(Effect.scoped), - ); - - it.effect("tolerates missing or malformed registries and missing CLI files", () => - Effect.gen(function* () { - const f = yield* fixture; - yield* f.fs.remove(f.cli); - expect(yield* resolveRelayCliPath({ CLAUDE_CONFIG_DIR: f.home })).toBeUndefined(); - yield* f.fs.makeDirectory(f.cli); - expect(yield* resolveRelayCliPath({ CLAUDE_CONFIG_DIR: f.home })).toBeUndefined(); - for (const contents of ["not json", "{}", '{"plugins":null}']) { - yield* f.fs.writeFileString(f.registry, contents); - expect(yield* resolveRelayCliPath({ CLAUDE_CONFIG_DIR: f.home })).toBeUndefined(); - } - yield* f.fs.remove(f.registry); - expect(yield* resolveRelayCliPath({ CLAUDE_CONFIG_DIR: f.home })).toBeUndefined(); - }).pipe(Effect.scoped), - ); -}); diff --git a/apps/server/src/orchestration/RelayCli.ts b/apps/server/src/orchestration/RelayCli.ts deleted file mode 100644 index e5360c1f1e..0000000000 --- a/apps/server/src/orchestration/RelayCli.ts +++ /dev/null @@ -1,57 +0,0 @@ -import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; -import * as Path from "effect/Path"; -import * as Schema from "effect/Schema"; - -import { resolveClaudeHomePath } from "../provider/Drivers/ClaudeHome.ts"; -import { relayCliFromEnvironment } from "../provider/relayMcpConfig.ts"; - -const decodeInstalledPlugins = Schema.decodeUnknownOption( - Schema.fromJsonString( - Schema.Struct({ - plugins: Schema.Record(Schema.String, Schema.Unknown), - }), - ), -); -const decodeInstallations = Schema.decodeUnknownOption( - Schema.Array(Schema.Struct({ scope: Schema.String, installPath: Schema.String })), -); - -/** Observe an installed plugin without requiring GUI launches to inherit shell configuration. */ -export const resolveRelayCliPath = Effect.fn("resolveRelayCliPath")(function* ( - environment: NodeJS.ProcessEnv, -) { - // An explicit override remains authoritative, including an invalid one. Do - // not silently observe a different installation when configuration is wrong. - if (environment.PYLON_RELAY_CLI !== undefined) return relayCliFromEnvironment(environment); - - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const claudeHome = yield* resolveClaudeHomePath({ homePath: "" }, environment); - const contents = yield* fs - .readFileString(path.join(claudeHome, "plugins", "installed_plugins.json")) - .pipe(Effect.orElseSucceed(() => undefined)); - if (contents === undefined) return undefined; - const installed = decodeInstalledPlugins(contents); - if (installed._tag === "None") return undefined; - - const candidates = new Set(); - for (const [name, value] of Object.entries(installed.value.plugins)) { - if (!name.startsWith("relay-orchestrator@")) continue; - const entries = decodeInstallations(value); - // Unrelated registry entries do not govern Relay. A malformed Relay entry - // could hide another installation, though, so leave that selection explicit. - if (entries._tag === "None") return undefined; - for (const entry of entries.value) { - // The bridge is environment-wide. A project-scoped plugin must not pick - // the observer for other projects, nor should an old cached version. - if (entry.scope !== "user" || !path.isAbsolute(entry.installPath)) continue; - candidates.add(path.join(entry.installPath, "scripts", "relay.mjs")); - } - } - if (candidates.size !== 1) return undefined; - const [candidate] = candidates; - if (candidate === undefined) return undefined; - const stat = yield* fs.stat(candidate).pipe(Effect.orElseSucceed(() => undefined)); - return stat?.type === "File" ? candidate : undefined; -}); diff --git a/apps/server/src/orchestration/RelayWorkerBridge.test.ts b/apps/server/src/orchestration/RelayWorkerBridge.test.ts deleted file mode 100644 index f6797fb877..0000000000 --- a/apps/server/src/orchestration/RelayWorkerBridge.test.ts +++ /dev/null @@ -1,1822 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off -import { describe, expect, it } from "vite-plus/test"; -import { it as effectIt } from "@effect/vitest"; -import * as NodeServices from "@effect/platform-node/NodeServices"; -import * as NodeFS from "node:fs"; -import * as NodeOS from "node:os"; -import * as NodePath from "node:path"; -import { - EventId, - EnvironmentId, - ProviderDriverKind, - RuntimeItemId, - ThreadId, - TurnId, - type ProviderRuntimeEvent, -} from "@t3tools/contracts"; -import * as Effect from "effect/Effect"; -import * as SqlClient from "effect/unstable/sql/SqlClient"; -import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; -import { OrchestrationEngineService } from "./Services/OrchestrationEngine.ts"; -import { ServerEnvironment } from "../environment/ServerEnvironment.ts"; -import { - ThreadBackgroundLivenessService, - make as makeLiveness, -} from "./ThreadBackgroundLiveness.ts"; - -import { - makeWithCliPath, - parseObservation, - relayBindingFromToolEvent, -} from "./RelayWorkerBridge.ts"; -import { resolveRelayCliPath } from "./RelayCli.ts"; - -const jobId = "job-11111111-1111-4111-8111-111111111111"; -const panelInitialJobId = "job-44444444-4444-4444-8444-444444444444"; -const replacementJobId = "job-33333333-3333-4333-8333-333333333333"; -const panelId = "panel-22222222-2222-4222-8222-222222222222"; -const completedPanelId = "panel-55555555-5555-4555-8555-555555555555"; -const completedPanelJobId = "job-66666666-6666-4666-8666-666666666666"; -const encodeJson = (value: unknown) => JSON.stringify(value); -const decodeJson = (value: unknown): unknown => JSON.parse(String(value)); - -function toolEvent(input: { - readonly toolName: string; - readonly result: unknown; - readonly provider?: "claudeAgent" | "codex"; - readonly status?: "completed" | "failed"; - readonly server?: string; - readonly threadId?: string; - readonly turnId?: string; - readonly itemId?: string; -}): ProviderRuntimeEvent { - return { - type: "item.completed", - eventId: EventId.make("relay-tool-event"), - provider: ProviderDriverKind.make(input.provider ?? "claudeAgent"), - threadId: ThreadId.make(input.threadId ?? "thread-a"), - turnId: TurnId.make(input.turnId ?? "turn-a"), - itemId: RuntimeItemId.make(input.itemId ?? "tool-a"), - createdAt: "2026-09-22T00:00:00.000Z", - payload: { - itemType: "mcp_tool_call", - status: input.status ?? "completed", - data: - input.provider === "codex" - ? { - item: { - type: "mcpToolCall", - server: input.server ?? "relay", - tool: input.toolName, - result: input.result, - }, - } - : { toolName: input.toolName, result: input.result }, - }, - } as ProviderRuntimeEvent; -} - -describe("RelayWorkerBridge direct receipts", () => { - it("binds Claude plugin and Codex direct Relay tool results to the exact thread, turn and item", () => { - const claude = toolEvent({ - toolName: "mcp__plugin_relay-orchestrator_relay__relay_delegate", - result: { - type: "tool_result", - content: [ - { - type: "text", - text: JSON.stringify({ - content: [{ type: "text", text: "Worker dispatched" }], - structuredContent: { schemaVersion: 1, kind: "job", jobId, attempt: 1 }, - }), - }, - ], - }, - }); - expect(relayBindingFromToolEvent(claude)).toEqual([ - { - kind: "job", - id: jobId, - threadId: "thread-a", - turnId: "turn-a", - toolCallId: "tool-a", - }, - ]); - - // Claude SDK serializes the MCP tool result's content array as a single - // JSON string inside its tool_result block in live provider sessions. - const liveClaudeShape = toolEvent({ - toolName: "mcp__relay__relay_delegate", - result: { - type: "tool_result", - tool_use_id: "tool-a", - content: JSON.stringify({ - content: [{ type: "text", text: "Worker dispatched" }], - structuredContent: { schemaVersion: 1, kind: "job", jobId, attempt: 1 }, - }), - }, - }); - expect(relayBindingFromToolEvent(liveClaudeShape)).toEqual([ - { kind: "job", id: jobId, threadId: "thread-a", turnId: "turn-a", toolCallId: "tool-a" }, - ]); - - const largePanelResult = toolEvent({ - toolName: "mcp__relay__relay_panel", - result: { - type: "tool_result", - content: JSON.stringify({ - content: [ - { type: "text", text: "Private prompt: " + "x".repeat(20_000) }, - { type: "text", text: JSON.stringify({ schemaVersion: 1, kind: "panel", panelId }) }, - ], - }), - }, - }); - expect(relayBindingFromToolEvent(largePanelResult)).toEqual([ - { kind: "panel", id: panelId, threadId: "thread-a", turnId: "turn-a", toolCallId: "tool-a" }, - ]); - - const codex = toolEvent({ - provider: "codex", - toolName: "relay_panel", - result: { structuredContent: { schemaVersion: 1, kind: "panel", panelId } }, - }); - expect(relayBindingFromToolEvent(codex)).toEqual([ - { - kind: "panel", - id: panelId, - threadId: "thread-a", - turnId: "turn-a", - toolCallId: "tool-a", - }, - ]); - }); - - it("finds the short receipt after a 100k escaped prompt in Claude's nested MCP wrapper", () => { - const prompt = "\n\\".repeat(50_000); - const firstText = JSON.stringify({ prompt }); - const wrapper = JSON.stringify({ - content: [ - { type: "text", text: firstText }, - { type: "text", text: JSON.stringify({ schemaVersion: 1, kind: "panel", panelId }) }, - ], - }); - expect(prompt).toHaveLength(100_000); - expect(wrapper.length).toBeGreaterThan(256 * 1024); - expect(wrapper.length).toBeLessThan(1024 * 1024); - expect( - relayBindingFromToolEvent( - toolEvent({ - toolName: "mcp__relay__relay_panel", - result: { type: "tool_result", content: wrapper }, - }), - ), - ).toEqual([ - { kind: "panel", id: panelId, threadId: "thread-a", turnId: "turn-a", toolCallId: "tool-a" }, - ]); - }); - - it("rejects lookalike servers, failed calls, model text and malformed IDs", () => { - const structured = { structuredContent: { schemaVersion: 1, kind: "job", jobId, attempt: 1 } }; - expect( - relayBindingFromToolEvent( - toolEvent({ toolName: "mcp__other__relay_delegate", result: structured }), - ), - ).toEqual([]); - expect( - relayBindingFromToolEvent( - toolEvent({ - provider: "codex", - server: "other", - toolName: "relay_delegate", - result: structured, - }), - ), - ).toEqual([]); - expect( - relayBindingFromToolEvent( - toolEvent({ toolName: "mcp__relay__relay_delegate", status: "failed", result: structured }), - ), - ).toEqual([]); - expect( - relayBindingFromToolEvent( - toolEvent({ - toolName: "mcp__relay__relay_delegate", - result: { content: [{ type: "text", text: `job ${jobId}` }] }, - }), - ), - ).toEqual([]); - expect( - relayBindingFromToolEvent( - toolEvent({ - toolName: "mcp__relay__relay_delegate", - result: { structuredContent: { schemaVersion: 1, kind: "job", jobId: "../../other" } }, - }), - ), - ).toEqual([]); - }); - - it("binds the installed Claude plugin's unversioned job result", () => { - const result = { - type: "tool_result", - content: [ - { - type: "text", - text: JSON.stringify({ - id: jobId, - status: "queued", - providerType: "codex", - createdAt: "2026-09-24T00:00:00.000Z", - }), - }, - ], - }; - expect( - relayBindingFromToolEvent( - toolEvent({ - toolName: "mcp__plugin_relay-orchestrator_relay__relay_delegate", - result, - }), - ), - ).toEqual([ - { kind: "job", id: jobId, threadId: "thread-a", turnId: "turn-a", toolCallId: "tool-a" }, - ]); - expect( - relayBindingFromToolEvent( - toolEvent({ - toolName: "mcp__relay__relay_delegate", - result, - }), - ), - ).toEqual([ - { kind: "job", id: jobId, threadId: "thread-a", turnId: "turn-a", toolCallId: "tool-a" }, - ]); - expect( - relayBindingFromToolEvent( - toolEvent({ - toolName: "mcp__plugin_relay-orchestrator_relay__relay_panel", - result: { - type: "tool_result", - content: [ - { - type: "text", - text: JSON.stringify({ - id: panelId, - jobs: [jobId], - unstarted: [{ index: 1 }], - }), - }, - ], - }, - }), - ), - ).toEqual([ - { kind: "panel", id: panelId, threadId: "thread-a", turnId: "turn-a", toolCallId: "tool-a" }, - ]); - }); -}); - -describe("Relay observe v1", () => { - it("accepts bounded job and panel snapshots and rejects malformed members/statuses", () => { - const job = { - schemaVersion: 1, - kind: "job", - id: jobId, - attempt: 1, - sequence: 3, - status: "running", - pending: false, - }; - expect(parseObservation(job, jobId)).toEqual(job); - expect(parseObservation({ ...job, status: "made-up" }, jobId)).toBeUndefined(); - expect( - parseObservation({ ...job, id: "job-33333333-3333-4333-8333-333333333333" }, jobId), - ).toBeUndefined(); - const panel = { - schemaVersion: 1, - kind: "panel", - id: panelId, - complete: false, - members: [ - { index: 0, slotAttempt: 2, state: "started", jobId, job }, - { index: 1, slotAttempt: 1, state: "unstarted", jobId: null, job: null }, - ], - }; - expect(parseObservation(panel, panelId)).toEqual(panel); - expect(parseObservation({ ...panel, members: [null] }, panelId)).toBeUndefined(); - expect( - parseObservation( - { ...panel, members: [{ index: 0, slotAttempt: 0, state: "started", jobId }] }, - panelId, - ), - ).toBeUndefined(); - }); -}); - -function fakeRelayCli() { - const dir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "pylon-relay-bridge-")); - const scripts = NodePath.join(dir, "scripts"); - NodeFS.mkdirSync(scripts); - const path = NodePath.join(scripts, "relay.mjs"); - NodeFS.writeFileSync( - path, - ` -import fs from "node:fs"; -const dataPath = new URL("./state.json", import.meta.url); -const callsPath = new URL("./calls.json", import.meta.url); -const state = JSON.parse(fs.readFileSync(dataPath, "utf8")); -const calls = JSON.parse(fs.readFileSync(callsPath, "utf8")); -const args = process.argv.slice(2); -calls.push(args); -fs.writeFileSync(callsPath, JSON.stringify(calls)); -const id = args[0] === "observe" && args[1] === "--panel" ? args[2] : args[1]; -if (args[0] === "observe" && state[id]?.schemaVersion === 1) console.log(JSON.stringify(state[id])); -else if (args[0] === "observe" && state[id]) console.log(JSON.stringify({ usage: "relay init | status [id]", request: "JSON from --request file" })); -else if (args[0] === "status" && state[id]) console.log(JSON.stringify(state[id])); -else if (args[0] === "panel-result" && state[id]) console.log(JSON.stringify(state[id])); -else if (args[0] === "cancel" && state[id]) console.log(JSON.stringify({id, status: "cancelling"})); -else process.exit(1); -`, - ); - const statePath = NodePath.join(scripts, "state.json"); - const callsPath = NodePath.join(scripts, "calls.json"); - NodeFS.writeFileSync(statePath, "{}"); - NodeFS.writeFileSync(callsPath, "[]"); - NodeFS.mkdirSync(NodePath.join(dir, "plugins")); - NodeFS.writeFileSync( - NodePath.join(dir, "plugins", "installed_plugins.json"), - encodeJson({ - plugins: { "relay-orchestrator@relay-local": [{ scope: "user", installPath: dir }] }, - }), - ); - return { - path, - environment: { CLAUDE_CONFIG_DIR: dir }, - setState: (state: unknown) => NodeFS.writeFileSync(statePath, JSON.stringify(state)), - calls: (): ReadonlyArray> => - JSON.parse(NodeFS.readFileSync(callsPath, "utf8")), - cleanup: () => NodeFS.rmSync(dir, { recursive: true, force: true }), - }; -} - -const fakeEnvironment = ServerEnvironment.of({ - getEnvironmentId: Effect.succeed(EnvironmentId.make("test-environment")), - getDescriptor: Effect.die("unused"), -}); - -const persistence = effectIt.layer(SqlitePersistenceMemory); - -persistence("Relay persisted observer and controls", (it) => { - it.effect( - "discovers the plugin without an override, recovers owned workers, and emits stable lifecycle and cancel controls", - () => { - const cli = fakeRelayCli(); - const job = { - schemaVersion: 1, - kind: "job", - id: jobId, - attempt: 1, - sequence: 1, - status: "running", - providerType: "codex", - model: "gpt-6-sol", - effort: "high", - activity: { kind: "editing", label: "Editing files" }, - usage: { totalTokens: 18, inputTokens: 12, outputTokens: 6 }, - pending: false, - outcome: null, - }; - cli.setState({ [jobId]: job }); - return Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - const commands = new Set(); - const engine = { - dispatch: (command: { - readonly type: string; - readonly commandId: string; - readonly threadId: string; - readonly activity?: { - readonly id: string; - readonly turnId: string | null; - readonly kind: string; - readonly tone: string; - readonly summary: string; - readonly payload: unknown; - readonly createdAt: string; - }; - }) => - Effect.gen(function* () { - if (commands.has(command.commandId)) - return { sequence: commands.size, eventCount: 0 }; - if (command.type !== "thread.activity.append" || !command.activity) - throw new Error("unexpected command"); - commands.add(command.commandId); - const activity = command.activity; - yield* sql` - INSERT INTO projection_thread_activities ( - activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at - ) VALUES ( - ${activity.id}, ${command.threadId}, ${activity.turnId}, ${activity.tone}, - ${activity.kind}, ${activity.summary}, ${encodeJson(activity.payload)}, ${commands.size}, ${activity.createdAt} - ) ON CONFLICT(activity_id) DO UPDATE SET - payload_json = excluded.payload_json, sequence = excluded.sequence, - created_at = excluded.created_at - `; - return { sequence: commands.size, eventCount: 1 }; - }), - } as unknown as OrchestrationEngineService["Service"]; - const liveness = makeLiveness(); - const discoveredCli = yield* resolveRelayCliPath(cli.environment).pipe( - Effect.provide(NodeServices.layer), - ); - expect(discoveredCli).toBe(cli.path); - const bridge = yield* makeWithCliPath(discoveredCli).pipe( - Effect.provideService(OrchestrationEngineService, engine), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, liveness), - ); - const persistedReceipt = { - itemType: "mcp_tool_call", - toolCallId: "tool-a", - status: "completed", - data: { - toolName: "mcp__plugin_relay-orchestrator_relay__relay_delegate", - result: { - type: "tool_result", - tool_use_id: "tool-a", - content: [{ type: "text", text: encodeJson({ id: jobId, status: "queued" }) }], - }, - }, - }; - yield* sql` - INSERT INTO projection_thread_activities ( - activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at - ) VALUES ( - 'persisted-relay-tool', 'thread-a', 'turn-a', 'tool', 'tool.completed', - 'Relay delegate', ${encodeJson(persistedReceipt)}, NULL, '2026-09-22T00:00:00.000Z' - ) - `; - yield* bridge.reconcile; - const started = - yield* sql`SELECT payload_json AS payload FROM projection_thread_activities WHERE kind = 'task.started'`; - expect(started).toHaveLength(1); - expect(decodeJson(started[0]?.payload)).toMatchObject({ - taskId: `relay:${jobId}`, - agentKind: "agent", - source: "relay", - attempt: 1, - relaySequence: 1, - model: "gpt-6-sol", - effort: "high", - cancellable: true, - }); - const legacyJobId = "job-77777777-7777-4777-8777-777777777777"; - cli.setState({ - [jobId]: job, - [legacyJobId]: { - id: legacyJobId, - status: "running", - providerType: "codex", - model: "gpt-6-sol", - effort: "low", - createdAt: "2026-09-24T00:00:00.000Z", - updatedAt: "2026-09-24T00:00:01.000Z", - }, - }); - yield* bridge.recordToolResult( - toolEvent({ - toolName: "mcp__plugin_relay-orchestrator_relay__relay_delegate", - itemId: "legacy-tool", - turnId: "legacy-turn", - result: { - type: "tool_result", - content: [ - { - type: "text", - text: encodeJson({ - id: legacyJobId, - status: "queued", - providerType: "codex", - }), - }, - ], - }, - }), - ); - yield* bridge.reconcile; - const legacyRows = yield* sql` - SELECT turn_id AS turnId, payload_json AS payload - FROM projection_thread_activities - WHERE kind = 'task.started' - AND json_extract(payload_json, '$.taskId') = ${`relay:${legacyJobId}`} - `; - expect(legacyRows).toHaveLength(1); - expect(legacyRows[0]?.turnId).toBe("legacy-turn"); - expect(decodeJson(legacyRows[0]?.payload)).toMatchObject({ - taskId: `relay:${legacyJobId}`, - source: "relay", - model: "gpt-6-sol", - effort: "low", - toolUseId: "legacy-tool", - }); - const wrongThread = yield* Effect.result( - bridge.cancel(ThreadId.make("thread-b"), `relay:${jobId}`), - ); - expect(wrongThread._tag).toBe("Failure"); - expect(cli.calls().some((args) => args[0] === "cancel")).toBe(false); - - // Removing the integration while a worker is visible downgrades it - // to reversible idle and hides cancellation without claiming death. - const unavailable = yield* makeWithCliPath(undefined).pipe( - Effect.provideService(OrchestrationEngineService, engine), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, liveness), - ); - yield* unavailable.reconcile; - yield* unavailable.reconcile; - yield* unavailable.reconcile; - const idleRows = - yield* sql`SELECT payload_json AS payload FROM projection_thread_activities WHERE kind = 'task.progress' AND summary = 'Relay observer unavailable' AND json_extract(payload_json, '$.taskId') = ${`relay:${jobId}`}`; - expect(idleRows).toHaveLength(1); - expect(decodeJson(idleRows[0]?.payload)).toMatchObject({ - taskId: `relay:${jobId}`, - status: "idle", - cancellable: false, - outageEpoch: 1, - }); - - const reconnected = yield* makeWithCliPath(cli.path).pipe( - Effect.provideService(OrchestrationEngineService, engine), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, liveness), - ); - yield* reconnected.reconcile; - const recoveredProgress = yield* sql` - SELECT payload_json AS payload FROM projection_thread_activities - WHERE activity_id = ${`relay-progress:${jobId}`} - `; - expect(decodeJson(recoveredProgress[0]?.payload)).toMatchObject({ - status: "running", - relaySequence: 1, - }); - cli.setState({}); - yield* reconnected.reconcile; - yield* reconnected.reconcile; - yield* reconnected.reconcile; - const repeatedOutage = yield* sql` - SELECT payload_json AS payload FROM projection_thread_activities - WHERE kind = 'task.progress' AND summary = 'Relay observer unavailable' - AND json_extract(payload_json, '$.taskId') = ${`relay:${jobId}`} - ORDER BY sequence - `; - expect(repeatedOutage).toHaveLength(2); - expect(decodeJson(repeatedOutage[1]?.payload)).toMatchObject({ - status: "idle", - relaySequence: 1, - outageEpoch: 2, - }); - - cli.setState({ - [jobId]: { - ...job, - status: "completed", - sequence: 2, - outcome: { - kind: "completed", - label: "Completed", - summary: "Reviewed the change", - error: null, - }, - }, - }); - yield* reconnected.reconcile; - const completed = - yield* sql`SELECT payload_json AS payload FROM projection_thread_activities WHERE kind = 'task.completed' AND json_extract(payload_json, '$.taskId') = ${`relay:${jobId}`}`; - expect(completed).toHaveLength(1); - expect(decodeJson(completed[0]?.payload)).toMatchObject({ - taskId: `relay:${jobId}`, - status: "completed", - summary: "Reviewed the change", - typedUsage: { totalTokens: 18 }, - }); - - // A new bridge instance reconstructs the binding from persisted - // activity; a resumed worker keeps its row identity and advances attempt. - cli.setState({ [jobId]: { ...job, attempt: 2, sequence: 1, status: "running" } }); - const recovered = yield* makeWithCliPath(cli.path).pipe( - Effect.provideService(OrchestrationEngineService, engine), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, liveness), - ); - yield* recovered.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_resume", - threadId: "thread-b", - turnId: "turn-foreign", - itemId: "tool-foreign", - result: { structuredContent: { schemaVersion: 1, kind: "job", jobId, attempt: 2 } }, - }), - ); - yield* recovered.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_resume", - turnId: "turn-resume", - itemId: "tool-resume", - result: { structuredContent: { schemaVersion: 1, kind: "job", jobId, attempt: 2 } }, - }), - ); - // A completed prior attempt must not retire a resumed worker when - // observe is unavailable before the first new task.started receipt. - cli.setState({}); - // The attempt origin survives a server restart before the next poll. - const resumedBridge = yield* makeWithCliPath(cli.path).pipe( - Effect.provideService(OrchestrationEngineService, engine), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, liveness), - ); - yield* resumedBridge.reconcile; - yield* resumedBridge.reconcile; - yield* resumedBridge.reconcile; - const pendingResume = yield* sql` - SELECT turn_id AS turnId, payload_json AS payload FROM projection_thread_activities - WHERE kind = 'task.progress' AND activity_id LIKE 'relay-observer-unavailable:%' - AND json_extract(payload_json, '$.taskId') = ${`relay:${jobId}`} - ORDER BY sequence DESC LIMIT 1 - `; - expect(pendingResume).toHaveLength(1); - expect(pendingResume[0]?.turnId).toBe("turn-resume"); - expect(decodeJson(pendingResume[0]?.payload)).toMatchObject({ - taskId: `relay:${jobId}`, - attempt: 2, - relaySequence: 0, - status: "idle", - toolUseId: "tool-resume", - relayPriorUsage: { totalTokens: 18, inputTokens: 12, outputTokens: 6 }, - }); - cli.setState({ [jobId]: { ...job, attempt: 2, sequence: 1, status: "running" } }); - yield* resumedBridge.reconcile; - const resumed = - yield* sql`SELECT turn_id AS turnId, payload_json AS payload FROM projection_thread_activities WHERE kind = 'task.started' AND json_extract(payload_json, '$.taskId') = ${`relay:${jobId}`} ORDER BY created_at`; - expect(resumed).toHaveLength(2); - expect(decodeJson(resumed[1]?.payload)).toMatchObject({ - taskId: `relay:${jobId}`, - attempt: 2, - toolUseId: "tool-resume", - relayPriorUsage: { totalTokens: 18, inputTokens: 12, outputTokens: 6 }, - }); - expect(resumed[1]?.turnId).toBe("turn-resume"); - const activations = yield* sql` - SELECT thread_id AS threadId, payload_json AS payload FROM projection_thread_activities - WHERE kind = 'relay.activation' AND activity_id = ${`relay-activation:${jobId}:2`} - `; - expect(activations).toHaveLength(1); - expect(activations[0]?.threadId).toBe("thread-a"); - expect(decodeJson(activations[0]?.payload)).toMatchObject({ - id: jobId, - attempt: 2, - toolCallId: "tool-resume", - environmentId: "test-environment", - }); - const cancelled = yield* resumedBridge.cancel(ThreadId.make("thread-a"), `relay:${jobId}`); - expect(cancelled.disposition).toBe("cancel-requested"); - expect(cli.calls().some((args) => args[0] === "cancel" && args[1] === jobId)).toBe(true); - - const panelJob = { ...job, id: panelInitialJobId, attempt: 1, sequence: 1 }; - const replacementJob = { ...job, id: replacementJobId, attempt: 1, sequence: 1 }; - const panel = { - schemaVersion: 1, - kind: "panel", - id: panelId, - complete: false, - members: [ - { - index: 0, - slotAttempt: 1, - state: "started", - jobId: panelInitialJobId, - job: panelJob, - }, - { index: 1, slotAttempt: 1, state: "unstarted", jobId: null, job: null }, - ], - }; - cli.setState({ - [panelInitialJobId]: panelJob, - [replacementJobId]: replacementJob, - [panelId]: panel, - }); - yield* resumedBridge.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_panel", - turnId: "turn-panel", - itemId: "tool-panel", - result: { - structuredContent: { - schemaVersion: 1, - kind: "panel", - panelId, - jobIds: [panelInitialJobId], - }, - }, - }), - ); - // A continuation can arrive before the first panel observation. Its - // cumulative jobIds must not steal the initial member's origin. - yield* resumedBridge.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_panel_continue", - turnId: "turn-continue", - itemId: "tool-continue", - result: { - structuredContent: { - schemaVersion: 1, - kind: "panel", - panelId, - jobIds: [panelInitialJobId, replacementJobId], - }, - }, - }), - ); - yield* resumedBridge.reconcile; - const slotId = `relay-panel:${panelId}:member:0`; - const slotRows = - yield* sql`SELECT payload_json AS payload FROM projection_thread_activities WHERE kind = 'task.started' AND json_extract(payload_json, '$.taskId') = ${slotId}`; - expect(slotRows).toHaveLength(1); - expect(decodeJson(slotRows[0]?.payload)).toMatchObject({ - taskId: slotId, - attempt: 1_000_001, - toolUseId: "tool-panel", - }); - const unstarted = - yield* sql`SELECT COUNT(*) AS count FROM projection_thread_activities WHERE kind = 'task.started' AND json_extract(payload_json, '$.taskId') = ${`relay-panel:${panelId}:member:1`}`; - expect(unstarted[0]?.count).toBe(0); - - cli.setState({ - [panelInitialJobId]: { ...panelJob, status: "failed", sequence: 2 }, - [replacementJobId]: replacementJob, - [panelId]: { - ...panel, - members: [ - { ...panel.members[0], job: { ...panelJob, status: "failed", sequence: 2 } }, - panel.members[1], - ], - }, - }); - yield* resumedBridge.reconcile; - const completedMemberObserveCount = cli - .calls() - .filter((args) => args[0] === "observe" && args[1] === panelInitialJobId).length; - yield* resumedBridge.reconcile; - expect( - cli.calls().filter((args) => args[0] === "observe" && args[1] === panelInitialJobId), - ).toHaveLength(completedMemberObserveCount); - - cli.setState({ - [panelInitialJobId]: { ...panelJob, status: "failed", sequence: 3 }, - [replacementJobId]: replacementJob, - [panelId]: { - ...panel, - members: [ - { - index: 0, - slotAttempt: 2, - state: "started", - jobId: replacementJobId, - job: replacementJob, - }, - { index: 1, slotAttempt: 1, state: "unstarted", jobId: null, job: null }, - ], - }, - }); - yield* resumedBridge.reconcile; - const replaced = - yield* sql`SELECT turn_id AS turnId, payload_json AS payload FROM projection_thread_activities WHERE kind = 'task.started' AND json_extract(payload_json, '$.taskId') = ${slotId} ORDER BY created_at`; - expect(replaced).toHaveLength(2); - expect(decodeJson(replaced[1]?.payload)).toMatchObject({ - taskId: slotId, - attempt: 2_000_001, - toolUseId: "tool-continue", - }); - expect(replaced[1]?.turnId).toBe("turn-continue"); - const slotCancel = yield* resumedBridge.cancel(ThreadId.make("thread-a"), slotId); - expect(slotCancel.disposition).toBe("cancel-requested"); - expect( - cli.calls().some((args) => args[0] === "cancel" && args[1] === replacementJobId), - ).toBe(true); - - // An incomplete panel with no active members remains visible as idle, - // while the observer parks until a continuation or resume receipt. - const failedReplacement = { ...replacementJob, status: "failed", sequence: 2 }; - const idlePanel = { - ...panel, - members: [ - { - ...panel.members[0], - slotAttempt: 2, - jobId: replacementJobId, - job: failedReplacement, - }, - panel.members[1], - ], - }; - cli.setState({ [panelId]: idlePanel, [replacementJobId]: failedReplacement }); - yield* resumedBridge.reconcile; - const idlePanelObserveCount = cli - .calls() - .filter( - (args) => args[0] === "observe" && args[1] === "--panel" && args[2] === panelId, - ).length; - yield* resumedBridge.reconcile; - expect( - cli - .calls() - .filter( - (args) => args[0] === "observe" && args[1] === "--panel" && args[2] === panelId, - ), - ).toHaveLength(idlePanelObserveCount); - const idlePanelProgress = yield* sql` - SELECT payload_json AS payload FROM projection_thread_activities - WHERE kind = 'task.progress' AND json_extract(payload_json, '$.taskId') = ${`relay-panel:${panelId}`} - ORDER BY sequence DESC LIMIT 1 - `; - expect(decodeJson(idlePanelProgress[0]?.payload)).toMatchObject({ status: "idle" }); - - // Relay recalculates panel.complete from current member jobs. A - // resumed member keeps its slot and job ID, while job.attempt grows. - const completedPanelJob = { - ...job, - id: completedPanelJobId, - status: "completed", - sequence: 2, - }; - const completedPanel = { - schemaVersion: 1, - kind: "panel", - id: completedPanelId, - complete: true, - members: [ - { - index: 0, - slotAttempt: 1, - state: "started", - jobId: completedPanelJobId, - job: completedPanelJob, - }, - ], - }; - cli.setState({ - [completedPanelId]: completedPanel, - [completedPanelJobId]: completedPanelJob, - }); - yield* resumedBridge.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_panel", - turnId: "turn-completed-panel", - itemId: "tool-completed-panel", - result: { - structuredContent: { - schemaVersion: 1, - kind: "panel", - panelId: completedPanelId, - jobIds: [completedPanelJobId], - }, - }, - }), - ); - yield* resumedBridge.reconcile; - const completedPanelMemberId = `relay-panel:${completedPanelId}:member:0`; - const completedPanelRows = yield* sql` - SELECT payload_json AS payload FROM projection_thread_activities - WHERE kind = 'task.completed' AND json_extract(payload_json, '$.taskId') = ${completedPanelMemberId} - `; - expect(completedPanelRows).toHaveLength(1); - const resumedPanelJob = { - ...completedPanelJob, - attempt: 2, - sequence: 1, - status: "running", - }; - cli.setState({ - [completedPanelId]: { - ...completedPanel, - complete: false, - members: [{ ...completedPanel.members[0], job: resumedPanelJob }], - }, - [completedPanelJobId]: resumedPanelJob, - }); - yield* resumedBridge.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_resume", - turnId: "turn-panel-resume", - itemId: "tool-panel-resume", - result: { - structuredContent: { - schemaVersion: 1, - kind: "job", - jobId: completedPanelJobId, - attempt: 2, - }, - }, - }), - ); - yield* resumedBridge.reconcile; - const resumedPanelRows = yield* sql` - SELECT turn_id AS turnId, payload_json AS payload FROM projection_thread_activities - WHERE kind = 'task.started' AND json_extract(payload_json, '$.taskId') = ${completedPanelMemberId} - ORDER BY sequence - `; - expect(resumedPanelRows).toHaveLength(2); - expect(decodeJson(resumedPanelRows[1]?.payload)).toMatchObject({ - attempt: 1_000_002, - toolUseId: "tool-panel-resume", - cancellable: true, - }); - expect(resumedPanelRows[1]?.turnId).toBe("turn-panel-resume"); - const panelCoordinatorRows = yield* sql` - SELECT payload_json AS payload FROM projection_thread_activities - WHERE kind = 'task.started' AND json_extract(payload_json, '$.taskId') = ${`relay-panel:${completedPanelId}`} - ORDER BY sequence - `; - expect(panelCoordinatorRows).toHaveLength(2); - expect(decodeJson(panelCoordinatorRows[1]?.payload)).toMatchObject({ attempt: 2 }); - expect( - (yield* resumedBridge.cancel(ThreadId.make("thread-a"), completedPanelMemberId)) - .disposition, - ).toBe("cancel-requested"); - expect( - cli.calls().some((args) => args[0] === "cancel" && args[1] === completedPanelJobId), - ).toBe(true); - - // Crash after a later resume receipt but before observation: startup - // must reactivate the completed parent from its active child binding. - const completedSecondAttempt = { - ...resumedPanelJob, - status: "completed", - sequence: 2, - }; - cli.setState({ - [completedPanelId]: { - ...completedPanel, - members: [{ ...completedPanel.members[0], job: completedSecondAttempt }], - }, - [completedPanelJobId]: completedSecondAttempt, - }); - yield* resumedBridge.reconcile; - const thirdAttempt = { ...resumedPanelJob, attempt: 3 }; - cli.setState({ - [completedPanelId]: { - ...completedPanel, - complete: false, - members: [{ ...completedPanel.members[0], job: thirdAttempt }], - }, - [completedPanelJobId]: thirdAttempt, - }); - yield* resumedBridge.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_resume", - turnId: "turn-panel-resume-after-restart", - itemId: "tool-panel-resume-after-restart", - result: { - structuredContent: { - schemaVersion: 1, - kind: "job", - jobId: completedPanelJobId, - attempt: 3, - }, - }, - }), - ); - const restartedPanelBridge = yield* makeWithCliPath(cli.path).pipe( - Effect.provideService(OrchestrationEngineService, engine), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, liveness), - ); - // Three failed observations must park the reactivated coordinator as - // idle at its new attempt, not discard the resumed child binding. - cli.setState({}); - yield* restartedPanelBridge.reconcile; - yield* restartedPanelBridge.reconcile; - yield* restartedPanelBridge.reconcile; - const unavailablePanel = yield* sql` - SELECT payload_json AS payload FROM projection_thread_activities - WHERE kind = 'task.progress' AND summary = 'Relay observer unavailable' - AND json_extract(payload_json, '$.taskId') = ${`relay-panel:${completedPanelId}`} - ORDER BY sequence DESC LIMIT 1 - `; - expect(unavailablePanel).toHaveLength(1); - expect(decodeJson(unavailablePanel[0]?.payload)).toMatchObject({ - attempt: 3, - status: "idle", - toolUseId: "tool-panel-resume-after-restart", - }); - cli.setState({ - [completedPanelId]: { - ...completedPanel, - complete: false, - members: [{ ...completedPanel.members[0], job: thirdAttempt }], - }, - [completedPanelJobId]: thirdAttempt, - }); - yield* restartedPanelBridge.reconcile; - const thirdStart = yield* sql` - SELECT turn_id AS turnId, payload_json AS payload FROM projection_thread_activities - WHERE kind = 'task.started' AND json_extract(payload_json, '$.taskId') = ${completedPanelMemberId} - AND json_extract(payload_json, '$.attempt') = 1000003 - `; - expect(thirdStart).toHaveLength(1); - expect(thirdStart[0]?.turnId).toBe("turn-panel-resume-after-restart"); - expect(decodeJson(thirdStart[0]?.payload)).toMatchObject({ - taskId: completedPanelMemberId, - toolUseId: "tool-panel-resume-after-restart", - }); - expect( - (yield* restartedPanelBridge.cancel(ThreadId.make("thread-a"), completedPanelMemberId)) - .disposition, - ).toBe("cancel-requested"); - - // Startup pages historical bindings without observing settled jobs or - // retaining a terminal seen entry for each completed worker. - yield* sql` - WITH RECURSIVE jobs(n) AS ( - SELECT 1 UNION ALL SELECT n + 1 FROM jobs WHERE n < 300 - ) - INSERT INTO projection_thread_activities ( - activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at - ) - SELECT - 'relay-binding:' || printf('job-00000000-0000-4000-8000-%012d', n), - 'thread-a', 'turn-a', 'info', 'relay.binding', 'Relay worker bound', - json_object('kind', 'job', 'id', printf('job-00000000-0000-4000-8000-%012d', n), - 'toolCallId', 'tool-old', 'environmentId', 'test-environment'), - n, '2026-09-22T00:00:00.000Z' - FROM jobs - `; - yield* sql` - WITH RECURSIVE jobs(n) AS ( - SELECT 1 UNION ALL SELECT n + 1 FROM jobs WHERE n < 300 - ) - INSERT INTO projection_thread_activities ( - activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at - ) - SELECT - 'relay-complete:' || printf('job-00000000-0000-4000-8000-%012d', n), - 'thread-a', 'turn-a', 'info', 'task.completed', 'Relay worker completed', - json_object('taskId', 'relay:' || printf('job-00000000-0000-4000-8000-%012d', n), - 'source', 'relay', 'attempt', 1, 'relaySequence', 1, 'status', 'completed'), - n + 1000, '2026-09-22T00:00:01.000Z' - FROM jobs - `; - const bootAfterChurn = yield* makeWithCliPath(cli.path).pipe( - Effect.provideService(OrchestrationEngineService, engine), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, liveness), - ); - const callsBeforeBoot = cli.calls().length; - yield* bootAfterChurn.reconcile; - const bootCalls = cli.calls().slice(callsBeforeBoot); - const observed = (id: string) => - bootCalls.findIndex((args) => args[0] === "observe" && args[1] === id); - expect(observed(legacyJobId)).toBeGreaterThan(-1); - expect(observed(completedPanelJobId)).toBeGreaterThan(-1); - expect(observed(completedPanelJobId)).toBeLessThan(observed(legacyJobId)); - expect( - cli - .calls() - .filter((args) => args[0] === "observe" && args[1]?.startsWith("job-00000000-")), - ).toHaveLength(0); - - const missingId = "job-cccccccc-cccc-4ccc-8ccc-cccccccccccc"; - yield* sql` - INSERT INTO projection_thread_activities ( - activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at - ) VALUES ( - ${`relay-binding:${missingId}`}, 'thread-a', 'turn-missing', 'info', - 'relay.binding', 'Relay worker bound', - ${encodeJson({ kind: "job", id: missingId, toolCallId: "tool-missing", environmentId: "test-environment" })}, - 9999, '2026-09-24T00:00:00.000Z' - ) - `; - const missingBridge = yield* makeWithCliPath(cli.path).pipe( - Effect.provideService(OrchestrationEngineService, engine), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, liveness), - ); - yield* missingBridge.reconcile; - yield* missingBridge.reconcile; - yield* missingBridge.reconcile; - // A binding whose worker never reached the thread has nothing to - // downgrade, so its missing Relay files stay quiet rather than - // opening an idle row nobody can act on. - const missingRows = yield* sql` - SELECT payload_json AS payload FROM projection_thread_activities - WHERE kind = 'task.progress' AND json_extract(payload_json, '$.taskId') = ${`relay:${missingId}`} - `; - expect(missingRows).toEqual([]); - const missingPolls = cli - .calls() - .filter((args) => args[0] === "observe" && args[1] === missingId).length; - yield* missingBridge.reconcile; - expect( - cli.calls().filter((args) => args[0] === "observe" && args[1] === missingId), - ).toHaveLength(missingPolls); - - const legacyPanelId = "panel-88888888-8888-4888-8888-888888888888"; - const priorId = "job-99999999-9999-4999-8999-999999999999"; - const currentId = "job-aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; - const otherId = "job-bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; - const legacyDate = "2026-09-24T00:00:01.000Z"; - const legacyJobs = [ - { - id: currentId, - status: "running", - panelId: legacyPanelId, - panelMemberIndex: 0, - providerType: "codex", - createdAt: legacyDate, - updatedAt: legacyDate, - }, - { - id: otherId, - status: "running", - panelId: legacyPanelId, - panelMemberIndex: 1, - providerType: "claude-alt", - createdAt: legacyDate, - updatedAt: legacyDate, - }, - ]; - cli.setState({ - [legacyPanelId]: { - id: legacyPanelId, - request: { members: [{}, {}] }, - jobs: legacyJobs, - previousJobs: [priorId], - unstarted: [], - complete: false, - createdAt: legacyDate, - }, - [priorId]: { - id: priorId, - status: "failed", - panelId: legacyPanelId, - panelMemberIndex: 0, - createdAt: legacyDate, - updatedAt: legacyDate, - }, - [currentId]: legacyJobs[0], - [otherId]: legacyJobs[1], - }); - yield* bootAfterChurn.recordToolResult( - toolEvent({ - toolName: "mcp__plugin_relay-orchestrator_relay__relay_panel", - itemId: "legacy-panel-tool", - turnId: "legacy-panel-turn", - result: { - type: "tool_result", - content: [ - { - type: "text", - text: encodeJson({ - id: legacyPanelId, - jobs: [currentId, otherId], - unstarted: [], - }), - }, - ], - }, - }), - ); - yield* bootAfterChurn.reconcile; - const legacyMemberId = `relay-panel:${legacyPanelId}:member:0`; - const legacyMember = yield* sql` - SELECT turn_id AS turnId, payload_json AS payload - FROM projection_thread_activities WHERE kind = 'task.started' - AND json_extract(payload_json, '$.taskId') = ${legacyMemberId} - `; - expect(legacyMember).toHaveLength(1); - expect(legacyMember[0]?.turnId).toBe("legacy-panel-turn"); - expect(decodeJson(legacyMember[0]?.payload)).toMatchObject({ - taskId: legacyMemberId, - attempt: 2_000_001, - parentAgentId: `relay-panel:${legacyPanelId}`, - }); - expect( - (yield* bootAfterChurn.cancel(ThreadId.make("thread-a"), legacyMemberId)).disposition, - ).toBe("cancel-requested"); - expect(cli.calls().some((args) => args[0] === "cancel" && args[1] === currentId)).toBe( - true, - ); - }).pipe(Effect.ensuring(Effect.sync(() => cli.cleanup()))); - }, - ); -}); - -function recordingEngine(sql: SqlClient.SqlClient) { - const commands = new Set(); - return { - dispatch: (command: { - readonly type: string; - readonly commandId: string; - readonly threadId: string; - readonly activity?: { - readonly id: string; - readonly turnId: string | null; - readonly kind: string; - readonly tone: string; - readonly summary: string; - readonly payload: unknown; - readonly createdAt: string; - }; - }) => - Effect.gen(function* () { - if (commands.has(command.commandId)) return { sequence: commands.size, eventCount: 0 }; - if (command.type !== "thread.activity.append" || !command.activity) - throw new Error("unexpected command"); - commands.add(command.commandId); - const activity = command.activity; - yield* sql` - INSERT INTO projection_thread_activities ( - activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at - ) VALUES ( - ${activity.id}, ${command.threadId}, ${activity.turnId}, ${activity.tone}, - ${activity.kind}, ${activity.summary}, ${encodeJson(activity.payload)}, ${commands.size}, ${activity.createdAt} - ) ON CONFLICT(activity_id) DO UPDATE SET - payload_json = excluded.payload_json, sequence = excluded.sequence, - created_at = excluded.created_at - `; - return { sequence: commands.size, eventCount: 1 }; - }), - } as unknown as OrchestrationEngineService["Service"]; -} - -persistence("Relay progress-only outage", (it) => { - it.effect("keeps an outage for a progress-only worker after restart", () => { - const cli = fakeRelayCli(); - cli.setState({}); - return Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - const jobId = "job-aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa4"; - const taskId = `relay:${jobId}`; - yield* sql` - INSERT INTO projection_thread_activities ( - activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at - ) VALUES ( - ${`relay-binding:${jobId}`}, 'thread-a', 'turn-progress', 'info', - 'relay.binding', 'Relay worker bound', - ${encodeJson({ kind: "job", id: jobId, toolCallId: "tool-progress", environmentId: "test-environment" })}, - 1, '2026-09-24T00:00:00.000Z' - ), ( - ${`relay-progress:${jobId}`}, 'thread-a', 'turn-progress', 'info', - 'task.progress', 'Relay worker running', - ${encodeJson({ taskId, attempt: 1, relaySequence: 1, status: "running" })}, - 2, '2026-09-24T00:00:01.000Z' - ) - `; - const bridge = yield* makeWithCliPath(cli.path).pipe( - Effect.provideService(OrchestrationEngineService, recordingEngine(sql)), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, makeLiveness()), - ); - yield* bridge.reconcile; - yield* bridge.reconcile; - yield* bridge.reconcile; - const outage = yield* sql` - SELECT payload_json AS payload FROM projection_thread_activities - WHERE kind = 'task.progress' AND summary = 'Relay observer unavailable' - AND json_extract(payload_json, '$.taskId') = ${taskId} - `; - expect(outage).toHaveLength(1); - expect(decodeJson(outage[0]?.payload)).toMatchObject({ status: "idle" }); - }).pipe(Effect.ensuring(Effect.sync(() => cli.cleanup()))); - }); -}); - -persistence("Relay startup receipt adoption", (it) => { - it.effect( - "does not replay retained terminal history, but recovers active work and later resumes", - () => { - const cli = fakeRelayCli(); - return Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - const engine = recordingEngine(sql); - const boot = () => - makeWithCliPath(cli.path).pipe( - Effect.provideService(OrchestrationEngineService, engine), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, makeLiveness()), - ); - const statuses = [ - "completed", - "failed", - "cancelled", - "interrupted", - "timed_out", - "output_limit", - "blocked_permissions", - "needs_review", - ]; - const jobs = statuses.map((status, index) => ({ - schemaVersion: 1 as const, - kind: "job" as const, - id: `job-aaaaaaaa-aaaa-4aaa-8aaa-${String(index).padStart(12, "0")}`, - attempt: 1, - sequence: 1, - status, - })); - const activeJob = { ...jobs[0]!, id: jobId, status: "running" }; - const pendingJob = { ...jobs[0]!, id: replacementJobId, pending: true }; - const state = Object.fromEntries( - [...jobs, activeJob, pendingJob].map((job, index) => [ - job.id, - index < jobs.length && index % 2 === 1 - ? { - id: job.id, - status: job.status, - createdAt: "2026-09-20T00:00:00.000Z", - updatedAt: "2026-09-20T00:01:00.000Z", - } - : job, - ]), - ); - cli.setState(state); - // Mix old bindings from the previously disabled observer with direct - // receipts lost in the crash window before binding registration. - const old = yield* boot(); - for (const [index, job] of [...jobs, activeJob, pendingJob].entries()) { - const event = toolEvent({ - toolName: "mcp__relay__relay_delegate", - itemId: `dispatch-${index}`, - result: { structuredContent: { schemaVersion: 1, kind: "job", jobId: job.id } }, - }); - if (index % 2 === 0) yield* old.recordToolResult(event); - else - yield* sql` - INSERT INTO projection_thread_activities ( - activity_id, thread_id, turn_id, tone, kind, summary, payload_json, created_at - ) VALUES ( - ${`receipt-${index}`}, 'thread-a', 'turn-a', 'tool', 'tool.completed', 'Relay delegate', - ${encodeJson({ ...event.payload, toolCallId: String(event.itemId) })}, - '2026-09-20T00:00:00.000Z' - ) - `; - } - const bridge = yield* boot(); - yield* bridge.reconcile; - const taskIds = yield* sql` - SELECT DISTINCT json_extract(payload_json, '$.taskId') AS taskId - FROM projection_thread_activities WHERE kind LIKE 'task.%' ORDER BY taskId - `; - expect(taskIds).toEqual( - [jobId, replacementJobId].sort().map((id) => ({ taskId: `relay:${id}` })), - ); - const retired = yield* sql` - SELECT COUNT(*) AS n FROM projection_thread_activities - WHERE kind = 'relay.binding' AND json_extract(payload_json, '$.retiredAfterRowId') IS NOT NULL - `; - expect(retired[0]?.n).toBe(jobs.length); - - // The workers shown as active before shutdown must still settle if - // they finish while Pylon is offline. - cli.setState({ - ...state, - [jobId]: { ...activeJob, sequence: 2, status: "completed" }, - [replacementJobId]: { ...pendingJob, pending: false, sequence: 2 }, - }); - const beforeRestart = cli.calls().length; - const restarted = yield* boot(); - yield* restarted.reconcile; - expect( - cli - .calls() - .slice(beforeRestart) - .some((args) => jobs.some((job) => args.includes(job.id))), - ).toBe(false); - const completed = - yield* sql`SELECT activity_id FROM projection_thread_activities WHERE kind = 'task.completed'`; - expect(completed).toHaveLength(2); - const beforeSecondRestart = cli.calls().length; - yield* (yield* boot()).reconcile; - expect(cli.calls()).toHaveLength(beforeSecondRestart); - - // A live dispatch that finishes before the first poll still belongs - // in the UI, and a suppressed historical worker can be resumed. - yield* restarted.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_delegate", - result: { - structuredContent: { schemaVersion: 1, kind: "job", jobId: panelInitialJobId }, - }, - }), - ); - cli.setState({ - ...state, - [panelInitialJobId]: { ...activeJob, id: panelInitialJobId, status: "completed" }, - }); - yield* restarted.reconcile; - yield* restarted.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_resume", - itemId: "resume-old", - turnId: "turn-new", - result: { - structuredContent: { schemaVersion: 1, kind: "job", jobId: jobs[0]!.id, attempt: 2 }, - }, - }), - ); - const resuming = yield* boot(); - yield* resuming.reconcile; - const premature = - yield* sql`SELECT activity_id FROM projection_thread_activities WHERE activity_id = ${`relay-start:${jobs[0]!.id}:2`}`; - expect(premature).toHaveLength(0); - cli.setState({ ...state, [jobs[0]!.id]: { ...jobs[0]!, attempt: 2, status: "running" } }); - yield* resuming.reconcile; - const resumed = - yield* sql`SELECT turn_id AS turnId FROM projection_thread_activities WHERE activity_id = ${`relay-start:${jobs[0]!.id}:2`}`; - expect(resumed).toEqual([{ turnId: "turn-new" }]); - const fast = - yield* sql`SELECT activity_id FROM projection_thread_activities WHERE activity_id = ${`relay-complete:${panelInitialJobId}:1`}`; - expect(fast).toHaveLength(1); - }).pipe(Effect.ensuring(Effect.sync(() => cli.cleanup()))); - }, - ); -}); - -persistence("Relay startup panel adoption", (it) => { - it.effect("skips settled panels and old members while recovering a running panel", () => { - const cli = fakeRelayCli(); - return Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - const engine = recordingEngine(sql); - const boot = () => - makeWithCliPath(cli.path).pipe( - Effect.provideService(OrchestrationEngineService, engine), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, makeLiveness()), - ); - const job = { - schemaVersion: 1, - kind: "job", - id: jobId, - attempt: 1, - sequence: 1, - status: "completed", - }; - const running = { ...job, id: replacementJobId, status: "running" }; - const panel = { - schemaVersion: 1, - kind: "panel", - id: panelId, - complete: true, - members: [{ index: 0, slotAttempt: 1, state: "started", jobId, job }], - }; - const partialPanelId = "panel-88888888-8888-4888-8888-888888888888"; - const partial = { - ...panel, - id: partialPanelId, - complete: false, - members: [ - ...panel.members, - { index: 1, slotAttempt: 1, state: "unstarted", jobId: null, job: null }, - ], - }; - const active = { - ...panel, - id: completedPanelId, - complete: false, - members: [ - ...panel.members, - { index: 1, slotAttempt: 1, state: "started", jobId: replacementJobId, job: running }, - ], - }; - cli.setState({ - [panelId]: panel, - [partialPanelId]: partial, - [completedPanelId]: active, - [jobId]: job, - [replacementJobId]: running, - }); - const old = yield* boot(); - for (const id of [panelId, partialPanelId, completedPanelId]) - yield* old.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_panel", - itemId: id, - result: { structuredContent: { schemaVersion: 1, kind: "panel", panelId: id } }, - }), - ); - yield* (yield* boot()).reconcile; - const tasks = - yield* sql`SELECT DISTINCT json_extract(payload_json, '$.taskId') AS taskId FROM projection_thread_activities WHERE kind LIKE 'task.%' ORDER BY taskId`; - expect(tasks).toEqual([ - { taskId: `relay-panel:${completedPanelId}` }, - { taskId: `relay-panel:${completedPanelId}:member:1` }, - ]); - const beforeRestart = cli.calls().length; - yield* (yield* boot()).reconcile; - expect( - cli - .calls() - .slice(beforeRestart) - .some( - (args) => - args.includes(panelId) || args.includes(partialPanelId) || args.includes(jobId), - ), - ).toBe(false); - // A continuation receipt can be durable before the panel snapshot - // catches up. Do not retire it again on the stale, settled snapshot. - yield* old.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_panel_continue", - itemId: "continue-partial", - result: { - structuredContent: { - schemaVersion: 1, - kind: "panel", - panelId: partialPanelId, - jobIds: [panelInitialJobId], - }, - }, - }), - ); - const continuing = yield* boot(); - yield* continuing.reconcile; - const premature = - yield* sql`SELECT activity_id FROM projection_thread_activities WHERE kind = 'task.started' AND json_extract(payload_json, '$.taskId') = ${`relay-panel:${partialPanelId}`}`; - expect(premature).toHaveLength(0); - const continuedJob = { ...running, id: panelInitialJobId }; - cli.setState({ - [partialPanelId]: { - ...partial, - members: [ - { - index: 0, - slotAttempt: 2, - state: "started", - jobId: panelInitialJobId, - job: continuedJob, - }, - ], - }, - [panelInitialJobId]: continuedJob, - [completedPanelId]: active, - [replacementJobId]: running, - }); - yield* continuing.reconcile; - const continued = - yield* sql`SELECT activity_id FROM projection_thread_activities WHERE kind = 'task.started' AND json_extract(payload_json, '$.taskId') = ${`relay-panel:${partialPanelId}:member:0`}`; - expect(continued).toHaveLength(1); - }).pipe(Effect.ensuring(Effect.sync(() => cli.cleanup()))); - }); -}); - -persistence("Relay missing historical jobs", (it) => { - it.effect("stays silent about adopted historical jobs it never projected", () => { - const cli = fakeRelayCli(); - // The startup scan adopts every persisted Relay receipt, including turns - // whose workers finished and were swept long before this boot. - cli.setState({}); - return Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - const historical = [ - { job: "job-aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1", turn: "turn-old-1", tool: "tool-old-1" }, - { job: "job-aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa2", turn: "turn-old-2", tool: "tool-old-2" }, - ]; - for (const entry of historical) { - yield* sql` - INSERT INTO projection_thread_activities ( - activity_id, thread_id, turn_id, tone, kind, summary, payload_json, sequence, created_at - ) VALUES ( - ${`persisted-${entry.tool}`}, 'thread-a', ${entry.turn}, 'tool', 'tool.completed', - 'Relay delegate', - ${encodeJson({ - itemType: "mcp_tool_call", - toolCallId: entry.tool, - status: "completed", - data: { - toolName: "mcp__plugin_relay-orchestrator_relay__relay_delegate", - result: { - type: "tool_result", - tool_use_id: entry.tool, - content: [ - { type: "text", text: encodeJson({ id: entry.job, status: "queued" }) }, - ], - }, - }, - })}, - NULL, '2026-09-20T00:00:00.000Z' - ) - `; - } - const bridge = yield* makeWithCliPath(cli.path).pipe( - Effect.provideService(OrchestrationEngineService, recordingEngine(sql)), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, makeLiveness()), - ); - yield* bridge.reconcile; - yield* bridge.reconcile; - yield* bridge.reconcile; - yield* bridge.reconcile; - - // The scan still adopts them so a worker that outlived a crash can be - // reclaimed, but a job it can never observe stays out of the thread. - const adopted = yield* sql` - SELECT activity_id AS activityId FROM projection_thread_activities - WHERE kind = 'relay.binding' ORDER BY activity_id - `; - expect(adopted).toEqual( - historical.map((entry) => ({ activityId: `relay-binding:${entry.job}` })), - ); - const noise = yield* sql` - SELECT activity_id AS activityId FROM projection_thread_activities - WHERE kind LIKE 'task.%' - `; - expect(noise).toEqual([]); - // Relay answered that these jobs are gone and nothing about them ever - // reached the thread, so they are retired instead of holding a CLI spawn - // for the life of the process. - const settledPolls = cli.calls().filter((args) => args[0] === "observe").length; - expect(settledPolls).toBeGreaterThan(0); - yield* bridge.reconcile; - yield* bridge.reconcile; - expect(cli.calls().filter((args) => args[0] === "observe")).toHaveLength(settledPolls); - - // Retirement is persisted on the binding itself, so a restart does not - // re-probe every job the environment has ever run. - const restarted = yield* makeWithCliPath(cli.path).pipe( - Effect.provideService(OrchestrationEngineService, recordingEngine(sql)), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, makeLiveness()), - ); - const beforeRestart = cli.calls().filter((args) => args[0] === "observe").length; - yield* restarted.reconcile; - yield* restarted.reconcile; - expect( - cli - .calls() - .slice(beforeRestart) - .filter((args) => args[0] === "observe" && historical.some((e) => e.job === args[1])), - ).toEqual([]); - // No extra rows were written to carry the retirement. - const bindingRows = yield* sql` - SELECT COUNT(*) AS n FROM projection_thread_activities WHERE kind = 'relay.binding' - `; - expect(Number((bindingRows[0] as { n: number }).n)).toBe(historical.length); - - // A worker this environment did project still reports its outage. - const liveJob = "job-aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa3"; - cli.setState({ - [liveJob]: { - schemaVersion: 1, - kind: "job", - id: liveJob, - attempt: 1, - sequence: 1, - status: "running", - providerType: "codex", - model: "gpt-6-sol", - effort: "high", - activity: { kind: "editing", label: "Editing files" }, - usage: { totalTokens: 4, inputTokens: 3, outputTokens: 1 }, - pending: false, - outcome: null, - }, - }); - yield* bridge.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_delegate", - turnId: "turn-live", - itemId: "tool-live", - result: { structuredContent: { schemaVersion: 1, kind: "job", jobId: liveJob } }, - }), - ); - yield* bridge.reconcile; - const started = yield* sql` - SELECT payload_json AS payload FROM projection_thread_activities - WHERE kind = 'task.started' AND json_extract(payload_json, '$.taskId') = ${`relay:${liveJob}`} - `; - expect(started).toHaveLength(1); - - cli.setState({}); - yield* bridge.reconcile; - yield* bridge.reconcile; - yield* bridge.reconcile; - const outage = yield* sql` - SELECT json_extract(payload_json, '$.taskId') AS taskId - FROM projection_thread_activities - WHERE kind = 'task.progress' AND summary = 'Relay observer unavailable' - `; - expect(outage).toEqual([{ taskId: `relay:${liveJob}` }]); - - // Reopening the thread after a restart must not republish the same - // history the previous boot already declined to announce. - const rebooted = yield* makeWithCliPath(cli.path).pipe( - Effect.provideService(OrchestrationEngineService, recordingEngine(sql)), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, makeLiveness()), - ); - yield* rebooted.reconcile; - yield* rebooted.reconcile; - yield* rebooted.reconcile; - yield* rebooted.reconcile; - const afterReboot = yield* sql` - SELECT DISTINCT json_extract(payload_json, '$.taskId') AS taskId - FROM projection_thread_activities - WHERE kind = 'task.progress' AND summary = 'Relay observer unavailable' - `; - expect(afterReboot).toEqual([{ taskId: `relay:${liveJob}` }]); - }).pipe(Effect.ensuring(Effect.sync(() => cli.cleanup()))); - }); -}); - -persistence("Relay retirement reversal", (it) => { - it.effect("wakes a retired worker for a resume accepted after it was retired", () => { - const cli = fakeRelayCli(); - const revived = "job-bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1"; - cli.setState({}); - return Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - const engine = recordingEngine(sql); - const boot = () => - makeWithCliPath(cli.path).pipe( - Effect.provideService(OrchestrationEngineService, engine), - Effect.provideService(ServerEnvironment, fakeEnvironment), - Effect.provideService(ThreadBackgroundLivenessService, makeLiveness()), - ); - - // Dispatch a worker, then let Relay lose it before anything projects. - const first = yield* boot(); - yield* first.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_delegate", - turnId: "turn-1", - itemId: "tool-1", - result: { structuredContent: { schemaVersion: 1, kind: "job", jobId: revived } }, - }), - ); - yield* first.reconcile; - yield* first.reconcile; - yield* first.reconcile; - yield* first.reconcile; - const retired = yield* sql` - SELECT payload_json AS payload FROM projection_thread_activities - WHERE activity_id = ${`relay-binding:${revived}`} - `; - expect(decodeJson(retired[0]?.payload)).toMatchObject({ id: revived }); - expect( - (decodeJson(retired[0]?.payload) as { retiredAfterRowId?: number }).retiredAfterRowId, - ).toBeGreaterThan(0); - - // A restart leaves it asleep. - const asleep = yield* boot(); - const beforeRestart = cli.calls().length; - yield* asleep.reconcile; - yield* asleep.reconcile; - expect( - cli - .calls() - .slice(beforeRestart) - .filter((args) => args[0] === "observe" && args[1] === revived), - ).toEqual([]); - - // A resume recorded after the retirement outranks it. - cli.setState({ - [revived]: { - schemaVersion: 1, - kind: "job", - id: revived, - attempt: 2, - sequence: 1, - status: "running", - providerType: "codex", - model: "gpt-6-sol", - effort: "high", - pending: false, - outcome: null, - }, - }); - yield* asleep.recordToolResult( - toolEvent({ - toolName: "mcp__relay__relay_resume", - turnId: "turn-2", - itemId: "tool-2", - result: { - structuredContent: { schemaVersion: 1, kind: "job", jobId: revived, attempt: 2 }, - }, - }), - ); - const awake = yield* boot(); - const beforeWake = cli.calls().length; - yield* awake.reconcile; - expect( - cli - .calls() - .slice(beforeWake) - .some((args) => args[0] === "observe" && args[1] === revived), - ).toBe(true); - const started = yield* sql` - SELECT payload_json AS payload FROM projection_thread_activities - WHERE kind = 'task.started' AND json_extract(payload_json, '$.taskId') = ${`relay:${revived}`} - `; - expect(started).toHaveLength(1); - }).pipe(Effect.ensuring(Effect.sync(() => cli.cleanup()))); - }); -}); diff --git a/apps/server/src/orchestration/RelayWorkerBridge.ts b/apps/server/src/orchestration/RelayWorkerBridge.ts deleted file mode 100644 index dade145714..0000000000 --- a/apps/server/src/orchestration/RelayWorkerBridge.ts +++ /dev/null @@ -1,1858 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off -import * as NodeChildProcess from "node:child_process"; -import * as NodeCrypto from "node:crypto"; -import * as NodePath from "node:path"; -import * as NodeUtil from "node:util"; - -import { - CommandId, - EventId, - ProviderDriverKind, - ProviderCancelSessionAgentError, - type ProviderCancelSessionAgentResult, - type ProviderRuntimeEvent, - ThreadId, - RuntimeItemId, - TurnId, - type OrchestrationThreadActivity, -} from "@t3tools/contracts"; -import * as Context from "effect/Context"; -import * as Clock from "effect/Clock"; -import * as Effect from "effect/Effect"; -import * as DateTime from "effect/DateTime"; -import * as Layer from "effect/Layer"; -import * as Option from "effect/Option"; -import * as Schedule from "effect/Schedule"; -import * as Schema from "effect/Schema"; -import type * as Scope from "effect/Scope"; -import * as SqlClient from "effect/unstable/sql/SqlClient"; - -import { OrchestrationEngineService } from "./Services/OrchestrationEngine.ts"; -import { ThreadBackgroundLivenessService } from "./ThreadBackgroundLiveness.ts"; -import { forkParked } from "../serverActivation.ts"; -import { ServerEnvironment } from "../environment/ServerEnvironment.ts"; -import { resolveRelayCliPath } from "./RelayCli.ts"; - -const execFile = NodeUtil.promisify(NodeChildProcess.execFile); -const decodeJson = Schema.decodeOption(Schema.fromJsonString(Schema.Unknown)); -const JOB_ID = /^job-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; -const PANEL_ID = /^panel-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; -const MAX_OBSERVE_BYTES = 128 * 1024; -// Claude can JSON-serialize a Relay result that already contains JSON-escaped -// prompt text. Keep the wrapper bounded while allowing Relay's 100k prompt cap. -const MAX_MCP_RECEIPT_CODE_UNITS = 1024 * 1024; -const MAX_LABEL = 300; - -interface RelayBinding { - readonly kind: "job" | "panel"; - readonly id: string; - readonly threadId: ThreadId; - readonly turnId: string | null; - readonly toolCallId: string; - readonly environmentId: string; - readonly panelId?: string; - readonly agentIndex?: number; - readonly slotAttempt?: number; - /** - * Watermark written once Relay reported this job gone or already settled - * during recovery while nothing about it had reached the thread. Boot skips - * a retired binding unless an activation was recorded after this point, so - * adopted history stops costing a probe on every start. A row id rather than - * a timestamp: the rest of this file already orders recovery that way, and - * same-millisecond writes are common. - */ - readonly retiredAfterRowId?: number; -} - -interface RelayToolReceipt { - readonly binding: Omit; - readonly toolName: "relay_delegate" | "relay_panel" | "relay_resume" | "relay_panel_continue"; - readonly attempt?: number; - readonly jobIds?: ReadonlyArray; -} - -interface RelayJobObservation { - readonly schemaVersion: 1; - readonly kind: "job"; - readonly id: string; - readonly attempt: number; - readonly sequence: number; - readonly status: string; - readonly providerType?: string | null; - readonly model?: string | null; - readonly effort?: string | null; - readonly panelId?: string | null; - readonly panelMemberIndex?: number | null; - readonly createdAt?: string; - readonly updatedAt?: string; - readonly activity?: { readonly kind?: string; readonly label?: string } | null; - readonly usage?: { - readonly totalTokens?: number; - readonly inputTokens?: number; - readonly outputTokens?: number; - readonly cachedInputTokens?: number; - } | null; - readonly outcome?: { - readonly kind?: string; - readonly label?: string; - readonly summary?: string | null; - readonly error?: string | null; - } | null; - readonly pending?: boolean; -} - -interface RelayPanelObservation { - readonly schemaVersion: 1; - readonly kind: "panel"; - readonly id: string; - readonly createdAt?: string; - readonly complete: boolean; - readonly members: ReadonlyArray<{ - readonly index: number; - readonly slotAttempt: number; - readonly state: string; - readonly jobId: string | null; - readonly job?: RelayJobObservation | null; - }>; -} - -interface RelayUsageRollup { - readonly totalTokens: number; - readonly inputTokens?: number; - readonly outputTokens?: number; - readonly cachedInputTokens?: number; -} - -const record = (value: unknown): Record | undefined => - value !== null && typeof value === "object" && !Array.isArray(value) - ? (value as Record) - : undefined; -const boundedText = (value: unknown, max = MAX_LABEL): string | undefined => - typeof value === "string" && value.trim().length > 0 ? value.trim().slice(0, max) : undefined; -const nonNegativeInteger = (value: unknown): number | undefined => - typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; - -/** Only a direct successful result from the configured Relay MCP server can bind a job. */ -export function relayBindingFromToolEvent( - event: ProviderRuntimeEvent, -): ReadonlyArray> { - const receipt = relayReceiptFromToolEvent(event); - return receipt ? [receipt.binding] : []; -} - -function relayReceiptFromToolEvent(event: ProviderRuntimeEvent): RelayToolReceipt | undefined { - if ( - event.type !== "item.completed" || - event.payload.itemType !== "mcp_tool_call" || - event.payload.status !== "completed" || - event.itemId === undefined || - event.turnId === undefined - ) - return undefined; - const data = record(event.payload.data); - const item = record(data?.item); - const toolName = - item?.server === "relay" && typeof item.tool === "string" - ? item.tool - : typeof data?.toolName === "string" && - /^(?:mcp__relay__|mcp__plugin_relay-orchestrator_relay__)relay_/.test(data.toolName) - ? data.toolName.split("__").at(-1) - : undefined; - if ( - !toolName || - !["relay_delegate", "relay_panel", "relay_resume", "relay_panel_continue"].includes(toolName) - ) { - return undefined; - } - const result = item?.result ?? data?.result; - const structured = relayStructuredResult(result) ?? relayLegacyResult(result); - if (!structured || structured.schemaVersion !== 1) return undefined; - const base = { - threadId: event.threadId, - turnId: String(event.turnId), - toolCallId: String(event.itemId), - }; - if ( - (toolName === "relay_delegate" || toolName === "relay_resume") && - structured.kind === "job" && - typeof structured.jobId === "string" && - JOB_ID.test(structured.jobId) - ) { - const attempt = nonNegativeInteger(structured.attempt); - if (toolName === "relay_resume" && (attempt === undefined || attempt < 1)) return undefined; - return { - binding: { kind: "job", id: structured.jobId, ...base }, - toolName, - ...(attempt !== undefined ? { attempt } : {}), - }; - } - if ( - (toolName === "relay_panel" || toolName === "relay_panel_continue") && - structured.kind === "panel" && - typeof structured.panelId === "string" && - PANEL_ID.test(structured.panelId) - ) { - const jobIds = Array.isArray(structured.jobIds) - ? structured.jobIds.filter((id): id is string => typeof id === "string" && JOB_ID.test(id)) - : []; - return { - binding: { kind: "panel", id: structured.panelId, ...base }, - toolName, - jobIds: jobIds.slice(0, 8), - }; - } - return undefined; -} - -/** The installed Claude plugin still returns its job/panel record as text. */ -function relayLegacyResult(value: unknown): Record | undefined { - const object = record(value); - const content = object?.content; - if (!Array.isArray(content) || content.length > 4) return undefined; - const text = record(content[0])?.text; - if (typeof text !== "string" || text.length > MAX_MCP_RECEIPT_CODE_UNITS) return undefined; - try { - const parsed = record(JSON.parse(text)); - if (!parsed) return undefined; - if ( - typeof parsed.id === "string" && - JOB_ID.test(parsed.id) && - typeof parsed.status === "string" && - parsed.status !== "not_dispatched" - ) { - return { - schemaVersion: 1, - kind: "job", - jobId: parsed.id, - ...(nonNegativeInteger(parsed.attempt) !== undefined ? { attempt: parsed.attempt } : {}), - }; - } - if ( - typeof parsed.id === "string" && - PANEL_ID.test(parsed.id) && - Array.isArray(parsed.jobs) && - parsed.jobs.length <= 8 - ) { - return { - schemaVersion: 1, - kind: "panel", - panelId: parsed.id, - jobIds: parsed.jobs.filter((id): id is string => typeof id === "string" && JOB_ID.test(id)), - }; - } - } catch { - /* Invalid plugin text is not a dispatch receipt. */ - } - return undefined; -} - -function relayStructuredResult(value: unknown, depth = 0): Record | undefined { - if (depth > 4) return undefined; - const object = record(value); - if (!object) { - if (typeof value !== "string" || value.length > MAX_MCP_RECEIPT_CODE_UNITS) return undefined; - try { - return relayStructuredResult(JSON.parse(value), depth + 1); - } catch { - return undefined; - } - } - if (object.schemaVersion === 1 && (object.kind === "job" || object.kind === "panel")) - return object; - if (object.structuredContent !== undefined) - return relayStructuredResult(object.structuredContent, depth + 1); - if (object.result !== undefined) return relayStructuredResult(object.result, depth + 1); - if (typeof object.content === "string") { - return relayStructuredResult(object.content, depth + 1); - } - if (Array.isArray(object.content)) { - // Relay appends a short versioned receipt after its human-readable result. - // Inspect the tail first so large prompt/result text is never interpreted - // as an origin receipt or copied into a task projection. - for (const block of object.content.slice(-4).toReversed()) { - const nested = record(block); - const parsed = relayStructuredResult(nested?.text ?? nested?.content, depth + 1); - if (parsed) return parsed; - } - } - return undefined; -} - -export function parseObservation( - value: unknown, - expectedId: string, -): RelayJobObservation | RelayPanelObservation | undefined { - const object = record(value); - if (!object || object.schemaVersion !== 1 || object.id !== expectedId) return undefined; - if ( - object.kind === "job" && - JOB_ID.test(expectedId) && - nonNegativeInteger(object.attempt) !== undefined && - object.attempt !== 0 && - nonNegativeInteger(object.sequence) !== undefined && - typeof object.status === "string" && - [ - "starting", - "queued", - "running", - "cancelling", - "orphaned", - "completed", - "failed", - "cancelled", - "interrupted", - "timed_out", - "output_limit", - "blocked_permissions", - "needs_review", - ].includes(object.status) - ) { - return object as unknown as RelayJobObservation; - } - if ( - object.kind === "panel" && - PANEL_ID.test(expectedId) && - typeof object.complete === "boolean" && - Array.isArray(object.members) && - object.members.length <= 8 && - object.members.every((member: unknown) => { - const slot = record(member); - return ( - slot !== undefined && - nonNegativeInteger(slot.index) !== undefined && - nonNegativeInteger(slot.index)! <= 7 && - nonNegativeInteger(slot.slotAttempt) !== undefined && - nonNegativeInteger(slot.slotAttempt)! > 0 && - (slot.state === "started" || slot.state === "unstarted") && - (slot.jobId === null || (typeof slot.jobId === "string" && JOB_ID.test(slot.jobId))) - ); - }) - ) { - return object as unknown as RelayPanelObservation; - } - return undefined; -} - -function legacyObservation( - value: unknown, - expectedId: string, - priorJobSlots: ReadonlyMap = new Map(), -): RelayJobObservation | RelayPanelObservation | undefined { - const object = record(value); - if (!object || object.id !== expectedId) return undefined; - if (JOB_ID.test(expectedId)) { - const attempt = nonNegativeInteger(object.attempt) ?? 1; - const sequence = Date.parse(String(object.updatedAt ?? object.createdAt)); - if (attempt < 1 || !Number.isSafeInteger(sequence) || typeof object.status !== "string") - return undefined; - return parseObservation( - { - schemaVersion: 1, - kind: "job", - id: expectedId, - attempt, - sequence, - status: object.status, - providerType: object.providerType, - model: object.actualModel ?? object.model, - effort: object.effort, - panelId: object.panelId, - panelMemberIndex: object.panelMemberIndex, - createdAt: object.createdAt, - updatedAt: object.updatedAt, - }, - expectedId, - ); - } - if ( - !PANEL_ID.test(expectedId) || - !Array.isArray(object.jobs) || - !Array.isArray(object.unstarted) || - object.jobs.length > 8 - ) - return undefined; - const request = record(object.request); - const requested = request?.members; - if (!Array.isArray(requested) || requested.length < 2 || requested.length > 8) return undefined; - const jobs = object.jobs.map(record); - const members = requested.map((_, index) => { - const prior = Array.isArray(object.previousJobs) ? object.previousJobs : []; - const slotAttempt = 1 + prior.filter((id) => priorJobSlots.get(id) === index).length; - const job = jobs.find((candidate) => candidate?.panelMemberIndex === index); - const jobId = typeof job?.id === "string" && JOB_ID.test(job.id) ? job.id : null; - return { - index, - slotAttempt, - state: jobId ? "started" : "unstarted", - jobId, - ...(jobId ? { job: legacyObservation(job, jobId) } : {}), - }; - }); - return parseObservation( - { - schemaVersion: 1, - kind: "panel", - id: expectedId, - createdAt: object.createdAt, - complete: object.complete === true, - members, - }, - expectedId, - ); -} - -function legacyObserveUnavailable(value: unknown): boolean { - const object = record(value); - return typeof object?.usage === "string" && typeof object.request === "string"; -} - -function observationFingerprint(observation: RelayJobObservation | RelayPanelObservation): string { - return NodeCrypto.createHash("sha256") - .update(JSON.stringify(observation)) - .digest("hex") - .slice(0, 16); -} - -// A completed panel can reopen when a member resumes. Its coordinator needs -// an attempt that advances with either a resumed job or a replaced slot, so a -// previous terminal coordinator cannot mask the new run in the client fold. -function panelAttempt(observation: RelayPanelObservation): number | undefined { - const attempt = observation.members.reduce((sum, member) => { - const jobAttempt = nonNegativeInteger(record(member.job)?.attempt) ?? 1; - return sum + member.slotAttempt - 1 + jobAttempt - 1; - }, 1); - return Number.isSafeInteger(attempt) && attempt > 0 ? attempt : undefined; -} - -const relayStatus = ( - status: string, -): "pending" | "running" | "idle" | "completed" | "failed" | "cancelled" | "interrupted" => { - if (status === "queued" || status === "starting") return "pending"; - if (status === "running" || status === "cancelling") return "running"; - if (status === "completed") return "completed"; - if (status === "cancelled") return "cancelled"; - if (status === "interrupted") return "interrupted"; - if ( - ["failed", "blocked_permissions", "needs_review", "timed_out", "output_limit"].includes(status) - ) - return "failed"; - return "idle"; -}; - -function isSettledObservation(observation: RelayJobObservation | RelayPanelObservation): boolean { - if (observation.kind === "job") { - return ( - !observation.pending && - ["completed", "failed", "cancelled", "interrupted"].includes(relayStatus(observation.status)) - ); - } - return ( - observation.complete || - observation.members.every((member) => { - if (member.state === "unstarted") return true; - const job = member.jobId ? parseObservation(member.job, member.jobId) : undefined; - return job?.kind === "job" && isSettledObservation(job); - }) - ); -} - -function bindingPayload(binding: RelayBinding) { - return { - taskType: binding.kind === "panel" ? "local_workflow" : "subagent", - agentKind: "agent", - source: "relay", - cancellable: binding.kind === "job", - watchable: false, - messageable: false, - timelineBypass: true, - toolUseId: binding.toolCallId, - ...(binding.panelId - ? { parentAgentId: `relay-panel:${binding.panelId}`, workflowName: "Relay panel" } - : {}), - ...(binding.agentIndex !== undefined ? { agentIndex: binding.agentIndex } : {}), - }; -} - -function bindingTaskId(binding: RelayBinding): string { - if (binding.kind === "panel") return `relay-panel:${binding.id}`; - if (binding.panelId !== undefined && binding.agentIndex !== undefined) { - return `relay-panel:${binding.panelId}:member:${binding.agentIndex}`; - } - return `relay:${binding.id}`; -} - -export class RelayWorkerBridge extends Context.Service< - RelayWorkerBridge, - { - readonly start: Effect.Effect; - readonly reconcile: Effect.Effect; - readonly recordToolResult: (event: ProviderRuntimeEvent) => Effect.Effect; - readonly cancel: ( - threadId: ThreadId, - agentId: string, - ) => Effect.Effect; - } ->()("t3/orchestration/RelayWorkerBridge") {} - -export const makeWithCliPath = Effect.fn("RelayWorkerBridge.makeWithCliPath")(function* ( - cliPath: string | undefined, -) { - const enabled = cliPath !== undefined && NodePath.isAbsolute(cliPath); - const sql = yield* SqlClient.SqlClient; - const engine = yield* OrchestrationEngineService; - const environmentId = String(yield* (yield* ServerEnvironment).getEnvironmentId); - const liveness = yield* ThreadBackgroundLivenessService; - const active = new Map(); - // Recovery may discover years of dispatch receipts. Only work still alive - // or already shown in this thread should produce lifecycle activity. - const recovered = new Set(); - const seen = new Map(); - const currentPanelJobs = new Map>(); - const failedObservations = new Map(); - const missingObservationRetryAt = new Map(); - const unavailable = new Set(); - const outageEpochs = new Map(); - const pendingPanelDispatchPolls = new Map(); - const priorUsageCache = new Map< - string, - { attempt: number; usage: RelayUsageRollup | undefined } - >(); - const legacyPriorJobSlots = new Map>(); - - /** Drop every trace of a binding this process has stopped observing. */ - const retire = (id: string): void => { - active.delete(id); - recovered.delete(id); - legacyPriorJobSlots.delete(id); - seen.delete(id); - failedObservations.delete(id); - missingObservationRetryAt.delete(id); - unavailable.delete(id); - outageEpochs.delete(id); - pendingPanelDispatchPolls.delete(id); - priorUsageCache.delete(id); - }; - - const append = Effect.fn("RelayWorkerBridge.append")(function* ( - binding: RelayBinding, - kind: string, - payload: Record, - summary: string, - id: string, - fingerprint: string, - ) { - const createdAt = DateTime.formatIso(yield* DateTime.now); - const activity: OrchestrationThreadActivity = { - id: EventId.make(id), - createdAt, - turnId: binding.turnId as OrchestrationThreadActivity["turnId"], - tone: kind === "task.completed" && payload.status === "failed" ? "error" : "info", - kind, - summary, - payload, - }; - yield* engine.dispatch({ - type: "thread.activity.append", - commandId: CommandId.make(`relay:v2:${id}:${fingerprint}`), - threadId: binding.threadId, - activity, - createdAt, - }); - if (kind.startsWith("task.")) { - liveness.recordTaskLiveness({ - threadId: binding.threadId, - taskId: bindingTaskId(binding), - source: "relay", - taskType: binding.kind === "panel" ? "local_workflow" : "subagent", - status: typeof payload.status === "string" ? payload.status : undefined, - kind: - kind === "task.started" - ? "started" - : kind === "task.completed" - ? "completed" - : "progress", - }); - } - }); - - const decodeBindingRow = (raw: unknown): RelayBinding | undefined => { - const row = record(raw); - if (!row || typeof row.payload !== "string" || typeof row.threadId !== "string") - return undefined; - const decoded = decodeJson(row.payload); - const payload = Option.isSome(decoded) ? record(decoded.value) : undefined; - if ( - !payload || - !( - (payload.kind === "job" && typeof payload.id === "string" && JOB_ID.test(payload.id)) || - (payload.kind === "panel" && typeof payload.id === "string" && PANEL_ID.test(payload.id)) - ) || - typeof payload.toolCallId !== "string" || - typeof payload.environmentId !== "string" - ) - return undefined; - return { - kind: payload.kind as "job" | "panel", - id: payload.id as string, - threadId: ThreadId.make(row.threadId), - turnId: typeof row.turnId === "string" ? row.turnId : null, - toolCallId: payload.toolCallId, - environmentId: payload.environmentId, - ...(typeof payload.panelId === "string" ? { panelId: payload.panelId } : {}), - ...(nonNegativeInteger(payload.agentIndex) !== undefined - ? { agentIndex: payload.agentIndex as number } - : {}), - ...(nonNegativeInteger(payload.slotAttempt) !== undefined - ? { slotAttempt: payload.slotAttempt as number } - : {}), - ...(nonNegativeInteger(payload.retiredAfterRowId) !== undefined - ? { retiredAfterRowId: payload.retiredAfterRowId as number } - : {}), - }; - }; - - const readBinding = Effect.fn("RelayWorkerBridge.readBinding")(function* (id: string) { - const rows = yield* sql` - SELECT thread_id AS threadId, turn_id AS turnId, payload_json AS payload - FROM projection_thread_activities - WHERE activity_id = ${`relay-binding:${id}`} AND kind = 'relay.binding' - LIMIT 1 - `; - return decodeBindingRow(rows[0]); - }); - - const readBindingPage = Effect.fn("RelayWorkerBridge.readBindingPage")(function* ( - beforeRowId: number, - ) { - const rows = yield* sql` - SELECT rowid AS rowId, thread_id AS threadId, turn_id AS turnId, payload_json AS payload - FROM projection_thread_activities - WHERE rowid < ${beforeRowId} AND kind = 'relay.binding' - AND json_extract(payload_json, '$.environmentId') = ${environmentId} - ORDER BY rowid DESC LIMIT 256 - `; - return rows.map((raw) => ({ - rowId: nonNegativeInteger(record(raw)?.rowId) ?? 0, - binding: decodeBindingRow(raw), - })); - }); - - const readActivation = Effect.fn("RelayWorkerBridge.readActivation")(function* ( - binding: RelayBinding, - attempt: number, - ) { - const rows = yield* sql` - SELECT turn_id AS turnId, payload_json AS payload - FROM projection_thread_activities - WHERE activity_id = ${`relay-activation:${binding.id}:${attempt}`} - AND thread_id = ${binding.threadId} AND kind = 'relay.activation' - AND json_extract(payload_json, '$.environmentId') = ${environmentId} - LIMIT 1 - `; - const row = record(rows[0]); - const decoded = typeof row?.payload === "string" ? decodeJson(row.payload) : Option.none(); - const payload = Option.isSome(decoded) ? record(decoded.value) : undefined; - return payload?.id === binding.id && - payload.attempt === attempt && - typeof payload.toolCallId === "string" - ? { - ...binding, - turnId: typeof row?.turnId === "string" ? row.turnId : null, - toolCallId: payload.toolCallId, - } - : undefined; - }); - - const readPanelJobOrigin = Effect.fn("RelayWorkerBridge.readPanelJobOrigin")(function* ( - panel: RelayBinding, - jobId: string, - ) { - const rows = yield* sql` - SELECT turn_id AS turnId, payload_json AS payload - FROM projection_thread_activities - WHERE thread_id = ${panel.threadId} AND kind = 'relay.activation' - AND json_extract(payload_json, '$.environmentId') = ${environmentId} - AND json_extract(payload_json, '$.id') = ${panel.id} - AND EXISTS ( - SELECT 1 FROM json_each(json_extract(payload_json, '$.jobIds')) - WHERE value = ${jobId} - ) - ORDER BY rowid ASC LIMIT 1 - `; - const row = record(rows[0]); - const decoded = typeof row?.payload === "string" ? decodeJson(row.payload) : Option.none(); - const payload = Option.isSome(decoded) ? record(decoded.value) : undefined; - return typeof payload?.toolCallId === "string" - ? { - turnId: typeof row?.turnId === "string" ? row.turnId : null, - toolCallId: payload.toolCallId, - } - : undefined; - }); - - const hasUnobservedPanelDispatch = Effect.fn("RelayWorkerBridge.hasUnobservedPanelDispatch")( - function* (panel: RelayBinding, observedJobIds?: ReadonlySet) { - const rows = yield* sql` - SELECT payload_json AS payload FROM projection_thread_activities - WHERE thread_id = ${panel.threadId} AND kind = 'relay.activation' - AND json_extract(payload_json, '$.environmentId') = ${environmentId} - AND json_extract(payload_json, '$.id') = ${panel.id} - ORDER BY rowid DESC LIMIT 1 - `; - const raw = record(rows[0])?.payload; - const decoded = typeof raw === "string" ? decodeJson(raw) : Option.none(); - const jobIds = Option.isSome(decoded) ? record(decoded.value)?.jobIds : undefined; - if (!Array.isArray(jobIds)) return false; - for (const id of jobIds.slice(0, 8)) { - if (typeof id !== "string" || !JOB_ID.test(id)) continue; - if (observedJobIds?.has(id)) continue; - const existing = yield* readBinding(id); - if (!existing || existing.panelId !== panel.id || existing.threadId !== panel.threadId) - return true; - } - return false; - }, - ); - - const register = Effect.fn("RelayWorkerBridge.register")(function* ( - binding: RelayBinding, - activate = true, - recovering = false, - ) { - const existing = yield* readBinding(binding.id); - if ( - existing && - (existing.threadId !== binding.threadId || - existing.kind !== binding.kind || - existing.environmentId !== binding.environmentId) - ) { - yield* Effect.logWarning("Relay ID already belongs to another thread", { id: binding.id }); - return false; - } - if (!existing) { - yield* append( - binding, - "relay.binding", - { - kind: binding.kind, - id: binding.id, - toolCallId: binding.toolCallId, - environmentId: binding.environmentId, - ...(binding.panelId ? { panelId: binding.panelId } : {}), - ...(binding.agentIndex !== undefined ? { agentIndex: binding.agentIndex } : {}), - ...(binding.slotAttempt !== undefined ? { slotAttempt: binding.slotAttempt } : {}), - }, - "Relay worker bound", - `relay-binding:${binding.id}`, - "v1", - ); - } - if (activate) { - if ( - recovering && - existing?.retiredAfterRowId !== undefined && - !(yield* hasActivationSince(existing, existing.retiredAfterRowId)) - ) - return true; - active.set(binding.id, existing ?? binding); - if (recovering) recovered.add(binding.id); - else recovered.delete(binding.id); - } - return true; - }); - - const recordReceipt = Effect.fn("RelayWorkerBridge.recordReceipt")(function* ( - receipt: RelayToolReceipt, - activate = true, - ) { - const binding = { ...receipt.binding, environmentId }; - if (!(yield* register(binding, activate))) return; - if (activate && receipt.toolName === "relay_resume") { - const member = yield* readBinding(binding.id); - if (member?.panelId) { - const panel = yield* readBinding(member.panelId); - if ( - panel?.kind === "panel" && - panel.threadId === member.threadId && - panel.environmentId === member.environmentId - ) { - active.set(panel.id, panel); - } - } - } - if (receipt.toolName === "relay_resume" && receipt.attempt !== undefined) { - if (yield* readActivation(binding, receipt.attempt)) return; - yield* append( - binding, - "relay.activation", - { - id: binding.id, - kind: "job", - attempt: receipt.attempt, - toolCallId: binding.toolCallId, - environmentId, - }, - "Relay worker resumed", - `relay-activation:${binding.id}:${receipt.attempt}`, - "v1", - ); - } else if ( - (receipt.toolName === "relay_panel" || receipt.toolName === "relay_panel_continue") && - receipt.jobIds?.length - ) { - yield* append( - binding, - "relay.activation", - { - id: binding.id, - kind: "panel", - jobIds: receipt.jobIds, - toolCallId: binding.toolCallId, - environmentId, - }, - "Relay panel dispatched", - `relay-activation:${binding.id}:${binding.toolCallId}`, - "v1", - ); - } - }); - - // Provider ingestion persists the direct MCP completion before it invokes - // this observer. If the process dies in between, the accepted provider - // receipt can still establish the same exact thread/turn/tool-call binding. - const recoverPersistedToolReceipts = Effect.fn("RelayWorkerBridge.recoverPersistedToolReceipts")( - function* () { - let offset = 0; - for (;;) { - const rows = yield* sql` - SELECT activity_id AS activityId, thread_id AS threadId, - turn_id AS turnId, created_at AS createdAt, payload_json AS payload - FROM projection_thread_activities - WHERE kind = 'tool.completed' - AND ( - json_extract(payload_json, '$.data.toolName') IN ( - 'mcp__relay__relay_delegate', - 'mcp__relay__relay_resume', - 'mcp__relay__relay_panel', - 'mcp__relay__relay_panel_continue', - 'mcp__plugin_relay-orchestrator_relay__relay_delegate', - 'mcp__plugin_relay-orchestrator_relay__relay_resume', - 'mcp__plugin_relay-orchestrator_relay__relay_panel', - 'mcp__plugin_relay-orchestrator_relay__relay_panel_continue' - ) OR ( - json_extract(payload_json, '$.data.item.server') = 'relay' - AND json_extract(payload_json, '$.data.item.tool') IN ( - 'relay_delegate', 'relay_resume', 'relay_panel', 'relay_panel_continue' - ) - ) - ) - ORDER BY created_at, activity_id LIMIT 256 OFFSET ${offset} - `; - for (const raw of rows) { - const row = record(raw); - if ( - !row || - typeof row.payload !== "string" || - typeof row.threadId !== "string" || - typeof row.turnId !== "string" || - typeof row.activityId !== "string" || - typeof row.createdAt !== "string" - ) - continue; - const decoded = decodeJson(row.payload); - const payload = Option.isSome(decoded) ? record(decoded.value) : undefined; - if (!payload || typeof payload.toolCallId !== "string") continue; - const event = { - type: "item.completed", - eventId: EventId.make(row.activityId), - provider: ProviderDriverKind.make("claudeAgent"), - threadId: ThreadId.make(row.threadId), - turnId: TurnId.make(row.turnId), - itemId: RuntimeItemId.make(payload.toolCallId), - createdAt: row.createdAt, - payload: { - itemType: "mcp_tool_call", - status: payload.status, - data: payload.data, - }, - } as ProviderRuntimeEvent; - const receipt = relayReceiptFromToolEvent(event); - if (!receipt) continue; - const existing = yield* readBinding(receipt.binding.id); - if (existing && receipt.toolName === "relay_delegate") continue; - if ( - existing && - receipt.toolName === "relay_resume" && - receipt.attempt !== undefined && - (yield* readActivation(existing, receipt.attempt)) - ) - continue; - if ( - existing && - (receipt.toolName === "relay_panel" || receipt.toolName === "relay_panel_continue") && - (!receipt.jobIds?.length || - (yield* sql` - SELECT 1 FROM projection_thread_activities - WHERE activity_id = ${`relay-activation:${existing.id}:${receipt.binding.toolCallId}`} - AND kind = 'relay.activation' LIMIT 1 - `).length > 0) - ) - continue; - yield* recordReceipt(receipt, false); - } - if (rows.length < 256) break; - offset += rows.length; - } - }, - ); - - const runCli = Effect.fn("RelayWorkerBridge.runCli")(function* ( - args: ReadonlyArray, - timeout = 12_000, - ) { - if (!enabled || !cliPath) return Option.none<{ stdout: string; stderr: string }>(); - return yield* Effect.tryPromise({ - try: () => - execFile(process.execPath, [cliPath, ...args], { - timeout, - maxBuffer: args[0] === "panel-result" ? MAX_MCP_RECEIPT_CODE_UNITS : MAX_OBSERVE_BYTES, - windowsHide: true, - }), - catch: () => undefined, - }).pipe(Effect.option); - }); - - const parseLegacyCliObservation = Effect.fn("RelayWorkerBridge.parseLegacyCliObservation")( - function* (value: unknown, id: string) { - if (!PANEL_ID.test(id)) return legacyObservation(value, id); - const previousJobs = record(value)?.previousJobs; - if (!Array.isArray(previousJobs) || previousJobs.length > 64) return undefined; - const previousIds = previousJobs.filter( - (previousId): previousId is string => - typeof previousId === "string" && JOB_ID.test(previousId), - ); - if (previousIds.length !== previousJobs.length) return undefined; - const priorSlots = legacyPriorJobSlots.get(id) ?? new Map(); - const missing = previousIds.filter((previousId) => !priorSlots.has(previousId)); - // Old retry records are immutable. Cache resolved slots and bound an - // unavailable history lookup so a current panel cannot stall the sweep. - const resolved = yield* Effect.forEach( - missing, - (previousId) => - Effect.gen(function* () { - const result = yield* runCli(["status", previousId], 2_000); - const decoded = Option.isSome(result) ? decodeJson(result.value.stdout) : Option.none(); - const job = Option.isSome(decoded) ? record(decoded.value) : undefined; - const index = nonNegativeInteger(job?.panelMemberIndex); - return job?.id === previousId && job.panelId === id && index !== undefined && index <= 7 - ? ([previousId, index] as const) - : undefined; - }), - { concurrency: 8 }, - ); - if (resolved.some((entry) => entry === undefined)) return undefined; - for (const entry of resolved) if (entry) priorSlots.set(entry[0], entry[1]); - legacyPriorJobSlots.set(id, priorSlots); - return legacyObservation(value, id, priorSlots); - }, - ); - - const readLatestTaskActivity = Effect.fn("RelayWorkerBridge.readLatestTaskActivity")(function* ( - binding: RelayBinding, - ) { - const rows = yield* sql` - SELECT rowid AS rowId, kind, payload_json AS payload FROM projection_thread_activities - WHERE thread_id = ${binding.threadId} - AND kind IN ('task.started', 'task.progress', 'task.updated', 'task.completed') - AND CASE WHEN json_valid(payload_json) THEN json_extract(payload_json, '$.taskId') END = ${bindingTaskId(binding)} - ORDER BY sequence DESC, created_at DESC, rowid DESC LIMIT 1 - `; - const row = record(rows[0]); - const decoded = typeof row?.payload === "string" ? decodeJson(row.payload) : Option.none(); - return { - kind: row?.kind, - rowId: nonNegativeInteger(row?.rowId) ?? 0, - payload: Option.isSome(decoded) ? record(decoded.value) : undefined, - }; - }); - - const hasProjectedWorkerActivity = Effect.fn("RelayWorkerBridge.hasProjectedWorkerActivity")( - function* (binding: RelayBinding) { - const rows = yield* sql` - SELECT 1 FROM projection_thread_activities - WHERE thread_id = ${binding.threadId} AND kind LIKE 'task.%' - AND activity_id NOT LIKE 'relay-observer-unavailable:%' - AND CASE WHEN json_valid(payload_json) THEN json_extract(payload_json, '$.taskId') END = ${bindingTaskId(binding)} - LIMIT 1 - `; - return rows.length > 0; - }, - ); - - /** An activation for this binding (or its panel) recorded after `afterRowId`. */ - const hasActivationSince = Effect.fn("RelayWorkerBridge.hasActivationSince")(function* ( - binding: RelayBinding, - afterRowId: number, - ) { - const rows = yield* sql` - SELECT 1 FROM projection_thread_activities - WHERE thread_id = ${binding.threadId} AND kind = 'relay.activation' - AND rowid > ${afterRowId} - AND json_extract(payload_json, '$.environmentId') = ${environmentId} - AND ( - json_extract(payload_json, '$.id') = ${binding.id} - OR ( - json_extract(payload_json, '$.id') = ${binding.panelId ?? ""} - AND EXISTS ( - SELECT 1 FROM json_each(json_extract(payload_json, '$.jobIds')) - WHERE value = ${binding.id} - ) - ) - ) - LIMIT 1 - `; - return rows.length > 0; - }); - - /** - * Record that Relay reported this job gone or already settled during recovery - * while nothing had reached the thread. The binding row is upserted in place, - * so this costs no new activity and stays invisible to the transcript. Boot - * then skips it instead of re-probing every job the environment has ever run. - */ - const markRetired = Effect.fn("RelayWorkerBridge.markRetired")(function* (binding: RelayBinding) { - const watermark = yield* sql` - SELECT MAX(rowid) AS rowId FROM projection_thread_activities - WHERE thread_id = ${binding.threadId} - `; - const retiredAfterRowId = nonNegativeInteger(record(watermark[0])?.rowId); - if (retiredAfterRowId === undefined) return; - yield* append( - binding, - "relay.binding", - { - kind: binding.kind, - id: binding.id, - toolCallId: binding.toolCallId, - environmentId: binding.environmentId, - ...(binding.panelId ? { panelId: binding.panelId } : {}), - ...(binding.agentIndex !== undefined ? { agentIndex: binding.agentIndex } : {}), - ...(binding.slotAttempt !== undefined ? { slotAttempt: binding.slotAttempt } : {}), - retiredAfterRowId, - }, - "Relay worker bound", - `relay-binding:${binding.id}`, - `retired:${retiredAfterRowId}`, - ); - }); - - const shouldObserveOnBoot = Effect.fn("RelayWorkerBridge.shouldObserveOnBoot")(function* ( - binding: RelayBinding, - bindingRowId: number, - ) { - // A retired binding stays asleep unless an activation arrived after it was - // retired. One that did wake still answers to the normal rules below, so a - // resumed worker that has since finished is not re-probed forever. - if ( - binding.retiredAfterRowId !== undefined && - !(yield* hasActivationSince(binding, binding.retiredAfterRowId)) - ) - return false; - const latest = yield* readLatestTaskActivity(binding); - if (latest.kind !== "task.completed" || bindingRowId > latest.rowId) return true; - // A resume or panel continuation accepted after the terminal row must be - // reconciled even if the process stopped before emitting task.started. - const activations = yield* sql` - SELECT 1 FROM projection_thread_activities - WHERE rowid > ${latest.rowId} AND thread_id = ${binding.threadId} - AND kind = 'relay.activation' - AND json_extract(payload_json, '$.environmentId') = ${environmentId} - AND ( - json_extract(payload_json, '$.id') = ${binding.id} - OR ( - json_extract(payload_json, '$.id') = ${binding.panelId ?? ""} - AND EXISTS ( - SELECT 1 FROM json_each(json_extract(payload_json, '$.jobIds')) - WHERE value = ${binding.id} - ) - ) - ) - LIMIT 1 - `; - return activations.length > 0; - }); - - const pendingGenerationAfter = Effect.fn("RelayWorkerBridge.pendingGenerationAfter")(function* ( - binding: RelayBinding, - completedRowId: number, - completedAttempt: number, - ) { - if (binding.kind === "panel") { - // A resumed member reopens its completed panel even though the direct - // relay_resume receipt names the job, not the panel. Resolve that ID - // through the persisted owner binding before considering the receipt. - const rows = yield* sql` - SELECT activation.turn_id AS turnId, activation.payload_json AS payload - FROM projection_thread_activities AS activation - WHERE activation.rowid > ${completedRowId} - AND activation.thread_id = ${binding.threadId} - AND activation.kind = 'relay.activation' - AND json_extract(activation.payload_json, '$.environmentId') = ${environmentId} - AND ( - json_extract(activation.payload_json, '$.id') = ${binding.id} - OR EXISTS ( - SELECT 1 FROM projection_thread_activities AS child - WHERE child.activity_id = 'relay-binding:' || json_extract(activation.payload_json, '$.id') - AND child.kind = 'relay.binding' - AND child.thread_id = ${binding.threadId} - AND json_extract(child.payload_json, '$.environmentId') = ${environmentId} - AND json_extract(child.payload_json, '$.panelId') = ${binding.id} - ) - ) - ORDER BY activation.rowid DESC LIMIT 1 - `; - const activation = record(rows[0]); - const decoded = - typeof activation?.payload === "string" ? decodeJson(activation.payload) : Option.none(); - const payload = Option.isSome(decoded) ? record(decoded.value) : undefined; - const attempt = completedAttempt + 1; - return activation && Number.isSafeInteger(attempt) && typeof payload?.toolCallId === "string" - ? { - attempt, - origin: { - ...binding, - turnId: typeof activation.turnId === "string" ? activation.turnId : null, - toolCallId: payload.toolCallId, - }, - } - : undefined; - } - const activations = yield* sql` - SELECT turn_id AS turnId, payload_json AS payload - FROM projection_thread_activities - WHERE rowid > ${completedRowId} AND thread_id = ${binding.threadId} - AND kind = 'relay.activation' - AND json_extract(payload_json, '$.environmentId') = ${environmentId} - AND ( - json_extract(payload_json, '$.id') = ${binding.id} - OR ( - json_extract(payload_json, '$.id') = ${binding.panelId ?? ""} - AND EXISTS ( - SELECT 1 FROM json_each(json_extract(payload_json, '$.jobIds')) - WHERE value = ${binding.id} - ) - ) - ) - ORDER BY rowid DESC LIMIT 1 - `; - const activation = record(activations[0]); - const decoded = - typeof activation?.payload === "string" ? decodeJson(activation.payload) : Option.none(); - const payload = Option.isSome(decoded) ? record(decoded.value) : undefined; - const jobAttempt = payload?.kind === "job" ? nonNegativeInteger(payload.attempt) : undefined; - const attempt = - binding.slotAttempt === undefined - ? jobAttempt - : binding.slotAttempt * 1_000_000 + (jobAttempt ?? 1); - if ( - attempt !== undefined && - Number.isSafeInteger(attempt) && - attempt > completedAttempt && - typeof payload?.toolCallId === "string" - ) { - return { - attempt, - origin: { - ...binding, - turnId: typeof activation?.turnId === "string" ? activation.turnId : null, - toolCallId: payload.toolCallId, - }, - }; - } - if (binding.slotAttempt === undefined) return undefined; - const rows = yield* sql` - SELECT rowid AS rowId FROM projection_thread_activities - WHERE activity_id = ${`relay-binding:${binding.id}`} AND kind = 'relay.binding' - LIMIT 1 - `; - const bindingRowId = nonNegativeInteger(record(rows[0])?.rowId) ?? 0; - const firstAttempt = binding.slotAttempt * 1_000_000 + 1; - return bindingRowId > completedRowId && firstAttempt > completedAttempt - ? { attempt: firstAttempt, origin: binding } - : undefined; - }); - - const getOutageEpoch = Effect.fn("RelayWorkerBridge.getOutageEpoch")(function* ( - binding: RelayBinding, - ) { - const cached = outageEpochs.get(binding.id); - if (cached !== undefined) return cached; - const rows = yield* sql` - SELECT MAX(CAST(json_extract(payload_json, '$.outageEpoch') AS INTEGER)) AS epoch - FROM projection_thread_activities - WHERE thread_id = ${binding.threadId} AND kind = 'task.progress' - AND json_extract(payload_json, '$.taskId') = ${bindingTaskId(binding)} - AND json_type(payload_json, '$.outageEpoch') = 'integer' - `; - const epoch = nonNegativeInteger(record(rows[0])?.epoch) ?? 0; - outageEpochs.set(binding.id, epoch); - return epoch; - }); - - // A single bounded prefix replaces all prior usage rows in the client fold. - // Each attempt contributes its maximum observed cumulative usage once. - const readPriorUsage = Effect.fn("RelayWorkerBridge.readPriorUsage")(function* ( - binding: RelayBinding, - attempt: number, - ) { - const cached = priorUsageCache.get(binding.id); - if (cached?.attempt === attempt) return cached.usage; - const rows = yield* sql` - WITH prior_attempts AS ( - SELECT - MAX(json_extract(payload_json, '$.typedUsage.totalTokens')) AS totalTokens, - MAX(json_extract(payload_json, '$.typedUsage.inputTokens')) AS inputTokens, - MAX(json_extract(payload_json, '$.typedUsage.outputTokens')) AS outputTokens, - MAX(json_extract(payload_json, '$.typedUsage.cachedInputTokens')) AS cachedInputTokens - FROM projection_thread_activities - WHERE thread_id = ${binding.threadId} - AND kind IN ('task.progress', 'task.completed') - AND json_extract(payload_json, '$.source') = 'relay' - AND json_extract(payload_json, '$.taskId') = ${bindingTaskId(binding)} - AND json_type(payload_json, '$.attempt') = 'integer' - AND json_extract(payload_json, '$.attempt') < ${attempt} - AND json_type(payload_json, '$.typedUsage.totalTokens') = 'integer' - GROUP BY json_extract(payload_json, '$.attempt') - ) - SELECT SUM(totalTokens) AS totalTokens, SUM(inputTokens) AS inputTokens, - SUM(outputTokens) AS outputTokens, SUM(cachedInputTokens) AS cachedInputTokens - FROM prior_attempts - `; - const row = record(rows[0]); - const totalTokens = nonNegativeInteger(row?.totalTokens); - const inputTokens = nonNegativeInteger(row?.inputTokens); - const outputTokens = nonNegativeInteger(row?.outputTokens); - const cachedInputTokens = nonNegativeInteger(row?.cachedInputTokens); - const usage: RelayUsageRollup | undefined = - totalTokens === undefined - ? undefined - : { - totalTokens, - ...(inputTokens === undefined ? {} : { inputTokens }), - ...(outputTokens === undefined ? {} : { outputTokens }), - ...(cachedInputTokens === undefined ? {} : { cachedInputTokens }), - }; - priorUsageCache.set(binding.id, { attempt, usage }); - return usage; - }); - - const markObserverUnavailable = Effect.fn("RelayWorkerBridge.markObserverUnavailable")(function* ( - binding: RelayBinding, - ) { - const failures = (failedObservations.get(binding.id) ?? 0) + 1; - failedObservations.set(binding.id, failures); - if (failures < 3) return; - if (unavailable.has(binding.id)) { - // An announced outage is not re-announced, but the binding still owns a - // CLI spawn on every sweep until its recheck is pushed back out. - missingObservationRetryAt.set(binding.id, (yield* Clock.currentTimeMillis) + 60_000); - return; - } - const taskId = bindingTaskId(binding); - // An outage row explains a worker the thread already shows. The startup - // receipt scan also adopts long-finished historical jobs whose Relay - // files are gone; those never projected a row, so announcing them turns - // every reopened thread into a wall of idle workers. - if (!(yield* hasProjectedWorkerActivity(binding))) { - // Relay itself answered that this job is gone, and nothing about it ever - // reached the thread, so there is no observation left to resume. Retire - // it for good: otherwise every job the environment has ever run is - // re-probed on each boot and holds a CLI spawn a minute in between. A - // worker that does come back re-registers through its own receipt, and - // its activation outranks the retirement. When Relay is absent entirely - // the silence says nothing about the job, so keep the binding live and - // only slow the recheck down. - if (enabled && cliPath) { - yield* markRetired(binding); - retire(binding.id); - } else { - unavailable.add(binding.id); - missingObservationRetryAt.set(binding.id, (yield* Clock.currentTimeMillis) + 60_000); - } - return; - } - const latest = yield* readLatestTaskActivity(binding); - const payload = latest.payload; - const completedAttempt = nonNegativeInteger(payload?.attempt); - const completedSequence = nonNegativeInteger(payload?.relaySequence); - if (completedAttempt === undefined || completedSequence === undefined) { - unavailable.add(binding.id); - missingObservationRetryAt.set(binding.id, (yield* Clock.currentTimeMillis) + 60_000); - return; - } - const pending = - latest.kind === "task.completed" - ? yield* pendingGenerationAfter(binding, latest.rowId, completedAttempt) - : undefined; - if (latest.kind === "task.completed") { - if (!pending) { - retire(binding.id); - return; - } - } - const attempt = pending?.attempt ?? completedAttempt; - const relaySequence = pending ? 0 : completedSequence; - const origin = pending?.origin ?? binding; - const relayPriorUsage = yield* readPriorUsage(binding, attempt); - const outageEpoch = (yield* getOutageEpoch(binding)) + 1; - outageEpochs.set(binding.id, outageEpoch); - yield* append( - origin, - "task.progress", - { - taskId, - title: binding.kind === "panel" ? "Relay panel" : "Relay worker", - attempt, - relaySequence, - outageEpoch, - status: "idle", - summary: "Relay observer unavailable", - detail: "Relay observer unavailable", - ...bindingPayload(origin), - ...(relayPriorUsage ? { relayPriorUsage } : {}), - cancellable: false, - }, - "Relay observer unavailable", - `relay-observer-unavailable:${binding.id}:${outageEpoch}`, - `${attempt}:${relaySequence}:${outageEpoch}`, - ); - unavailable.add(binding.id); - }); - - const poll = Effect.fn("RelayWorkerBridge.poll")(function* (binding: RelayBinding) { - if ((missingObservationRetryAt.get(binding.id) ?? 0) > (yield* Clock.currentTimeMillis)) return; - const result = yield* runCli( - binding.kind === "job" ? ["observe", binding.id] : ["observe", "--panel", binding.id], - ); - const parsed = Option.isSome(result) ? decodeJson(result.value.stdout) : Option.none(); - let observation = parseObservation( - Option.isSome(parsed) ? parsed.value : undefined, - binding.id, - ); - if (!observation && Option.isSome(parsed) && legacyObserveUnavailable(parsed.value)) { - const legacy = yield* runCli( - binding.kind === "job" ? ["status", binding.id] : ["panel-result", binding.id], - ); - const decoded = Option.isSome(legacy) ? decodeJson(legacy.value.stdout) : Option.none(); - observation = yield* parseLegacyCliObservation( - Option.isSome(decoded) ? decoded.value : undefined, - binding.id, - ); - } - if (!observation) return yield* markObserverUnavailable(binding); - if ( - recovered.has(binding.id) && - isSettledObservation(observation) && - !(yield* hasProjectedWorkerActivity(binding)) - ) { - // A resume/continuation may have been accepted immediately before the - // crash, while Relay still reports the previous settled generation. - if (observation.kind === "job") { - const attempt = - binding.slotAttempt === undefined - ? observation.attempt - : binding.slotAttempt * 1_000_000 + observation.attempt; - if (yield* pendingGenerationAfter(binding, 0, attempt)) return; - } else if ( - yield* hasUnobservedPanelDispatch( - binding, - new Set(observation.members.flatMap((member) => (member.jobId ? [member.jobId] : []))), - ) - ) - return; - // Do not manufacture a fresh start/completion for an old, finished job - // just because its Relay files still exist. Persist retirement so later - // boots skip it; a real resume/continuation can still reactivate it. - yield* markRetired(binding); - retire(binding.id); - return; - } - missingObservationRetryAt.delete(binding.id); - const wasUnavailableInMemory = unavailable.delete(binding.id); - const lastPersisted = seen.has(binding.id) ? undefined : yield* readLatestTaskActivity(binding); - const wasUnavailable = - wasUnavailableInMemory || - (lastPersisted?.payload?.status === "idle" && - lastPersisted.payload.summary === "Relay observer unavailable"); - const recoveryEpoch = wasUnavailable ? yield* getOutageEpoch(binding) : 0; - failedObservations.delete(binding.id); - const taskId = bindingTaskId(binding); - if (observation.kind === "panel") { - const derivedAttempt = panelAttempt(observation); - if (derivedAttempt === undefined) return; - const previous = seen.get(binding.id); - const latest = previous ? undefined : yield* readLatestTaskActivity(binding); - // A retry can replace a member whose previous job was itself resumed. - // The new job's attempt resets to one, but the coordinator must never - // move backwards relative to its persisted generation. - const attempt = Math.max( - derivedAttempt, - previous?.attempt ?? nonNegativeInteger(latest?.payload?.attempt) ?? 0, - ); - const priorPanelJobs = currentPanelJobs.get(binding.id); - currentPanelJobs.set( - binding.id, - new Map( - observation.members.flatMap((member) => - member.jobId && JOB_ID.test(member.jobId) - ? [[member.index, member.jobId] as const] - : [], - ), - ), - ); - const fingerprint = observationFingerprint(observation); - if (!previous || previous.attempt !== attempt) { - yield* append( - binding, - "task.started", - { - taskId, - title: "Relay panel", - detail: "Relay panel", - attempt, - relaySequence: 0, - ...bindingPayload(binding), - }, - "Relay panel started", - attempt === 1 ? `relay-start:${binding.id}` : `relay-start:${binding.id}:${attempt}`, - "v1", - ); - } - for (const member of observation.members) { - if ( - nonNegativeInteger(member.index) === undefined || - member.index > 7 || - nonNegativeInteger(member.slotAttempt) === undefined || - member.slotAttempt < 1 || - !member.jobId || - !JOB_ID.test(member.jobId) - ) - continue; - if (priorPanelJobs?.get(member.index) === member.jobId) continue; - const origin = yield* readPanelJobOrigin(binding, member.jobId); - yield* register( - { - kind: "job", - id: member.jobId, - threadId: binding.threadId, - turnId: origin?.turnId ?? binding.turnId, - toolCallId: origin?.toolCallId ?? binding.toolCallId, - environmentId: binding.environmentId, - panelId: binding.id, - agentIndex: member.index, - slotAttempt: member.slotAttempt, - }, - true, - recovered.has(binding.id), - ); - } - const hasRunningMember = observation.members.some( - (member) => - member.job && ["starting", "queued", "running", "cancelling"].includes(member.job.status), - ); - if (previous?.state !== fingerprint || wasUnavailable) { - const panelStatus = observation.complete - ? "completed" - : hasRunningMember - ? "running" - : "idle"; - const summary = `${observation.members.filter((member) => member.jobId).length} of ${observation.members.length} members dispatched`; - yield* append( - binding, - "task.progress", - { - taskId, - title: "Relay panel", - attempt, - relaySequence: 0, - status: panelStatus, - summary, - detail: summary, - ...bindingPayload(binding), - }, - "Relay panel", - attempt === 1 - ? `relay-panel-progress:${binding.id}` - : `relay-panel-progress:${binding.id}:${attempt}`, - wasUnavailable ? `${fingerprint}:recovered:${recoveryEpoch}` : fingerprint, - ); - } - seen.set(binding.id, { attempt, sequence: 0, state: fingerprint }); - const hasPendingDispatch = - !observation.complete && !hasRunningMember - ? yield* hasUnobservedPanelDispatch(binding) - : false; - const pendingPolls = hasPendingDispatch - ? (pendingPanelDispatchPolls.get(binding.id) ?? 0) + 1 - : 0; - if (pendingPolls > 0) pendingPanelDispatchPolls.set(binding.id, pendingPolls); - else pendingPanelDispatchPolls.delete(binding.id); - if (observation.complete) { - yield* append( - binding, - "task.completed", - { - taskId, - status: "completed", - title: "Relay panel", - attempt, - relaySequence: 0, - ...bindingPayload(binding), - }, - "Relay panel completed", - attempt === 1 - ? `relay-panel-complete:${binding.id}` - : `relay-panel-complete:${binding.id}:${attempt}`, - fingerprint, - ); - retire(binding.id); - } else if (!hasRunningMember && (!hasPendingDispatch || pendingPolls >= 3)) { - // Failed/cancelled members and unstarted slots remain visible as an - // idle panel. A new continuation or member resume receipt reactivates - // observation; polling this unchanged snapshot forever is needless. - retire(binding.id); - } - return; - } - - const previous = seen.get(binding.id); - const origin = (yield* readActivation(binding, observation.attempt)) ?? binding; - const status = relayStatus(observation.status); - const fingerprint = observationFingerprint(observation); - const displayAttempt = - binding.slotAttempt === undefined - ? observation.attempt - : binding.slotAttempt * 1_000_000 + observation.attempt; - if (!Number.isSafeInteger(displayAttempt)) return; - const relayPriorUsage = yield* readPriorUsage(binding, displayAttempt); - if ( - previous && - (displayAttempt < previous.attempt || - (displayAttempt === previous.attempt && observation.sequence < previous.sequence)) - ) - return; - if (previous?.state === fingerprint && !wasUnavailable) return; - const fields = { - taskId, - title: "Relay worker", - model: boundedText(observation.model, 100), - effort: boundedText(observation.effort, 100), - role: boundedText(observation.providerType, 100), - attempt: displayAttempt, - relaySequence: observation.sequence, - ...bindingPayload(origin), - ...(relayPriorUsage ? { relayPriorUsage } : {}), - }; - if (!previous || previous.attempt !== displayAttempt) { - yield* append( - origin, - "task.started", - { ...fields, detail: "Relay worker" }, - "Relay worker started", - `relay-start:${binding.id}:${displayAttempt}`, - "v1", - ); - } - const usage = record(observation.usage); - const typedUsage = - usage && nonNegativeInteger(usage.totalTokens) !== undefined - ? { - totalTokens: usage.totalTokens, - ...(nonNegativeInteger(usage.inputTokens) !== undefined - ? { inputTokens: usage.inputTokens } - : {}), - ...(nonNegativeInteger(usage.outputTokens) !== undefined - ? { outputTokens: usage.outputTokens } - : {}), - ...(nonNegativeInteger(usage.cachedInputTokens) !== undefined - ? { cachedInputTokens: usage.cachedInputTokens } - : {}), - } - : undefined; - if ( - !["completed", "failed", "cancelled", "interrupted"].includes(status) || - observation.pending - ) { - const activity = boundedText(observation.activity?.label); - yield* append( - origin, - "task.progress", - { - ...fields, - status: - observation.pending && - ["completed", "failed", "cancelled", "interrupted"].includes(status) - ? "running" - : status, - ...(activity ? { summary: activity, detail: activity } : {}), - ...(boundedText(observation.activity?.kind) - ? { lastToolName: boundedText(observation.activity?.kind) } - : {}), - ...(typedUsage ? { typedUsage } : {}), - }, - "Relay worker", - `relay-progress:${binding.id}`, - wasUnavailable ? `${fingerprint}:recovered:${recoveryEpoch}` : fingerprint, - ); - } - seen.set(binding.id, { - attempt: displayAttempt, - sequence: observation.sequence, - state: fingerprint, - }); - if ( - ["completed", "failed", "cancelled", "interrupted"].includes(status) && - !observation.pending - ) { - yield* append( - origin, - "task.completed", - { - ...fields, - status: status === "interrupted" ? "stopped" : status, - ...(boundedText(observation.outcome?.summary, 2000) || - boundedText(observation.outcome?.error, 1000) || - boundedText(observation.outcome?.label) - ? { - summary: - boundedText(observation.outcome?.summary, 2000) ?? - boundedText(observation.outcome?.error, 1000) ?? - boundedText(observation.outcome?.label), - } - : {}), - ...(typedUsage ? { typedUsage } : {}), - }, - status === "completed" - ? "Relay worker completed" - : status === "cancelled" - ? "Relay worker stopped" - : "Relay worker failed", - `relay-complete:${binding.id}:${observation.attempt}`, - fingerprint, - ); - retire(binding.id); - } - }); - - let bootstrapped = false; - const sweep = Effect.fn("RelayWorkerBridge.sweep")(function* () { - // Direct receipts register immediately. This bounded startup scan closes - // the crash window after command dispatch and before observer registration. - if (!bootstrapped) { - yield* recoverPersistedToolReceipts(); - // Replay inserts historical bindings in receipt order. Visit newest - // bindings first so current workers project before old missing jobs. - let beforeRowId = Number.MAX_SAFE_INTEGER; - for (;;) { - const page = yield* readBindingPage(beforeRowId); - if (page.length === 0) break; - for (const { rowId, binding } of page) { - beforeRowId = Math.min(beforeRowId, rowId); - if (binding && !active.has(binding.id) && (yield* shouldObserveOnBoot(binding, rowId))) { - active.set(binding.id, binding); - recovered.add(binding.id); - } - } - if (page.length < 256) break; - } - bootstrapped = true; - } - const observeBound = (binding: RelayBinding) => - poll(binding).pipe( - Effect.catchCause((cause) => - Effect.logWarning("Relay observation failed", { id: binding.id, cause }), - ), - ); - // A member may be resumed after its completed panel was evicted. On boot - // and in-process, confirm the original parent binding before observing - // the panel again; its snapshot is the authority for the current slot. - for (const binding of Array.from(active.values())) { - if (binding.kind !== "job" || !binding.panelId) continue; - if (active.has(binding.panelId) || currentPanelJobs.has(binding.panelId)) continue; - const panel = yield* readBinding(binding.panelId); - if ( - panel?.kind === "panel" && - panel.threadId === binding.threadId && - panel.environmentId === binding.environmentId - ) { - active.set(panel.id, panel); - if (recovered.has(binding.id)) recovered.add(panel.id); - } - } - for (const binding of active.values()) { - if (binding.kind === "panel") yield* observeBound(binding); - } - for (const binding of active.values()) { - if (binding.kind !== "job") continue; - if (binding.panelId !== undefined && binding.agentIndex !== undefined) { - const current = currentPanelJobs.get(binding.panelId); - if (!current) continue; // Panel observation must confirm a current slot before projection. - if (current.get(binding.agentIndex) !== binding.id) { - active.delete(binding.id); - recovered.delete(binding.id); - continue; - } - } - yield* observeBound(binding); - } - for (const panelId of currentPanelJobs.keys()) { - if (active.has(panelId)) continue; - if ([...active.values()].some((binding) => binding.panelId === panelId)) continue; - currentPanelJobs.delete(panelId); - } - }); - - const start = forkParked( - sweep().pipe( - Effect.catchCause((cause) => Effect.logWarning("Relay binding sweep failed", { cause })), - Effect.repeat(Schedule.spaced("3 seconds")), - Effect.asVoid, - ), - ); - - const recordToolResult = (event: ProviderRuntimeEvent) => - enabled - ? Effect.gen(function* () { - const receipt = relayReceiptFromToolEvent(event); - if (receipt) yield* recordReceipt(receipt); - }).pipe( - Effect.catchCause((cause) => - Effect.logWarning("Relay binding failed", { eventId: event.eventId, cause }), - ), - ) - : Effect.void; - - const cancel = Effect.fn("RelayWorkerBridge.cancel")(function* ( - threadId: ThreadId, - agentId: string, - ) { - if (!enabled) return yield* new ProviderCancelSessionAgentError({ reason: "unsupported" }); - const readAuthorizedBinding = (id: string) => - readBinding(id).pipe( - Effect.mapError(() => new ProviderCancelSessionAgentError({ reason: "request-failed" })), - ); - const slot = /^relay-panel:(panel-[0-9a-f-]{36}):member:([0-7])$/i.exec(agentId); - let binding: RelayBinding | undefined; - if (slot && PANEL_ID.test(slot[1] ?? "")) { - const panelId = slot[1]!; - const index = Number(slot[2]); - const panel = yield* readAuthorizedBinding(panelId); - if ( - !panel || - panel.kind !== "panel" || - panel.threadId !== threadId || - panel.environmentId !== environmentId - ) - return yield* new ProviderCancelSessionAgentError({ reason: "agent-not-active" }); - const current = yield* runCli(["observe", "--panel", panelId]); - const decodedPanel = Option.isSome(current) - ? decodeJson(current.value.stdout) - : Option.none(); - let observedPanel = parseObservation( - Option.isSome(decodedPanel) ? decodedPanel.value : undefined, - panelId, - ); - if ( - !observedPanel && - Option.isSome(decodedPanel) && - legacyObserveUnavailable(decodedPanel.value) - ) { - const legacy = yield* runCli(["panel-result", panelId]); - const decoded = Option.isSome(legacy) ? decodeJson(legacy.value.stdout) : Option.none(); - observedPanel = yield* parseLegacyCliObservation( - Option.isSome(decoded) ? decoded.value : undefined, - panelId, - ); - } - if (observedPanel?.kind !== "panel") - return yield* new ProviderCancelSessionAgentError({ reason: "request-failed" }); - const member = observedPanel.members.find((candidate) => candidate.index === index); - if (!member?.jobId || !JOB_ID.test(member.jobId)) - return yield* new ProviderCancelSessionAgentError({ reason: "agent-not-active" }); - const currentBinding = yield* readAuthorizedBinding(member.jobId); - binding = - currentBinding?.kind === "job" && - currentBinding.panelId === panelId && - currentBinding.agentIndex === index && - currentBinding.threadId === threadId && - currentBinding.environmentId === environmentId - ? currentBinding - : undefined; - } else { - const id = agentId.startsWith("relay:") ? agentId.slice("relay:".length) : ""; - if (!JOB_ID.test(id)) - return yield* new ProviderCancelSessionAgentError({ reason: "unsupported" }); - const directBinding = yield* readAuthorizedBinding(id); - binding = - directBinding?.kind === "job" && - directBinding.panelId === undefined && - directBinding.threadId === threadId && - directBinding.environmentId === environmentId - ? directBinding - : undefined; - } - if (!binding) return yield* new ProviderCancelSessionAgentError({ reason: "agent-not-active" }); - const id = binding.id; - const observed = yield* runCli(["observe", id]); - const decodedJob = Option.isSome(observed) ? decodeJson(observed.value.stdout) : Option.none(); - let current = parseObservation(Option.isSome(decodedJob) ? decodedJob.value : undefined, id); - if (!current && Option.isSome(decodedJob) && legacyObserveUnavailable(decodedJob.value)) { - const legacy = yield* runCli(["status", id]); - const decoded = Option.isSome(legacy) ? decodeJson(legacy.value.stdout) : Option.none(); - current = yield* parseLegacyCliObservation( - Option.isSome(decoded) ? decoded.value : undefined, - id, - ); - } - if (current?.kind !== "job") { - return yield* new ProviderCancelSessionAgentError({ reason: "request-failed" }); - } - const result = yield* runCli(["cancel", id]); - if (Option.isNone(result)) - return yield* new ProviderCancelSessionAgentError({ reason: "request-failed" }); - const decoded = decodeJson(result.value.stdout); - const value = Option.isSome(decoded) ? record(decoded.value) : undefined; - if (value?.id !== id || typeof value.status !== "string") - return yield* new ProviderCancelSessionAgentError({ reason: "request-failed" }); - active.set(id, binding); - return { - agentId: agentId as ProviderCancelSessionAgentResult["agentId"], - disposition: [ - "completed", - "failed", - "cancelled", - "interrupted", - "timed_out", - "output_limit", - "blocked_permissions", - "needs_review", - ].includes(value.status) - ? "already-settled" - : "cancel-requested", - } satisfies ProviderCancelSessionAgentResult; - }); - - return { - start, - reconcile: sweep().pipe( - Effect.catchCause((cause) => Effect.logWarning("Relay binding sweep failed", { cause })), - ), - recordToolResult, - cancel, - }; -}); - -export const layer = Layer.effect( - RelayWorkerBridge, - resolveRelayCliPath(process.env).pipe(Effect.flatMap(makeWithCliPath)), -); diff --git a/apps/server/src/orchestration/ThreadBackgroundLiveness.test.ts b/apps/server/src/orchestration/ThreadBackgroundLiveness.test.ts index 10e956c0dd..3ed89ed8c0 100644 --- a/apps/server/src/orchestration/ThreadBackgroundLiveness.test.ts +++ b/apps/server/src/orchestration/ThreadBackgroundLiveness.test.ts @@ -116,20 +116,16 @@ describe("ThreadBackgroundLiveness", () => { expect(liveness.getThreadBackgroundLiveness(threadId)).toBeNull(); }); - it("identifies parent-owned monitors alongside a detached Relay worker", () => { + it("session death clears every live entry, agents and monitors alike", () => { const liveness = ThreadBackgroundLiveness.make(); const threadId = "mixed-thread"; liveness.recordTaskLiveness({ threadId, - taskId: "relay:job-1", + taskId: "agent-1", taskType: "subagent", status: "running", kind: "started", - source: "relay", }); - expect(liveness.getThreadBackgroundLiveness(threadId)).toBe("working"); - expect(liveness.hasNativeBackgroundWork(threadId)).toBe(false); - liveness.recordTaskLiveness({ threadId, taskId: "native-monitor", @@ -140,12 +136,13 @@ describe("ThreadBackgroundLiveness", () => { expect(liveness.getThreadBackgroundLiveness(threadId)).toBe("working"); expect(liveness.hasNativeBackgroundWork(threadId)).toBe(true); + // No work outlives its session now that detached workers are gone. liveness.clearThreadLiveness(threadId); - expect(liveness.getThreadBackgroundLiveness(threadId)).toBe("working"); + expect(liveness.getThreadBackgroundLiveness(threadId)).toBeNull(); expect(liveness.hasNativeBackgroundWork(threadId)).toBe(false); }); - it("uses trusted source metadata instead of task ID spelling", () => { + it("never classifies liveness from task ID spelling", () => { const liveness = ThreadBackgroundLiveness.make(); liveness.recordTaskLiveness({ threadId: "native-thread", diff --git a/apps/server/src/orchestration/ThreadBackgroundLiveness.ts b/apps/server/src/orchestration/ThreadBackgroundLiveness.ts index c84ee7a560..c6082efa96 100644 --- a/apps/server/src/orchestration/ThreadBackgroundLiveness.ts +++ b/apps/server/src/orchestration/ThreadBackgroundLiveness.ts @@ -6,8 +6,7 @@ * workflow runs, Monitor watch loops); the shell previously showed nothing. * Ingestion records task lifecycle transitions and the shell query reads the * derived state at mapping time — no persistence, no migration. After a - * server restart the registry is empty until new task events arrive. External - * Relay observers replay surviving worker liveness after recovery. + * server restart the registry is empty until new task events arrive. * * "monitoring" is reserved for watch loops (monitor tasks and background * shells) when they are the ONLY live work; any agent work presents as @@ -25,7 +24,6 @@ export type ThreadBackgroundLiveness = "working" | "monitoring" | null; interface ThreadLivenessState { readonly agents: Set; readonly monitors: Set; - readonly relay: Set; } // Classification sets are the shared contracts copies (MONITOR_TASK_TYPES: @@ -60,10 +58,9 @@ export class ThreadBackgroundLivenessService extends Context.Service< readonly status: string | undefined; readonly kind: "started" | "progress" | "updated" | "completed"; readonly agentId?: string | undefined; - readonly source?: "relay" | undefined; }) => void; - /** Session death orphans provider-owned work; detached Relay workers survive. */ + /** Session death orphans provider-owned work, so the thread's entry goes. */ readonly clearThreadLiveness: (threadId: string) => void; /** @@ -71,7 +68,7 @@ export class ThreadBackgroundLivenessService extends Context.Service< * "monitoring" only when watch loops are the ONLY live work. */ readonly getThreadBackgroundLiveness: (threadId: string) => ThreadBackgroundLiveness; - /** Source-aware stop routing for the client; includes native monitors. */ + /** Stop routing for the client; includes native monitors. */ readonly hasNativeBackgroundWork: (threadId: string) => boolean; } >()("t3/orchestration/ThreadBackgroundLiveness/ThreadBackgroundLivenessService") {} @@ -87,7 +84,6 @@ export function make(): ThreadBackgroundLivenessService["Service"] { const created: ThreadLivenessState = { agents: new Set(), monitors: new Set(), - relay: new Set(), }; stateByThreadId.set(threadId, created); return created; @@ -104,7 +100,6 @@ export function make(): ThreadBackgroundLivenessService["Service"] { } state.agents.delete(taskId); state.monitors.delete(taskId); - state.relay.delete(taskId); if (state.agents.size === 0 && state.monitors.size === 0) { stateByThreadId.delete(threadId); } @@ -158,19 +153,10 @@ export function make(): ThreadBackgroundLivenessService["Service"] { const bucket = taskType !== undefined && MONITOR_TASK_TYPES.has(taskType) ? state.monitors : state.agents; bucket.add(input.taskId); - if (input.source === "relay") state.relay.add(input.taskId); }, clearThreadLiveness: (threadId) => { - const state = stateByThreadId.get(threadId); - if (!state) return; - for (const id of state.agents) { - if (!state.relay.has(id)) state.agents.delete(id); - } - for (const id of state.monitors) { - if (!state.relay.has(id)) state.monitors.delete(id); - } - if (state.agents.size === 0 && state.monitors.size === 0) stateByThreadId.delete(threadId); + stateByThreadId.delete(threadId); }, getThreadBackgroundLiveness: (threadId) => { @@ -190,13 +176,7 @@ export function make(): ThreadBackgroundLivenessService["Service"] { hasNativeBackgroundWork: (threadId) => { const state = stateByThreadId.get(threadId); if (!state) return false; - for (const id of state.agents) { - if (!state.relay.has(id)) return true; - } - for (const id of state.monitors) { - if (!state.relay.has(id)) return true; - } - return false; + return state.agents.size > 0 || state.monitors.size > 0; }, }; } diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts index 61508f5a85..3d21e3ee0b 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts @@ -2508,27 +2508,6 @@ describe("ClaudeAdapterLive", () => { ); }); - it.effect("exposes the configured Relay CLI as an optional MCP server", () => { - const cli = "/tmp/pylon-relay-cli.mjs"; - const harness = makeHarness({ environment: { ...process.env, PYLON_RELAY_CLI: cli } }); - return Effect.gen(function* () { - const adapter = yield* ClaudeAdapter; - yield* adapter.startSession({ - threadId: THREAD_ID, - provider: ProviderDriverKind.make("claudeAgent"), - runtimeMode: "full-access", - }); - - assert.deepEqual(harness.getLastCreateQueryInput()?.options.mcpServers?.relay, { - command: process.execPath, - args: [cli, "mcp"], - }); - }).pipe( - Effect.provideService(Random.Random, makeDeterministicRandomService()), - Effect.provide(harness.layer), - ); - }); - it.effect("leaves Claude Code's own default alone when the task list is off", () => { const harness = makeHarness({ claudeConfig: { taskTools: false } }); return Effect.gen(function* () { diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 247961254a..a9de959992 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -88,7 +88,6 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; -import { relayCliFromEnvironment } from "../relayMcpConfig.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; import { resolveClaudeSdkExecutablePath } from "../Drivers/ClaudeExecutable.ts"; import { claudeSignedOutMessage, makeClaudeEnvironment } from "../Drivers/ClaudeHome.ts"; @@ -5452,7 +5451,6 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( const sessionEnvironment = claudeSettings.taskTools ? { ...claudeEnvironment, CLAUDE_CODE_ENABLE_TODO_TOOLS: "1" } : claudeEnvironment; - const relayCli = relayCliFromEnvironment(sessionEnvironment); const queryOptions: ClaudeQueryOptions = { ...(input.cwd ? { cwd: input.cwd } : {}), ...(apiModelId ? { model: apiModelId } : {}), @@ -5487,21 +5485,14 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( env: McpProviderSession.withAgentDeviceEnvironment(sessionEnvironment, mcpSession), additionalDirectories, ...(Object.keys(extraArgs).length > 0 ? { extraArgs } : {}), - ...(mcpSession || relayCli + ...(mcpSession ? { mcpServers: { - ...(mcpSession - ? { - "t3-code": { - type: "http" as const, - url: mcpSession.endpoint, - headers: { Authorization: mcpSession.authorizationHeader }, - }, - } - : {}), - ...(relayCli - ? { relay: { command: process.execPath, args: [relayCli, "mcp"] } } - : {}), + "t3-code": { + type: "http" as const, + url: mcpSession.endpoint, + headers: { Authorization: mcpSession.authorizationHeader }, + }, }, } : {}), diff --git a/apps/server/src/provider/Layers/CodexAdapter.test.ts b/apps/server/src/provider/Layers/CodexAdapter.test.ts index 81150a3497..033cc3befb 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.test.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.test.ts @@ -58,7 +58,6 @@ import type { EventNdjsonLogger } from "./EventNdjsonLogger.ts"; import { readCodexExactCursor } from "./CodexAbsoluteRollback.ts"; import type { CodexConversationSnapshot } from "./CodexAbsoluteHistory.ts"; const decodeCodexSettings = Schema.decodeSync(CodexSettings); -const encodeJsonString = Schema.encodeEffect(Schema.fromJsonString(Schema.String)); // Test-local service tag so the rest of the file can keep using `yield* CodexAdapter`. class CodexAdapter extends Context.Service()( @@ -387,42 +386,6 @@ validationLayer("CodexAdapterLive validation", (it) => { ); }); -const relayValidationLayer = it.layer( - Layer.effect( - CodexAdapter, - makeCodexAdapter(decodeCodexSettings({}), { - environment: { ...process.env, PYLON_RELAY_CLI: "/tmp/pylon-relay-cli.mjs" }, - makeRuntime: validationRuntimeFactory.factory, - }), - ).pipe( - Layer.provideMerge(ServerConfig.layerTest(process.cwd(), process.cwd())), - Layer.provideMerge(ServerSettingsService.layerTest()), - Layer.provideMerge(providerSessionDirectoryTestLayer), - Layer.provideMerge(NodeServices.layer), - ), -); - -relayValidationLayer("CodexAdapterLive Relay MCP", (it) => { - it.effect("passes the configured Relay CLI alongside Pylon MCP arguments", () => - Effect.scoped( - Effect.gen(function* () { - validationRuntimeFactory.factory.mockClear(); - const adapter = yield* CodexAdapter; - yield* adapter.startSession({ - provider: ProviderDriverKind.make("codex"), - threadId: asThreadId("thread-relay-mcp"), - runtimeMode: "full-access", - }); - const args = validationRuntimeFactory.factory.mock.calls[0]?.[0].appServerArgs ?? []; - const command = yield* encodeJsonString(process.execPath); - NodeAssert.ok(args.includes(`mcp_servers.relay.command=${command}`)); - NodeAssert.ok(args.includes('mcp_servers.relay.args=["/tmp/pylon-relay-cli.mjs","mcp"]')); - NodeAssert.ok(args.includes("mcp_servers.relay.tool_timeout_sec=180.0")); - }), - ), - ); -}); - const sessionRuntimeFactory = makeRuntimeFactory(); const sessionErrorLayer = it.layer( Layer.effect( diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 7f87f7a016..e348668fa8 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -70,7 +70,6 @@ import { type CodexAdapterShape } from "../Services/CodexAdapter.ts"; import { BUILT_IN_ADAPTER_CONVERSATION_ROLLBACK_MODES } from "../Services/ProviderAdapter.ts"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; -import { relayCliFromEnvironment, relayCodexAppServerArgs } from "../relayMcpConfig.ts"; import { CodexResumeCursorSchema, CodexSessionRuntimeThreadIdMissingError, @@ -2309,7 +2308,6 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( ? getCodexServiceTierOptionValue(input.modelSelection) : undefined; const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); - const relayCli = relayCliFromEnvironment(options?.environment ?? process.env); const hasPrivateResumeProof = typeof input.resumeCursor === "object" && input.resumeCursor !== null && @@ -2355,20 +2353,15 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( mcpCapabilities: mcpSession.capabilities, } : {}), - ...(mcpSession || relayCli + ...(mcpSession ? { appServerArgs: [ - ...(mcpSession - ? [ - "-c", - `mcp_servers.t3-code.url=${mcpSession.endpoint}`, - "-c", - 'mcp_servers.t3-code.bearer_token_env_var="T3_MCP_BEARER_TOKEN"', - "-c", - `mcp_servers.t3-code.tool_timeout_sec=${(McpProviderSession.MCP_PROVIDER_TOOL_TIMEOUT_MS / 1000).toFixed(1)}`, - ] - : []), - ...(relayCli ? relayCodexAppServerArgs(relayCli) : []), + "-c", + `mcp_servers.t3-code.url=${mcpSession.endpoint}`, + "-c", + 'mcp_servers.t3-code.bearer_token_env_var="T3_MCP_BEARER_TOKEN"', + "-c", + `mcp_servers.t3-code.tool_timeout_sec=${(McpProviderSession.MCP_PROVIDER_TOOL_TIMEOUT_MS / 1000).toFixed(1)}`, ], } : {}), diff --git a/apps/server/src/provider/relayMcpConfig.ts b/apps/server/src/provider/relayMcpConfig.ts deleted file mode 100644 index 63f7819c73..0000000000 --- a/apps/server/src/provider/relayMcpConfig.ts +++ /dev/null @@ -1,19 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off -import * as NodePath from "node:path"; - -/** The environment server opts in to its own local Relay CLI installation. */ -export function relayCliFromEnvironment(environment: NodeJS.ProcessEnv): string | undefined { - const value = environment.PYLON_RELAY_CLI; - return value && NodePath.isAbsolute(value) ? value : undefined; -} - -export function relayCodexAppServerArgs(cliPath: string): ReadonlyArray { - return [ - "-c", - `mcp_servers.relay.command=${JSON.stringify(process.execPath)}`, - "-c", - `mcp_servers.relay.args=${JSON.stringify([cliPath, "mcp"])}`, - "-c", - "mcp_servers.relay.tool_timeout_sec=180.0", - ]; -} diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index b75cad333c..101bf224ab 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -83,7 +83,6 @@ import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; import { OrchestrationReactorLive } from "./orchestration/Layers/OrchestrationReactor.ts"; import { RuntimeReceiptBusLive } from "./orchestration/Layers/RuntimeReceiptBus.ts"; import { ProviderRuntimeIngestionLive } from "./orchestration/Layers/ProviderRuntimeIngestion.ts"; -import * as RelayWorkerBridge from "./orchestration/RelayWorkerBridge.ts"; import { ProviderCommandReactorLive } from "./orchestration/Layers/ProviderCommandReactor.ts"; import { CheckpointReactorLive } from "./orchestration/Layers/CheckpointReactor.ts"; import { ThreadDeletionReactorLive } from "./orchestration/Layers/ThreadDeletionReactor.ts"; @@ -324,7 +323,6 @@ const RollbackSagaRuntimeLayerLive = RollbackSagaRunner.layer.pipe( const ReactorLayerLive = Layer.empty.pipe( Layer.provideMerge(OrchestrationReactorLive), Layer.provideMerge(ProviderRuntimeIngestionLive), - Layer.provideMerge(RelayWorkerBridge.layer), Layer.provideMerge(ProviderCommandReactorLive), Layer.provideMerge(CheckpointReactorLive), Layer.provideMerge(RollbackSagaRuntimeLayerLive), diff --git a/apps/server/src/testUtils/shardWeights.json b/apps/server/src/testUtils/shardWeights.json index 18a8565f31..e624ee1eba 100644 --- a/apps/server/src/testUtils/shardWeights.json +++ b/apps/server/src/testUtils/shardWeights.json @@ -126,8 +126,6 @@ "src/orchestration/Normalizer.test.ts": 0.5, "src/orchestration/ProjectSettingsReactor.test.ts": 0.5, "src/orchestration/PullRequestSyncReactor.test.ts": 2, - "src/orchestration/RelayCli.test.ts": 0.5, - "src/orchestration/RelayWorkerBridge.test.ts": 4.4, "src/orchestration/ThreadBackgroundLiveness.test.ts": 0.6, "src/orchestration/ThreadLiveEventCoalescer.test.ts": 0.6, "src/orchestration/ThreadPlanProgress.test.ts": 0.5, diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index e96ae941fc..0fac48d6fc 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -113,7 +113,6 @@ import { normalizeDispatchCommand, } from "./orchestration/Normalizer.ts"; import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts"; -import { RelayWorkerBridge } from "./orchestration/RelayWorkerBridge.ts"; import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts"; import { ThreadDeletionReactor } from "./orchestration/Services/ThreadDeletionReactor.ts"; import { RollbackSagaRepository } from "./persistence/Services/RollbackSagas.ts"; @@ -586,9 +585,6 @@ const makeWsRpcLayer = ( const modelManifest = yield* ModelManifest.ModelManifest; const providerVersionCache = yield* ProviderMaintenance.ProviderVersionCache; const providerService = yield* ProviderService.ProviderService; - const relayWorkerBridge = Option.getOrUndefined( - yield* Effect.serviceOption(RelayWorkerBridge), - ); const providerSessionDirectory = yield* ProviderSessionDirectory.ProviderSessionDirectory; const rollbackSagaRunner = yield* RollbackSagaRunner; const rollbackSagaRepository = yield* RollbackSagaRepository; @@ -2570,27 +2566,23 @@ const makeWsRpcLayer = ( [WS_METHODS.providerCancelSessionAgent]: (input) => observeRpcEffect( WS_METHODS.providerCancelSessionAgent, - input.agentId.startsWith("relay:") || input.agentId.startsWith("relay-panel:") - ? relayWorkerBridge - ? relayWorkerBridge.cancel(input.threadId, input.agentId) - : Effect.fail(new ProviderCancelSessionAgentError({ reason: "unsupported" })) - : providerService.cancelSessionAgent(input).pipe( - Effect.mapError((error) => { - const reason = - error._tag === "ProviderUnsupportedError" || - error._tag === "ProviderAdapterUnsupportedOperationError" - ? "unsupported" - : error._tag === "ProviderAdapterValidationError" - ? "agent-not-active" - : error._tag === "ProviderAdapterSessionNotFoundError" || - error._tag === "ProviderAdapterSessionClosedError" || - error._tag === "ProviderSessionNotFoundError" || - error._tag === "ProviderValidationError" - ? "session-not-ready" - : "request-failed"; - return new ProviderCancelSessionAgentError({ reason }); - }), - ), + providerService.cancelSessionAgent(input).pipe( + Effect.mapError((error) => { + const reason = + error._tag === "ProviderUnsupportedError" || + error._tag === "ProviderAdapterUnsupportedOperationError" + ? "unsupported" + : error._tag === "ProviderAdapterValidationError" + ? "agent-not-active" + : error._tag === "ProviderAdapterSessionNotFoundError" || + error._tag === "ProviderAdapterSessionClosedError" || + error._tag === "ProviderSessionNotFoundError" || + error._tag === "ProviderValidationError" + ? "session-not-ready" + : "request-failed"; + return new ProviderCancelSessionAgentError({ reason }); + }), + ), { "rpc.aggregate": "provider" }, ), [WS_METHODS.providerMessageSessionAgent]: (input) => diff --git a/apps/web/src/components/AgentsPanel.test.tsx b/apps/web/src/components/AgentsPanel.test.tsx index 48f18e9c56..6b1147bd7f 100644 --- a/apps/web/src/components/AgentsPanel.test.tsx +++ b/apps/web/src/components/AgentsPanel.test.tsx @@ -8,7 +8,6 @@ import type { import type { ProviderSessionAgentActivitySnapshot } from "@t3tools/contracts"; import { AgentLiveActivitySnapshot } from "./AgentLiveActivity"; import { AgentsPanel } from "./AgentsPanel"; -import { planBackgroundAgentStop } from "./ChatView.logic"; function agent(id: string, title: string, status: RuntimeSubagent["status"]): RuntimeSubagent { return { @@ -57,82 +56,6 @@ const model: AgentPanelModel = { liveCount: 1, }; -describe("background agent stop routing", () => { - const relay = { ...active, id: "relay:job-1", source: "relay" as const, cancellable: true }; - const canCancel = (candidate: RuntimeSubagent) => candidate.source === "relay"; - - it("interrupts native background work through the parent session", () => { - expect(planBackgroundAgentStop([active], canCancel, true, true)).toEqual({ - relayAgentIds: [], - interruptParent: true, - canStopAll: true, - }); - }); - - it("cancels a detached Relay worker after the parent session ends", () => { - expect(planBackgroundAgentStop([relay, completed], canCancel, false, false)).toEqual({ - relayAgentIds: [relay.id], - interruptParent: false, - canStopAll: true, - }); - expect(planBackgroundAgentStop([relay], canCancel, true, false)).toEqual({ - relayAgentIds: [relay.id], - interruptParent: false, - canStopAll: true, - }); - }); - - it("addresses both sources and disables Stop when a live worker cannot be controlled", () => { - expect(planBackgroundAgentStop([active, relay], canCancel, true, true)).toEqual({ - relayAgentIds: [relay.id], - interruptParent: true, - canStopAll: true, - }); - expect( - planBackgroundAgentStop([{ ...relay, cancellable: false }], () => false, true, false), - ).toEqual({ - relayAgentIds: [], - interruptParent: false, - canStopAll: false, - }); - }); - - it("also interrupts a native monitor that has no agent row beside Relay", () => { - expect(planBackgroundAgentStop([relay], canCancel, true, true)).toEqual({ - relayAgentIds: [relay.id], - interruptParent: true, - canStopAll: true, - }); - expect(planBackgroundAgentStop([relay], canCancel, false, true)).toEqual({ - relayAgentIds: [relay.id], - interruptParent: false, - canStopAll: false, - }); - }); - - it("trusts explicit server liveness over stale native rows from a previous session", () => { - expect(planBackgroundAgentStop([active, relay], canCancel, true, false)).toEqual({ - relayAgentIds: [relay.id], - interruptParent: false, - canStopAll: true, - }); - expect(planBackgroundAgentStop([active, relay], canCancel, true, undefined)).toEqual({ - relayAgentIds: [relay.id], - interruptParent: true, - canStopAll: true, - }); - }); - - it("does not interrupt the parent for a Relay panel before its members appear", () => { - const workflow = { ...relay, id: "relay:panel-1", kind: "workflow" as const }; - expect(planBackgroundAgentStop([workflow], canCancel, true, false)).toEqual({ - relayAgentIds: [], - interruptParent: false, - canStopAll: false, - }); - }); -}); - describe("AgentsPanel agent cancellation", () => { it("puts active states first and hides inactive history without changing status labels", () => { const statuses: RuntimeSubagent["status"][] = [ @@ -281,73 +204,6 @@ describe("AgentsPanel agent cancellation", () => { expect(markup).not.toContain('aria-label="Stop '); }); - it("offers detached worker stop after parent controls disappear without message or transcript controls", () => { - const detached = { - ...active, - id: "relay:job-1", - title: "Relay reviewer", - source: "relay" as const, - cancellable: true, - watchable: false, - model: "claude-sonnet-5", - effort: "high", - }; - const markup = renderToStaticMarkup( - - candidate.source === "relay" && candidate.cancellable === true - } - canMessageAgents - canWatchAgentActivity - onCancelAgent={async () => undefined} - onMessageAgent={async () => "delivered"} - />, - ); - expect(markup).toContain('aria-label="Stop Relay reviewer"'); - expect(markup).toContain("sonnet-5 · high"); - expect(markup).not.toContain('aria-label="Message Relay reviewer"'); - expect(markup).not.toContain('aria-label="Open live activity for Relay reviewer"'); - }); - - it("shows the coordinator's total when a Relay panel only partly dispatched", () => { - const coordinator = { - ...agent("relay-panel:panel-1", "Relay panel", "idle"), - kind: "workflow" as const, - source: "relay" as const, - progress: "2 of 3 members dispatched", - }; - const members = [0, 1].map((index) => ({ - ...agent(`relay-panel:panel-1:member:${index}`, `Worker ${index + 1}`, "completed"), - source: "relay" as const, - parentAgentId: coordinator.id, - agentIndex: index, - })); - const panelModel: AgentPanelModel = { - ...model, - workflows: [{ workflow: coordinator, phases: [], unphasedMembers: members }], - directAgents: [], - }; - - const expanded = renderToStaticMarkup(); - expect(expanded).toContain("2/2 settled · 2 of 3 members dispatched"); - - const collapsed = renderToStaticMarkup( - , - ); - expect(collapsed).toContain("2 agents"); - expect(collapsed).toContain("2 of 3 members dispatched"); - }); - it("offers messaging only for provider-marked active agents when enabled", () => { const unmessageable = { ...active, diff --git a/apps/web/src/components/AgentsPanel.tsx b/apps/web/src/components/AgentsPanel.tsx index db5f06557d..26c088800b 100644 --- a/apps/web/src/components/AgentsPanel.tsx +++ b/apps/web/src/components/AgentsPanel.tsx @@ -213,18 +213,11 @@ function AgentRow({ ].filter((value): value is string => value !== null); const active = isActiveSubagentStatus(agent.status); const messageable = - messageControls.enabled && - agent.source !== "relay" && - agent.kind !== "workflow" && - agent.messageable && - active; + messageControls.enabled && agent.kind !== "workflow" && agent.messageable && active; const cancellable = cancelControls.canRequest(agent) && agent.kind !== "workflow" && active; const stopping = cancellable && cancelControls.pendingIds.has(agent.id); const liveActivityEligible = - liveActivityControls.enabled && - agent.watchable !== false && - agent.source !== "relay" && - agent.kind !== "workflow"; + liveActivityControls.enabled && agent.watchable !== false && agent.kind !== "workflow"; const liveActivityAvailable = liveActivityEligible && active; return ( @@ -574,9 +567,6 @@ function ExpandedWorkflowSection({ ) : null} {settled}/{members.length} settled - {group.workflow.source === "relay" && group.workflow.progress - ? ` · ${group.workflow.progress}` - : null} @@ -6835,7 +6794,7 @@ export default function ChatView(props: ChatViewProps) { addAgentsSurface, agentsSurfaceOnScreen, agentPanelModel.liveCount, - backgroundStopPlan.canStopAll, + canStopBackgroundWork, handleStopBackgroundWork, isStoppingBackgroundWork, ]); diff --git a/apps/web/src/session-logic.test.ts b/apps/web/src/session-logic.test.ts index b40de5f84b..b3ba027be5 100644 --- a/apps/web/src/session-logic.test.ts +++ b/apps/web/src/session-logic.test.ts @@ -576,22 +576,6 @@ describe("workEntryIndicatesToolNeutralStatus", () => { }); describe("deriveWorkLogEntries", () => { - it("does not expose Relay ownership receipts in the conversation", () => { - expect( - deriveWorkLogEntries([ - makeActivity({ - kind: "relay.binding", - summary: "Relay binding", - payload: { id: "job-1", environmentId: "env-1", threadId: "thread-1" }, - }), - makeActivity({ - kind: "relay.activation", - summary: "Relay activation", - payload: { id: "job-1", attempt: 2, toolCallId: "tool-2" }, - }), - ]), - ).toEqual([]); - }); it.each([ { outcome: "completed", @@ -2463,120 +2447,6 @@ describe("deriveWorkLogEntries quiet-timeline guarantee", () => { expect(entries[0]!.agentSpawn?.agentTaskIds).toEqual(["child-1", "child-2"]); }); - it("anchors recovered Relay cards to their exact dispatch instead of restart time", () => { - const oldTime = "2026-09-24T10:00:00.000Z"; - const newTime = "2026-09-27T10:00:00.000Z"; - const restartTime = "2026-09-27T11:00:00.000Z"; - const receipt = makeActivity({ - id: "original-dispatch", - kind: "tool.completed", - turnId: "old-turn", - createdAt: oldTime, - sequence: 1, - payload: { toolCallId: "dispatch", status: "completed", itemType: "mcp_tool_call" }, - }); - const start = makeActivity({ - id: "recovered-start", - kind: "task.started", - turnId: "old-turn", - createdAt: restartTime, - sequence: 100, - payload: { - taskId: "relay:job-old", - source: "relay", - toolUseId: "dispatch", - timelineBypass: true, - }, - }); - const recent = makeActivity({ - id: "recent-work", - kind: "tool.completed", - turnId: "new-turn", - createdAt: newTime, - sequence: 2, - payload: { toolCallId: "dispatch", status: "completed" }, - }); - // Deriving a paginated window first must not poison the cached row when - // its exact receipt later arrives. Reused IDs in another turn do not match. - expect(deriveWorkLogEntries([start, recent]).find((entry) => entry.agentSpawn)?.createdAt).toBe( - restartTime, - ); - const entries = deriveWorkLogEntries([ - receipt, - recent, - start, - makeActivity({ - kind: "task.completed", - turnId: "old-turn", - createdAt: restartTime, - sequence: 101, - payload: { - taskId: "relay:job-old", - source: "relay", - toolUseId: "dispatch", - status: "completed", - timelineBypass: true, - }, - }), - ]); - expect(entries.find((entry) => entry.agentSpawn)?.createdAt).toBe(oldTime); - expect(deriveTimelineEntries([], [], entries).map((entry) => entry.id)).toEqual([ - "original-dispatch", - "recovered-start", - "recent-work", - ]); - expect(start.createdAt).toBe(restartTime); - const native = { - ...start, - id: EventId.make("native-start"), - payload: { taskId: "native", agentKind: "agent", toolUseId: "dispatch" }, - }; - expect( - deriveWorkLogEntries([receipt, native]).find((entry) => entry.agentSpawn)?.createdAt, - ).toBe(restartTime); - }); - - it("groups Relay panel members with their coordinator in one Agents CTA", () => { - const panel = "relay-panel:panel-22222222-2222-4222-8222-222222222222"; - const entries = deriveWorkLogEntries([ - makeActivity({ - kind: "task.started", - payload: { - taskId: panel, - taskType: "local_workflow", - source: "relay", - timelineBypass: true, - }, - turnId: "turn-panel", - }), - makeActivity({ - kind: "task.started", - payload: { - taskId: `${panel}:member:0`, - taskType: "subagent", - source: "relay", - timelineBypass: true, - }, - turnId: "turn-panel", - }), - makeActivity({ - kind: "task.started", - payload: { - taskId: `${panel}:member:1`, - taskType: "subagent", - source: "relay", - timelineBypass: true, - }, - turnId: "turn-panel", - }), - ]); - expect(entries).toHaveLength(1); - expect(entries[0]!.agentSpawn).toEqual({ - workflowId: panel, - agentTaskIds: [panel, `${panel}:member:0`, `${panel}:member:1`], - }); - }); - it("timelineBypass non-agent rows (background shells) stay suppressed", () => { const entries = deriveWorkLogEntries([ makeActivity({ diff --git a/apps/web/src/session-logic.ts b/apps/web/src/session-logic.ts index 4228212f53..f113e1c48b 100644 --- a/apps/web/src/session-logic.ts +++ b/apps/web/src/session-logic.ts @@ -558,14 +558,6 @@ export function deriveWorkLogEntries( activities: ReadonlyArray, ): WorkLogEntry[] { const ordered = [...activities].toSorted(compareActivitiesByOrder); - const toolOrigins = new Map(); - for (const activity of ordered) { - if (activity.kind !== "tool.started" && activity.kind !== "tool.completed") continue; - const toolCallId = extractToolCallId(asRecord(activity.payload)); - if (!toolCallId) continue; - const key = `${activity.turnId ?? ""}:${toolCallId}`; - if (!toolOrigins.has(key)) toolOrigins.set(key, activity.createdAt); - } // A task's agent-vs-background identity is resolved from the whole thread, // not from each row: an orphaned shell task settled by a later process // reports only its id and status, and judging that bare row alone turned @@ -574,7 +566,6 @@ export function deriveWorkLogEntries( const entries: DerivedWorkLogEntry[] = []; for (const activity of foldUserInputActivities(ordered)) { // Ownership receipts feed Relay recovery and control routing, not the transcript. - if (activity.kind === "relay.binding" || activity.kind === "relay.activation") continue; if ( activity.kind === "interaction.requested" || activity.kind === "interaction.resolved" || @@ -614,16 +605,7 @@ export function deriveWorkLogEntries( if (isNoContentRuntimeWarning(activity)) continue; if (isPlanBoundaryToolActivity(activity)) continue; if (isAgentInternalActivity(activity, backgroundTaskIds)) continue; - const entry = toDerivedWorkLogEntry(activity, backgroundTaskIds); - const payload = asRecord(activity.payload); - const origin = - payload?.source === "relay" && typeof payload.toolUseId === "string" - ? toolOrigins.get(`${activity.turnId ?? ""}:${payload.toolUseId}`) - : undefined; - // Older servers backfilled Relay lifecycle rows at restart time. Keep - // their cards beside the exact dispatch, without rewriting saved history - // or moving native agents and unrelated invocations with reused IDs. - entries.push(origin && origin < entry.createdAt ? { ...entry, createdAt: origin } : entry); + entries.push(toDerivedWorkLogEntry(activity, backgroundTaskIds)); } return collapseDerivedWorkLogEntries(entries); } @@ -813,10 +795,6 @@ function toDerivedWorkLogEntry( */ function agentSpawnGroupKey(entry: DerivedWorkLogEntry): string { const taskId = entry.taskId ?? ""; - const relayMemberSlot = taskId.startsWith("relay-panel:") ? taskId.indexOf(":member:") : -1; - if (relayMemberSlot !== -1) { - return `wf:${taskId.slice(0, relayMemberSlot)}`; - } const workflowSlot = taskId.indexOf(":wf:"); if (workflowSlot !== -1) { return `wf:${taskId.slice(0, workflowSlot)}`; diff --git a/docs/user/thread-sidebar.md b/docs/user/thread-sidebar.md index b9f982f661..3d00781386 100644 --- a/docs/user/thread-sidebar.md +++ b/docs/user/thread-sidebar.md @@ -158,20 +158,6 @@ them together. ## Inspect agent work On web and desktop, use **Agents** to follow work delegated to subagents. -When a Pylon session delegates through Relay Orchestrator, its detached workers also appear here. -They keep their status after the parent turn ends or disconnects. You can stop an active Relay worker -from its agent row or the background Stop banner; Relay keeps the worker's partial work and evidence. -Relay workers do not accept direct messages or expose a live private transcript in Pylon. A panel groups dispatched members -and shows how many have not yet dispatched. - -Pylon automatically finds a user-scoped Relay Orchestrator installation in Claude Code's plugin -registry on the computer running the environment server (`CLAUDE_CONFIG_DIR`, or `~/.claude`). -Restart the Pylon server after installing the plugin. For other installation methods, project-only -installs, provider-specific Claude homes, or multiple user installations, set `PYLON_RELAY_CLI` to -the absolute path of Relay's `scripts/relay.mjs` before starting the server. This setting overrides -automatic discovery. Relay and Pylon must run under the same local account and use the same Relay -home. A remote Pylon client reads the server's agent status; it does not need local access to -Relay's worker files. Expand a tool group in the conversation to see its calls, and expand a call to see its full command and output. Summaries show the program inside a shell wrapper, such as "Running vp", and the live diff --git a/packages/client-runtime/src/state/sessionAgentLiveActivity.ts b/packages/client-runtime/src/state/sessionAgentLiveActivity.ts index bcaedfdeca..1d6e5f3f5f 100644 --- a/packages/client-runtime/src/state/sessionAgentLiveActivity.ts +++ b/packages/client-runtime/src/state/sessionAgentLiveActivity.ts @@ -52,7 +52,6 @@ export function sessionAgentLiveActivitySelectionIsOpen(input: { input.capabilityEnabled && input.agent !== null && input.agent !== undefined && - input.agent.source !== "relay" && input.agent.watchable !== false && input.agent.kind !== "workflow" && (input.agent.status === "pending" || diff --git a/packages/client-runtime/src/state/subagentRuntime.test.ts b/packages/client-runtime/src/state/subagentRuntime.test.ts index 072840ad24..581a093761 100644 --- a/packages/client-runtime/src/state/subagentRuntime.test.ts +++ b/packages/client-runtime/src/state/subagentRuntime.test.ts @@ -1127,288 +1127,6 @@ describe("supportsSessionAgentCancel", () => { }); }); -describe("Relay workers in the native agent fold", () => { - const messagingProvider = { - featureCapabilities: { - version: 1, - agents: { support: "read-write", operations: ["message"] }, - } as NonNullable, - }; - const relay = (taskId: string, attempt: number, relaySequence: number, other = {}) => ({ - taskId, - taskType: "local_agent", - source: "relay", - cancellable: true, - watchable: false, - attempt, - relaySequence, - ...other, - }); - - it("survives parent disconnect and routes control by worker capability", () => { - const [worker, native] = foldSubagentActivities( - [ - activity( - "task.progress", - relay("relay:job-1", 1, 1, { - status: "running", - model: "claude-sonnet-5", - effort: "high", - summary: "Reading files", - }), - ), - activity("task.started", { taskId: "native", taskType: "local_agent" }), - ], - { sessionLive: false }, - ); - expect(worker).toMatchObject({ - id: "relay:job-1", - source: "relay", - status: "running", - model: "claude-sonnet-5", - effort: "high", - progress: "Reading files", - cancellable: true, - watchable: false, - }); - expect(native?.status).toBe("interrupted"); - expect(canCancelSessionAgent(worker!, false, true)).toBe(true); - expect(canCancelSessionAgent(worker!, false, false)).toBe(false); - expect(canCancelSessionAgent(native!, true, true)).toBe(false); - expect(canMessageSessionAgent(messagingProvider, worker!)).toBe(false); - }); - - it("fences stale attempts and sequences while summing cumulative usage once per attempt", () => { - const [worker] = fold([ - activity("task.started", relay("relay:job-2", 1, 0, { title: "Review" })), - activity( - "task.progress", - relay("relay:job-2", 1, 1, { - status: "running", - typedUsage: { totalTokens: 100, inputTokens: 80 }, - }), - ), - activity( - "task.completed", - relay("relay:job-2", 1, 2, { - status: "failed", - summary: "First attempt failed", - typedUsage: { totalTokens: 150, inputTokens: 100 }, - }), - ), - activity("task.started", relay("relay:job-2", 2, 0)), - activity( - "task.progress", - relay("relay:job-2", 2, 2, { - status: "running", - summary: "Retrying", - relayPriorUsage: { totalTokens: 150, inputTokens: 100 }, - typedUsage: { totalTokens: 40, inputTokens: 25 }, - }), - ), - activity( - "task.completed", - relay("relay:job-2", 1, 3, { - status: "failed", - summary: "Stale failure", - typedUsage: { totalTokens: 500 }, - }), - ), - activity( - "task.progress", - relay("relay:job-2", 2, 1, { - status: "running", - summary: "Duplicate frame", - typedUsage: { totalTokens: 80 }, - }), - ), - activity( - "task.completed", - relay("relay:job-2", 2, 2, { - status: "completed", - summary: "Done", - typedUsage: { totalTokens: 60, inputTokens: 40 }, - }), - ), - ]); - expect(worker).toMatchObject({ - id: "relay:job-2", - activationCount: 2, - attempt: 2, - status: "completed", - result: "Done", - error: null, - usage: { totalTokens: 210, inputTokens: 140 }, - }); - expect(worker?.recentActivity.some((entry) => entry.summary === "Duplicate frame")).toBe(false); - }); - - it("retains cumulative usage when an earlier Relay completion aged out of the activity window", () => { - const [worker] = fold([ - activity( - "task.progress", - relay("relay:job-aged", 2, 1, { - status: "running", - relayPriorUsage: { totalTokens: 150, inputTokens: 100, outputTokens: 50 }, - typedUsage: { totalTokens: 40, inputTokens: 25, outputTokens: 15 }, - }), - ), - ]); - expect(worker).toMatchObject({ - id: "relay:job-aged", - attempt: 2, - status: "running", - usage: { totalTokens: 190, inputTokens: 125, outputTokens: 65 }, - }); - }); - - it("shows a resumed worker as idle while its observer is unavailable before the first start", () => { - const [worker] = fold([ - activity( - "task.completed", - relay("relay:job-resume-outage", 1, 2, { - status: "completed", - typedUsage: { totalTokens: 150 }, - }), - ), - activity( - "task.progress", - relay("relay:job-resume-outage", 2, 0, { - status: "idle", - cancellable: false, - relayPriorUsage: { totalTokens: 150 }, - summary: "Relay observer unavailable", - }), - ), - ]); - expect(worker).toMatchObject({ - attempt: 2, - status: "idle", - cancellable: false, - usage: { totalTokens: 150 }, - }); - }); - - it("keeps one Relay run when observation recovers within the same attempt", () => { - const [worker] = fold([ - activity("task.started", relay("relay:job-observer", 1, 0)), - activity("task.progress", relay("relay:job-observer", 1, 1, { status: "running" })), - activity( - "task.progress", - relay("relay:job-observer", 1, 1, { - status: "idle", - cancellable: false, - summary: "Relay observer unavailable", - }), - ), - activity( - "task.progress", - relay("relay:job-observer", 1, 2, { - status: "running", - cancellable: true, - summary: "Running command", - }), - ), - ]); - expect(worker).toMatchObject({ - status: "running", - attempt: 1, - activationCount: 1, - cancellable: true, - progress: "Running command", - }); - }); - - it("keeps partially dispatched panel workers independent of a failed coordinator", () => { - const agents = fold([ - activity("task.started", { - taskId: "relay-panel:panel-1", - taskType: "local_workflow", - source: "relay", - attempt: 1, - relaySequence: 0, - title: "Panel review", - }), - activity( - "task.progress", - relay("relay:job-3", 1, 0, { - parentAgentId: "relay-panel:panel-1", - agentIndex: 0, - status: "running", - }), - ), - activity("task.completed", { - taskId: "relay-panel:panel-1", - taskType: "local_workflow", - source: "relay", - attempt: 1, - relaySequence: 1, - status: "failed", - summary: "One dispatch failed", - }), - ]); - const model = deriveAgentPanelModel({ agents }); - expect(model.workflows).toHaveLength(1); - expect(model.workflows[0]?.unphasedMembers[0]?.status).toBe("running"); - expect(model.runningCount).toBe(1); - }); - - it("keeps one panel member row when a failed job is replaced in the same slot", () => { - const coordinator = "relay-panel:panel-2"; - const memberId = `${coordinator}:member:0`; - const agents = fold([ - activity("task.started", { - taskId: coordinator, - taskType: "local_workflow", - source: "relay", - attempt: 1, - relaySequence: 0, - }), - activity( - "task.progress", - relay(memberId, 1_000_001, 1, { - parentAgentId: coordinator, - agentIndex: 0, - status: "failed", - error: "First job failed", - typedUsage: { totalTokens: 100 }, - }), - ), - activity( - "task.progress", - relay(memberId, 2_000_001, 0, { - parentAgentId: coordinator, - agentIndex: 0, - status: "running", - summary: "Replacement job running", - typedUsage: { totalTokens: 20 }, - }), - ), - activity( - "task.completed", - relay(memberId, 1_000_001, 2, { - parentAgentId: coordinator, - agentIndex: 0, - status: "failed", - summary: "Late first result", - typedUsage: { totalTokens: 500 }, - }), - ), - ]); - const model = deriveAgentPanelModel({ agents }); - expect(model.workflows[0]?.unphasedMembers).toHaveLength(1); - expect(model.workflows[0]?.unphasedMembers[0]).toMatchObject({ - id: memberId, - attempt: 2_000_001, - status: "running", - error: null, - usage: { totalTokens: 120 }, - }); - expect(model.runningCount).toBe(1); - expect(canCancelSessionAgent(model.workflows[0]!.unphasedMembers[0]!, false, true)).toBe(true); - }); -}); - describe("session agent messaging", () => { const provider = (support: "read-only" | "read-write", operations: ReadonlyArray) => ({ featureCapabilities: { @@ -1504,18 +1222,55 @@ describe("task classification is resolved per task, not per row", () => { }); it("still admits a worker that never carries a task type", () => { - // Relay workers and native subagents legitimately have no taskType; the - // stamp is the only evidence, so it must keep deciding for them. - const relay = [ + // Native subagents legitimately have no taskType; the stamp is the only + // evidence, so it must keep deciding for them. + const typeless = [ activity("task.started", { - taskId: "relay:job-1", - title: "Relay worker", + taskId: "job-1", + title: "Nested worker", agentKind: "agent", }), - activity("task.completed", { taskId: "relay:job-1", status: "completed" }), + activity("task.completed", { taskId: "job-1", status: "completed" }), ]; - expect(collectBackgroundTaskIds(relay)).toEqual(new Set()); - expect(foldSubagentActivities(relay).map((agent) => agent.id)).toEqual(["relay:job-1"]); + expect(collectBackgroundTaskIds(typeless)).toEqual(new Set()); + expect(foldSubagentActivities(typeless).map((agent) => agent.id)).toEqual(["job-1"]); + }); + + it("folds legacy detached-worker payloads as ordinary agents", () => { + // Removing the detached-worker fields did not rewrite saved history, so + // rows persisted before that removal still carry them. Extra keys must be + // ignored, not cause the row to be dropped or mis-folded. + const legacy = [ + activity("task.started", { + taskId: "job-legacy", + taskType: "local_agent", + title: "Legacy worker", + agentKind: "agent", + source: "relay", + attempt: 2, + relaySequence: 7, + cancellable: true, + watchable: false, + }), + activity("task.completed", { + taskId: "job-legacy", + status: "completed", + summary: "Done", + agentKind: "agent", + source: "relay", + relaySequence: 8, + relayPriorUsage: { totalTokens: 100 }, + typedUsage: { totalTokens: 40 }, + }), + ]; + const [agent] = foldSubagentActivities(legacy); + expect(agent).toMatchObject({ + id: "job-legacy", + status: "completed", + result: "Done", + // Only the row's own usage counts; cumulative prior attempts are gone. + usage: { totalTokens: 40 }, + }); }); it("does not let an unstamped row demote a real subagent", () => { diff --git a/packages/client-runtime/src/state/subagentRuntime.ts b/packages/client-runtime/src/state/subagentRuntime.ts index bf1cdac4f0..26254ed688 100644 --- a/packages/client-runtime/src/state/subagentRuntime.ts +++ b/packages/client-runtime/src/state/subagentRuntime.ts @@ -59,8 +59,6 @@ export interface SubagentRunHandles { export interface RuntimeSubagent { readonly id: string; - /** Detached workers retain their identity and lifetime beyond the parent session. */ - readonly source?: "relay" | undefined; /** Explicit worker control support; native rows use provider capabilities. */ readonly cancellable?: boolean | undefined; /** Explicit per-agent live transcript capability. */ @@ -129,14 +127,11 @@ export function supportsSessionAgentMessage( /** Resolve cancellation for the specific agent, not just its parent provider. */ export function canCancelSessionAgent( - agent: Pick, + agent: Pick, nativeControlsAvailable: boolean, - detachedControlsAvailable: boolean, ): boolean { if (agent.kind === "workflow" || !isActiveSubagentStatus(agent.status)) return false; - return agent.source === "relay" - ? agent.cancellable === true && detachedControlsAvailable - : agent.cancellable !== false && nativeControlsAvailable; + return agent.cancellable !== false && nativeControlsAvailable; } /** @@ -147,10 +142,9 @@ export function canCancelSessionAgent( */ export function canMessageSessionAgent( provider: Pick | null | undefined, - agent: Pick, + agent: Pick, ): boolean { return ( - agent.source !== "relay" && supportsSessionAgentMessage(provider) && agent.kind !== "workflow" && agent.messageable && @@ -201,8 +195,8 @@ export function isBackgroundTaskActivity( * Deliberately one-directional and evidence-based: only a row that NAMES a * background task type makes the claim sticky. A merely unstamped row is the * absence of evidence, not background evidence, so a legacy or truncated row - * cannot demote a real agent. Tasks that never carry a type at all — Relay - * workers, native subagents — keep the per-row stamp. + * cannot demote a real agent. Tasks that never carry a type at all — native + * subagents — keep the per-row stamp. */ export function collectBackgroundTaskIds( activities: ReadonlyArray, @@ -331,37 +325,8 @@ function mergeUsageMax( return merged; } -function addUsage(first: SubagentUsage | null, second: SubagentUsage | null): SubagentUsage | null { - if (!first) return second; - if (!second) return first; - const sum: { - totalTokens: number; - inputTokens?: number; - cachedInputTokens?: number; - outputTokens?: number; - reasoningOutputTokens?: number; - toolUses?: number; - durationMs?: number; - } = { totalTokens: first.totalTokens + second.totalTokens }; - const fields = [ - "inputTokens", - "cachedInputTokens", - "outputTokens", - "reasoningOutputTokens", - "toolUses", - "durationMs", - ] as const; - for (const field of fields) { - const a = first[field]; - const b = second[field]; - if (a !== undefined || b !== undefined) sum[field] = (a ?? 0) + (b ?? 0); - } - return sum; -} - interface MutableAgent { id: string; - source: "relay" | undefined; cancellable: boolean | undefined; watchable: boolean | undefined; kind: RuntimeSubagent["kind"]; @@ -422,7 +387,6 @@ function getOrCreate( } const created: MutableAgent = { id, - source: payload.source === "relay" ? "relay" : undefined, cancellable: typeof payload.cancellable === "boolean" ? payload.cancellable : undefined, watchable: typeof payload.watchable === "boolean" ? payload.watchable : undefined, kind: kindFromPayload(payload, id), @@ -464,7 +428,6 @@ function getOrCreate( */ function fillMetadata(agent: MutableAgent, payload: Record): boolean { let attemptBumped = false; - if (payload.source === "relay") agent.source = "relay"; if (typeof payload.cancellable === "boolean") agent.cancellable = payload.cancellable; if (typeof payload.watchable === "boolean") agent.watchable = payload.watchable; if (payload.taskType === "subagent_batch") agent.kind = "subagent_batch"; @@ -553,10 +516,7 @@ function applyStatus( // Duplicate terminal events remain first-write-wins too. return; } - if ( - (wasTerminal || (agent.status === "idle" && agent.source !== "relay")) && - isActiveSubagentStatus(status) - ) { + if ((wasTerminal || agent.status === "idle") && isActiveSubagentStatus(status)) { // Reactivation: same identity, new run. Clear the previous run's terminal // detail so a live card never shows the prior run's output. agent.activationCount += 1; @@ -652,79 +612,9 @@ export function foldSubagentActivities( // MutableAgent and the public RuntimeSubagent are 1:1 (the fold returns // `{ ...agent }`), and no consumer reads this. const activationToolUseIds = new Map(); - const relaySequences = new Map(); - const relayUsage = new Map< - string, - { attempt: number; prior: SubagentUsage | null; current: SubagentUsage | null } - >(); - - const acceptRelayEvent = ( - taskId: string, - payload: Record, - kind: OrchestrationThreadActivity["kind"], - ): boolean => { - const existing = agents.get(taskId); - if (payload.source !== "relay" && existing?.source !== "relay") return true; - const attempt = asCount(payload.attempt); - if (attempt === undefined || !Number.isInteger(attempt)) return false; - if ( - existing?.attempt !== null && - existing?.attempt !== undefined && - attempt < existing.attempt - ) { - return false; - } - const eventSequence = asCount(payload.relaySequence); - if (eventSequence === undefined || !Number.isInteger(eventSequence)) return false; - if (attempt === existing?.attempt) { - const lastSequence = relaySequences.get(taskId); - if (lastSequence !== undefined && eventSequence < lastSequence) return false; - // A worker can settle without writing another output line. The final - // receipt therefore shares its output cursor with its last progress row. - // Once settled, only a matching completion may enrich result and usage. - if (existing && isTerminalSubagentStatus(existing.status) && kind !== "task.completed") { - return false; - } - } - if (attempt > (existing?.attempt ?? -1)) relaySequences.delete(taskId); - relaySequences.set(taskId, eventSequence); - return true; - }; const updateUsage = (agent: MutableAgent, payload: Record): void => { - const incoming = asUsage(payload.typedUsage); - if (agent.source !== "relay") { - agent.usage = mergeUsageMax(agent.usage, incoming); - return; - } - const priorSnapshot = asUsage(payload.relayPriorUsage); - if ((!incoming && !priorSnapshot) || agent.attempt === null) return; - let state = relayUsage.get(agent.id); - if (!state) { - state = { attempt: agent.attempt, prior: null, current: null }; - relayUsage.set(agent.id, state); - } else if (agent.attempt > state.attempt) { - state.prior = addUsage(state.prior, state.current); - state.current = null; - state.attempt = agent.attempt; - } - state.prior = mergeUsageMax(state.prior, priorSnapshot); - state.current = mergeUsageMax(state.current, incoming); - agent.usage = addUsage(state.prior, state.current); - }; - - const resetRelayAttempt = (agent: MutableAgent, attemptBumped: boolean): void => { - if (!attemptBumped || agent.source !== "relay") return; - agent.activationCount += 1; - agent.status = "pending"; - agent.startedAt = null; - agent.completedAt = null; - agent.result = null; - agent.error = null; - agent.progress = null; - agent.lastToolName = null; - agent.outputFile = null; - agent.recentActivity = []; + agent.usage = mergeUsageMax(agent.usage, asUsage(payload.typedUsage)); }; for (const activity of activities) { @@ -742,10 +632,8 @@ export function foldSubagentActivities( // tasks are background work — they render in the ordinary work log, // not the Agents surface (a "Run 12s stall" shell is not a subagent). if (isBackgroundTaskActivity(payload, backgroundTaskIds)) break; - if (!acceptRelayEvent(taskId, payload, activity.kind)) break; const agent = getOrCreate(agents, taskId, payload, at); const attemptBumped = fillMetadata(agent, payload); - resetRelayAttempt(agent, attemptBumped); // Order-robustness: a start row arriving after a terminal state only // reopens the identity when its toolUseId proves a new invocation. // Guard on the status itself, not activationCount: a task first seen @@ -774,7 +662,6 @@ export function foldSubagentActivities( } const detail = asString(payload.detail); if (detail && agent.title === agent.id) agent.title = detail; - if (agent.source === "relay") updateUsage(agent, payload); agent.updatedAt = at; break; } @@ -786,10 +673,8 @@ export function foldSubagentActivities( // first row's classification instead of being re-judged. const existed = agents.has(taskId); if (!existed && isBackgroundTaskActivity(payload, backgroundTaskIds)) break; - if (!acceptRelayEvent(taskId, payload, activity.kind)) break; const agent = getOrCreate(agents, taskId, payload, at); const attemptBumped = fillMetadata(agent, payload); - resetRelayAttempt(agent, attemptBumped); if (agent.activationCount === 0) agent.activationCount = 1; const explicitStatus = asRuntimeStatus(payload.status); if (explicitStatus) { @@ -840,10 +725,8 @@ export function foldSubagentActivities( // rows often carry only taskId+status, no marker fields) inherit the // first row's classification instead of being re-judged. if (!agents.has(taskId) && isBackgroundTaskActivity(payload, backgroundTaskIds)) break; - if (!acceptRelayEvent(taskId, payload, activity.kind)) break; const agent = getOrCreate(agents, taskId, payload, at); const attemptBumped = fillMetadata(agent, payload); - resetRelayAttempt(agent, attemptBumped); if (agent.kind === "subagent_batch" && asString(payload.status) === "idle") { // The parent turn ended, so the batch cannot report child progress. // Replace the stale running summary with the explanation rather than @@ -887,9 +770,8 @@ export function foldSubagentActivities( // rows often carry only taskId+status, no marker fields) inherit the // first row's classification instead of being re-judged. if (!agents.has(taskId) && isBackgroundTaskActivity(payload, backgroundTaskIds)) break; - if (!acceptRelayEvent(taskId, payload, activity.kind)) break; const agent = getOrCreate(agents, taskId, payload, at); - resetRelayAttempt(agent, fillMetadata(agent, payload)); + fillMetadata(agent, payload); const toolUseId = asString(payload.toolUseId); if (toolUseId) activationToolUseIds.set(taskId, toolUseId); if (agent.activationCount === 0) agent.activationCount = 1; @@ -931,7 +813,6 @@ export function foldSubagentActivities( if (!taskId) break; const agent = agents.get(taskId); if (!agent) break; - if (!acceptRelayEvent(taskId, payload, activity.kind)) break; const toolName = asString(payload.toolName); if (toolName) { agent.lastToolName = toolName; @@ -951,11 +832,7 @@ export function foldSubagentActivities( // don't read as working forever (live-test finding: statuses drifted // whenever member terminal rows were lost or never emitted). for (const agent of agents.values()) { - if ( - agent.kind !== "workflow" || - agent.source === "relay" || - !isTerminalSubagentStatus(agent.status) - ) { + if (agent.kind !== "workflow" || !isTerminalSubagentStatus(agent.status)) { continue; } for (const member of agents.values()) { @@ -976,7 +853,7 @@ export function foldSubagentActivities( // session.exited, so panel and sidebar can never disagree. if (options?.sessionLive === false) { for (const agent of agents.values()) { - if (agent.source !== "relay" && isActiveSubagentStatus(agent.status)) { + if (isActiveSubagentStatus(agent.status)) { agent.status = "interrupted"; agent.completedAt = agent.completedAt ?? agent.updatedAt; } diff --git a/packages/contracts/src/providerRuntime.ts b/packages/contracts/src/providerRuntime.ts index 1134583c7c..ede6229e29 100644 --- a/packages/contracts/src/providerRuntime.ts +++ b/packages/contracts/src/providerRuntime.ts @@ -731,16 +731,10 @@ const taskAgentLinkageFields = { outputFile: Schema.optional(TrimmedNonEmptyStringSchema), /** Whether this live agent currently has a provider-owned direct-message endpoint. */ messageable: Schema.optional(Schema.Boolean), - /** External worker source. Omitted for native provider agents. */ - source: Schema.optional(Schema.Literal("relay")), /** Whether the source exposes a working cancellation endpoint. */ cancellable: Schema.optional(Schema.Boolean), /** Whether the source exposes a live activity watch endpoint. */ watchable: Schema.optional(Schema.Boolean), - /** Monotonic observation sequence within a Relay worker attempt. */ - relaySequence: Schema.optional(NonNegativeInt), - /** Cumulative usage from earlier Relay attempts, repeated on the current row for bounded snapshots. */ - relayPriorUsage: Schema.optional(RuntimeTaskUsage), /** Codex agent hierarchy path, e.g. "/root/marlow". */ agentPath: Schema.optional(TrimmedNonEmptyStringSchema), /**