From 78bd4f16f0bb340fad3f4269d77c7d1fb981d6e0 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 19:35:59 +0000 Subject: [PATCH 01/58] feat(server): prototype a remote provider driver with a checkout-less hub and a runner Splits provider execution out of the T3 server behind the provider driver. The runner build (`t3 runner`) hosts the real Claude/Codex drivers, the checkpoint store and workspace validation, and serves an Effect RPC protocol with a durable, replayable event outbox. In hub mode (T3CODE_RUNNER_URL) every built-in driver is wrapped by a remote driver, and checkpoint, git-probe and workspace-root services are delegated to the runner, while repository identity, workspace index and VCS status are stubbed. ProviderService and the orchestration reactors run unchanged. Adds PROTOTYPE-2.md with evidence for turns, approvals, interrupts, checkpoints and diffs, hub and runner restarts, and a local-mode parity run. Co-Authored-By: Claude Opus 5.5 (1M context) --- PROTOTYPE-2.md | 435 ++++++++++++++++++ apps/server/src/bin.ts | 2 + apps/server/src/cli/config.ts | 10 + apps/server/src/cli/runner.ts | 21 + apps/server/src/config.ts | 8 + apps/server/src/git/CheckoutGitProbe.ts | 18 + .../orchestration/Layers/CheckpointReactor.ts | 9 +- .../Layers/ProviderRuntimeIngestion.ts | 5 +- .../ProviderInstanceRegistryHydration.ts | 50 +- apps/server/src/runner/RunnerOutbox.test.ts | 115 +++++ apps/server/src/runner/RunnerOutbox.ts | 240 ++++++++++ apps/server/src/runner/RunnerProtocol.ts | 317 +++++++++++++ apps/server/src/runner/RunnerServer.ts | 360 +++++++++++++++ apps/server/src/runner/hub/HubLayers.ts | 225 +++++++++ .../src/runner/hub/RemoteProviderDriver.ts | 314 +++++++++++++ apps/server/src/runner/hub/RunnerClient.ts | 313 +++++++++++++ apps/server/src/runner/hub/RunnerDelivery.ts | 50 ++ apps/server/src/server.ts | 61 ++- prototype-2/ctl.sh | 31 ++ prototype-2/evidence/control-interrupt.out | 42 ++ prototype-2/evidence/control-turn1.out | 49 ++ prototype-2/evidence/diff-0-1.out | 7 + prototype-2/evidence/hub-delivery-log.txt | 350 ++++++++++++++ prototype-2/evidence/hub-restart-test.out | 11 + .../evidence/hub-strace-all-lifetimes.txt | 32 ++ prototype-2/evidence/outbox-summary.txt | 14 + .../evidence/runner-restart-midturn-test.out | 17 + prototype-2/evidence/runner-restart-test.out | 16 + prototype-2/evidence/runner-rpc-log.txt | 258 +++++++++++ prototype-2/evidence/turn1.out | 45 ++ prototype-2/evidence/turn10-continue.out | 110 +++++ prototype-2/evidence/turn11-title.out | 33 ++ prototype-2/evidence/turn6-interrupt.out | 72 +++ prototype-2/evidence/turn7-hub-restart.out | 124 +++++ .../evidence/turn8-after-runner-restart.out | 85 ++++ .../evidence/turn9-runner-restart-midturn.out | 92 ++++ prototype-2/hub-restart-test.sh | 18 + prototype-2/outbox-tail.mjs | 23 + prototype-2/runner-restart-midturn-test.sh | 18 + prototype-2/scenario.mjs | 335 ++++++++++++++ prototype-2/start-control.sh | 7 + prototype-2/start-hub.sh | 14 + prototype-2/start-runner.sh | 8 + 43 files changed, 4331 insertions(+), 33 deletions(-) create mode 100644 PROTOTYPE-2.md create mode 100644 apps/server/src/cli/runner.ts create mode 100644 apps/server/src/git/CheckoutGitProbe.ts create mode 100644 apps/server/src/runner/RunnerOutbox.test.ts create mode 100644 apps/server/src/runner/RunnerOutbox.ts create mode 100644 apps/server/src/runner/RunnerProtocol.ts create mode 100644 apps/server/src/runner/RunnerServer.ts create mode 100644 apps/server/src/runner/hub/HubLayers.ts create mode 100644 apps/server/src/runner/hub/RemoteProviderDriver.ts create mode 100644 apps/server/src/runner/hub/RunnerClient.ts create mode 100644 apps/server/src/runner/hub/RunnerDelivery.ts create mode 100755 prototype-2/ctl.sh create mode 100644 prototype-2/evidence/control-interrupt.out create mode 100644 prototype-2/evidence/control-turn1.out create mode 100644 prototype-2/evidence/diff-0-1.out create mode 100644 prototype-2/evidence/hub-delivery-log.txt create mode 100644 prototype-2/evidence/hub-restart-test.out create mode 100644 prototype-2/evidence/hub-strace-all-lifetimes.txt create mode 100644 prototype-2/evidence/outbox-summary.txt create mode 100644 prototype-2/evidence/runner-restart-midturn-test.out create mode 100644 prototype-2/evidence/runner-restart-test.out create mode 100644 prototype-2/evidence/runner-rpc-log.txt create mode 100644 prototype-2/evidence/turn1.out create mode 100644 prototype-2/evidence/turn10-continue.out create mode 100644 prototype-2/evidence/turn11-title.out create mode 100644 prototype-2/evidence/turn6-interrupt.out create mode 100644 prototype-2/evidence/turn7-hub-restart.out create mode 100644 prototype-2/evidence/turn8-after-runner-restart.out create mode 100644 prototype-2/evidence/turn9-runner-restart-midturn.out create mode 100755 prototype-2/hub-restart-test.sh create mode 100644 prototype-2/outbox-tail.mjs create mode 100755 prototype-2/runner-restart-midturn-test.sh create mode 100644 prototype-2/scenario.mjs create mode 100755 prototype-2/start-control.sh create mode 100755 prototype-2/start-hub.sh create mode 100755 prototype-2/start-runner.sh diff --git a/PROTOTYPE-2.md b/PROTOTYPE-2.md new file mode 100644 index 000000000000..2feb9ff87eb1 --- /dev/null +++ b/PROTOTYPE-2.md @@ -0,0 +1,435 @@ +# Prototype 2: remote provider driver (hub + runner) + +Question from `docs/cloud-agents-architecture.md` (remote-dev): does a remote provider +driver preserve T3 behavior? + +**Answer: yes for the turn lifecycle.** A checkout-less hub and a runner in separate +processes, with separate state directories, completed real Claude turns. The runs covered +streamed output, approvals, interrupts, checkpoints and diffs, a hub restart during a turn, +and runner restarts between and during turns. No events were lost or duplicated. For the +same prompts, a local-mode control run of the same build produced the same projections +and the same diff bytes. + +The cut works at the driver level, as proposed. `ProviderService` (the session directory, +resume cursors and `recoverSessionForThread`) and every orchestration reactor ran +unchanged in the hub, except that two `.git` checks now go through a service. The work +still needed is outside the turn loop: per-thread routing, caches that keep machines +asleep, crash reconciliation, and about 60 other call sites that act on the checkout +(terminals, files, VCS, attachments, MCP and so on). These are listed below. + +Branch `prototype/remote-runner` of the T3 fork. It is local only and not pushed. The +runs used Node 22.23 with sources run directly (`node src/bin.ts`), and the model +`claude-haiku-4-5` through the unmodified Claude Code 2.1.282 CLI and T3's +`ClaudeAdapter`, using this VM's existing login. Model usage: 12 short turns and 1 title +generation. + +## What was built + +``` +client ──HTTP/WS──▶ HUB :4411 (T3 server, T3CODE_RUNNER_URL set) + orchestration · projections · settings · ProviderService/session directory + RemoteProviderDriver ×7 kinds · RemoteCheckpointStore · CheckoutGitProbe + RunnerClient ──── WebSocket, Effect RPC (RunnerRpcGroup) ────┐ + ▼ + RUNNER :4412 (`t3 runner`) + real ClaudeDriver (+CodexDriver) · CheckpointStore/git + WorkspacePaths · RunnerOutbox (NDJSON, fsync) + cwd = /tmp/proto-runner/checkout +``` + +| File | Lines | Role | +| ---------------------------------------------------------------------------------------------- | ---------: | ------------------------------------------------------------------------------------------------------------------------------------ | +| `apps/server/src/runner/RunnerProtocol.ts` | 317 | Hub↔runner RPC group. Payloads are `@t3tools/contracts` schemas; errors are the adapter/VCS error classes | +| `apps/server/src/runner/RunnerServer.ts` | 360 | Runner build: registry of real drivers, event pump into the outbox, RPC handlers, token-checked WS route, and a shutdown drain | +| `apps/server/src/runner/RunnerOutbox.ts` (+ `.test.ts`) | 240 (+115) | Durable, replayable event log | +| `apps/server/src/runner/hub/RunnerClient.ts` | 313 | Supervised connection with reconnect, bounded wait for calls, cursor-based subscription, dedup, fan-out, cursor file | +| `apps/server/src/runner/hub/RemoteProviderDriver.ts` | 314 | Wraps each built-in driver's kind, schema and defaults. The adapter, snapshot and title/branch text generation forward to the runner | +| `apps/server/src/runner/hub/HubLayers.ts` | 225 | Hub replacements for services that touch the checkout (delegated or stubbed; table below) | +| `apps/server/src/runner/hub/RunnerDelivery.ts` | 50 | Starts delivery after activation. Every 500 ms it drains ingestion, then acknowledges | +| `apps/server/src/git/CheckoutGitProbe.ts` | 18 | `Context.Reference` that replaces the synchronous `isGitRepository` in two reactors. The default keeps the local behavior | +| edits: `server.ts`, `ProviderInstanceRegistryHydration.ts`, `config.ts`, `cli/*`, two reactors | +163/−33 | `byRunnerMode(local, hub)` layer switches; driver-list hydration; `T3CODE_RUNNER_URL`/`T3CODE_RUNNER_TOKEN`; `t3 runner` subcommand | +| `prototype-2/` | | Scenario driver (client over the hub's public API), launch and stop scripts, evidence | + +Checks: `pnpm typecheck` (apps/server) is clean. `vp lint` and `vp fmt --check` pass on +the changed files. `vp test run` passes for `RunnerOutbox.test.ts` (2 tests) and for the +touched suites `CheckpointReactor.test.ts`, `ProviderRuntimeIngestion.test.ts` and +`ProviderInstanceRegistryHydration.test.ts` (68 tests). Repo-wide suites were not run, per +AGENTS.md. + +### Running it + +```bash +# layout: /tmp/proto-runner/{checkout,hub,runner,logs}, settings.json with +# enableProviderUpdateChecks=false (no manifest/npm fetches), telemetry off. +cp prototype-2/{start-hub.sh,start-runner.sh,ctl.sh} /tmp/proto-runner/ +/tmp/proto-runner/ctl.sh start-runner # node src/bin.ts runner --base-dir /tmp/proto-runner/runner --port 4412 +/tmp/proto-runner/ctl.sh start-hub # T3CODE_RUNNER_URL=ws://127.0.0.1:4412/runner/ws node src/bin.ts --base-dir /tmp/proto-runner/hub --port 4411 +node apps/server/src/bin.ts auth session issue --base-dir /tmp/proto-runner/hub --token-only > /tmp/proto-runner/hub-token +node prototype-2/scenario.mjs setup +node prototype-2/scenario.mjs turn "" --approve [--interrupt-after-ms 4000] +node prototype-2/scenario.mjs diff 0 1 +prototype-2/hub-restart-test.sh ; prototype-2/runner-restart-midturn-test.sh +``` + +**How we show the hub never touches the checkout.** `start-hub.sh` runs the hub inside +`bwrap --dev-bind / / --tmpfs /tmp/proto-runner/checkout --chmod 0000 …`. In the hub's +mount namespace the checkout path is an empty directory with mode 000, so any stat, read, +`chdir` or git call on it fails with EACCES. The hub also runs under +`strace -f --seccomp-bpf -e trace=%file,%process,chdir,fchdir -P /tmp/proto-runner/checkout`, +which records every syscall, in the hub or any child process, that names that path. +**Result: across 4 hub lifetimes, 0 such syscalls.** The only lines recorded are 4 +SIGTERMs from `ctl.sh` and 28 SIGCHLDs from short-lived child processes +(`prototype-2/evidence/hub-strace-all-lifetimes.txt`). Everything the hub knows about the +checkout came from the runner. + +## Evidence + +Timestamps are from one VM clock. The raw output is in `prototype-2/evidence/`. + +### a) Turn runs on the runner and streams into hub projections + +`project.create` for `workspaceRoot=/tmp/proto-runner/checkout` was accepted in 89 ms. The +hub cannot stat that path; the runner validated it through +`runner.workspace.normalizeRoot`. `thread.create` took 21 ms. Turn 1, "append a line to +notes.txt", used runtime mode `approval-required`: + +``` +19:17:17.733 thread.turn.start accepted in 40 ms +19:17:21.057 activity tool.started / 21.468 tool.completed (first tool call) +19:17:22.695 activity tool.started: File change started +19:17:23.513 activity approval.requested: File-change approval requested +19:17:23.532 approval dispatched in 19 ms +19:17:23.941 activity approval.resolved / tool.completed: File change +19:17:24.782 message assistant: "done" +19:17:25.194 activity checkpoint.captured → turn completed after 7460 ms +checkpoints: turn 1 ready, notes.txt +1/-0 +``` + +On the runner, `claude` ran with `--add-dir /tmp/proto-runner/checkout --session-id 202a75b8…` +and `notes.txt` gained the line. Streaming: the "count to 1500" turn moved 117 +`content.delta` events through the outbox in 3.1 s, and the "count to 300" turn moved 151 +in 4.0 s. Hub ingestion buffered them into the message, as T3 does by default +(`enableLegacyTokenStreaming=false`). + +### b) Approval round trip + +Approvals are produced on the runner as `request.opened`, projected by the hub as +`approval.requested`, answered by the client with `thread.approval.respond`, and routed by +`ProviderCommandReactor` → `ProviderService` → the remote adapter → the runner's +`respondToRequest`. From the client's dispatch (19:27:40.447) to the runner handling +`respondToRequest` (19:27:40.470) took **23 ms**. In the control run the dispatch alone took +13 ms, against 17–46 ms remote. Case e1 below answered an approval that was created while +the hub was down. + +### c) Interrupt a running turn + +"Count from 1 to 1500" was interrupted 4 s in (`turn6-interrupt.out`): + +``` +client 19:24:27.260 thread.turn.interrupt +runner 19:24:27.277 rpc interruptTurn ok (2 ms) ← 17 ms end to end +outbox #365 item.completed · #367 turn.completed {state: interrupted, "Session stopped."} · #368 session.exited +hub turn settled 19:24:27.685, partial assistant text kept, session "stopped" +``` + +The local control run had the identical outcome: latestTurn `completed`, session +`stopped`, 2× `checkpoint.captured`. The Claude adapter implements interrupt as stop +session, so this is T3 behavior, not something the split introduced. + +### d) Checkpoint captured on the runner; diff served through the hub + +Refs `refs/t3/checkpoints//turn/{0,1,…}` exist only in the runner's checkout. +Runner-side `checkpoint.capture` took 42–74 ms. `orchestration.getTurnDiff` over the hub's +`/ws` returned the real patch in 121 ms, including the ws-ticket request and the socket +connect (control: 91 ms): + +``` +diff --git a/notes.txt b/notes.txt +index f31bc06..77957df 100644 +@@ -1 +1,2 @@ + hello from the runner ++edited by the remote runner +``` + +This is byte-identical to the control run, blob hashes included. After later turns the +per-turn summaries were still computed through the runner, for example turn 8 +`notes.txt +1/-0` after the hub restart. + +### e1) Hub restarted mid-turn; approval produced while the hub was down + +`prototype-2/hub-restart-test.sh`, prompt "sleep 20 in the foreground, then append 'after +hub restart'": + +``` +19:24:56.968 command running on runner → stop hub (SIGTERM, 107 ms). Outbox head 379, acked 379. + runner keeps working; events 380..395 appended with no hub attached +19:25:20.148 #395 request.opened file_change_approval ← approval requested while hub down +19:25:20.590 hub start (bwrap+strace, sources via type stripping) +19:25:27.725 hub: "runner event subscription opened { afterSequence: 379 }" +19:25:28.306 client sees approval.requested → approves (46 ms) → runner respondToRequest 19:25:28.372 +19:25:30.482 turn completed; notes.txt has 'after hub restart'; checkpoint turn 8 notes.txt +1/-0 +hub: throughSequence 405, deliveredCount 26, duplicateCount 0 +``` + +On restart the hub did not mark the session orphaned. `reconcileProviderSessions` asked +the remote adapters' `listSessions`, the runner reported the live session, and the session +was adopted. In the local architecture a hub restart kills the turn; here the turn +survived. The remote `stopAll` is a no-op, so hub shutdown does not stop sessions that +another machine hosts. + +### e2) Runner restarted between turns; resumed through the persisted `resumeCursor` + +``` +19:25:51.248 stop runner (107 ms) → hub: "runner connection ended" (retries every 1 s) +19:25:56.639 runner listening: outbox loaded head 405 / acked 405, same runnerId, new bootId +19:25:57.311 hub reconnected (0.67 s) → subscription afterSequence 405 +turn "which lines did you add to notes.txt earlier?": + runner rpc startSession {resumeCursor: {resume: '202a75b8…', resumeSessionAt: '3c54100f…'}} ok 110 ms + runner rpc sendTurn ok 1045 ms + assistant: "edited by the remote runner\nafter hub restart" ← memory from before both restarts +``` + +The runner no longer listed the session, so `hasSession` returned false and +`ProviderService.recoverSessionForThread` restarted it from the hub's +`provider_session_runtime.resume_cursor_json`. No recovery code was added. The first +restart of both processes, at 19:20, also resumed the same way. + +### e3) Runner restarted mid-turn + +`prototype-2/runner-restart-midturn-test.sh`. On graceful stop the runner stops its +sessions while the event pumps are still alive, so the exit events are persisted before +the process exits: + +``` +19:27:29.383 command running → stop runner (412 ms incl. drain) + outbox #432 item.completed(command, failed) #434 turn.completed {interrupted} #435 session.exited +19:27:31.860 runner restarted → hub reconnected 19:27:34.44 → turn settled in hub at 19:27:34.66 +follow-up turn "finish the cut-off task": startSession(resumeCursor) → approval → notes.txt 'after runner restart' ✓ +``` + +A runner crash (SIGKILL) would lose these exit events; see the gaps below. + +### Title generation through the runner + +For a second thread titled "New thread", `ProviderCommandReactor` called the remote +instance's `generateThreadTitle`. The runner ran it (`runner rpc text.generateThreadTitle` +took 14.2 s, spawning a fresh Claude CLI) and the hub renamed the thread to "Capital of +France". This also showed two threads sharing one runner and the session directory. + +### Totals + +| Measure | Value | +| ------------------------------------- | -------------------------------------------------------------------------------------------------- | +| Runner events delivered to the hub | 478 (sequences 1–478), **0 duplicates**, 0 schema rejects, 0 runner warnings | +| Process lifetimes | 6 runner boots, 4 hub boots, 0 lost events (every head equaled the acked sequence after reconnect) | +| Outbox cost | ~1.05 KB per event (NDJSON), one `fsync` per append; compacted to empty once acked | +| Turn 1, dispatch → settled | 7.46 s remote vs 7.89 s local control (model variance dominates) | +| Added hub hops | approval 23 ms and interrupt 17 ms client→runner; `getTurnDiff` +30 ms vs local | +| Runner reconnect after restart | 0.67 s (1 s retry loop) | +| Hub boot to replayed approval visible | 7.7 s (Node type stripping + strace; not tuned) | + +## Runner protocol (`RunnerRpcGroup`, protocol version 1) + +Transport: one WebSocket to `/runner/ws`, JSON serialization, runner token in the `token` +query parameter (prototype only). Each provider call names `instanceId`; each checkpoint +call names `cwd`. Provider errors carry the original tags +(`ProviderAdapter{Validation,SessionNotFound,SessionClosed,Request,Process}Error`), and +checkpoint errors are the contracts `VcsError` union. On the hub, transport failures +(`RunnerUnavailableError`, `RpcClientError`) become `ProviderAdapterRequestError` or +`VcsRepositoryDetectionError`, so upstream error handling is unchanged. + +| RPC | Payload → success | +| ---------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| `runner.hello` | {} → `{protocolVersion, runnerId, bootId, headSequence, ackedSequence, instances}` | +| `runner.provider.startSession` | `{instanceId, input: ProviderSessionStartInput}` → `ProviderSession` | +| `runner.provider.sendTurn` | `{instanceId, input: ProviderSendTurnInput}` → `ProviderTurnStartResult` | +| `runner.provider.interruptTurn` / `stopSession` | `{instanceId, threadId, turnId?}` → void | +| `runner.provider.respondToRequest` / `respondToUserInput` | `{instanceId, threadId, requestId, decision \| answers}` → void | +| `runner.provider.listSessions` | `{instanceId}` → `ProviderSession[]` (answers `hasSession` too) | +| `runner.provider.readThread` / `rollbackThread` | `{instanceId, threadId, numTurns?}` → `{threadId, turns[]}` | +| `runner.provider.getCapabilities` | `{instanceId, refresh?}` → `{snapshot: ServerProvider, sessionModelSwitch}` | +| `runner.text.generateThreadTitle` / `generateBranchName` | title/branch input + `instanceId` → `{title}` / `{branch}` | +| `runner.events.subscribe` (stream) | `{afterSequence}` → stream of `{sequence, bootId, event: ProviderRuntimeEvent}` | +| `runner.events.ack` | `{throughSequence}` → `{ackedSequence, retained}` | +| `runner.checkpoint.isGitRepository` / `capture` / `hasRef` / `restore` / `diff` / `deleteRefs` | `CheckpointStore` inputs → bool / void / bool / bool / patch / void | +| `runner.workspace.normalizeRoot` | `{workspaceRoot, createIfMissing?}` → normalized path \| `RunnerWorkspaceError` | + +The hub keeps `ProviderService` unchanged. It still persists resume cursors and cwd, +mints MCP credentials, holds the per-thread lifecycle locks, and runs recovery. The runner +stamps `providerInstanceId` on every event before appending it, so the hub fans events +out by instance. + +## Event outbox + +**Prototype** (`RunnerOutbox.ts`): + +- Every adapter's `streamEvents` is piped into `append`. Each event is schema-encoded as + one NDJSON line with `sequence = head + 1` and the current `bootId`, written with + `writeSync`, `fsync`ed, and only then published to live subscribers. An event that fails + the wire schema is counted and dropped, so it cannot poison every subscription. None + were dropped in this run. +- `subscribe(after)` subscribes to the live PubSub first, then replays retained entries + with `sequence > after`, then tails live entries past the replay point. There is no gap + and no overlap. +- `ack(n)` persists `outbox-ack.json`, drops acknowledged entries, and rewrites the file + when it is empty or when 500 acknowledged lines have accumulated. It is atomic + (tmp + rename, fd reopened). +- On load, unparseable (torn) lines are discarded, and `head = max(file, acked)`, so + sequences keep increasing across restarts. `runner-id` is stable; `bootId` changes on + every start. +- Hub (`RunnerClient` + `RunnerDelivery`): it subscribes from its persisted cursor + (`runner-client-cursor.json`, reset if `runnerId` changes) and drops any + `sequence <= delivered`. Every 500 ms it drains `ProviderRuntimeIngestion`, then writes + the cursor and acks. Delivery waits for server activation plus 500 ms, so ingestion is + subscribed before replay starts. +- Guarantee: at-least-once, in order, per runner. After a hub crash, events ingested + inside the last ack window (≤ 500 ms) could be ingested again, because ingestion command + ids are random. + +**Production design:** + +1. Runner outbox in the machine's SQLite, in the checkout's state disk that survives + pause and recreation: `(sequence PK, boot_id, event_json, created_at)`, with an ack + watermark and size- and time-based retention. If an event past retention is still + unacknowledged, the hub must resynchronize from `readThread`. +2. The hub stores `runner_cursor(thread_machine_id, acked_sequence)` in the same Postgres + transaction as the orchestration events that ingestion produced. Ingestion also dedups + on `(machine, sequence)`, so replay is exactly-once in effect, and ack becomes a + notification rather than a correctness step. +3. The hub consumes through the control-plane tunnel, one subscription per awake thread + machine. Sequence and `bootId` gaps are explicit signals (see crash reconciliation + below). +4. Backpressure: the runner keeps writing while the hub is away, and the outbox is the + buffer. The hub bounds the replay batch and paces delivery against the ingestion + queue depth. + +## Hub-side call sites that touch the checkout, and how each was handled + +A read-only survey by a subagent found about 60 call sites. The hot path is marked 🔥. +This table covers the ones that matter for the split. + +| Call site | What it does | Handling in this prototype | +| ----------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| 🔥 `CheckpointReactor` `resolveCheckpointCwd` and the revert check (was sync `isGitRepository`) | `.git` existence | **Delegated** through `CheckoutGitProbe` → `runner.checkpoint.isGitRepository` (same `.git` rule) | +| 🔥 `ProviderRuntimeIngestion` `turn.diff.updated` placeholder (sync `isGitRepository`) | `.git` existence | **Delegated** (same probe) | +| 🔥 `CheckpointStore` capture/hasRef/restore/diff/deleteRefs (CheckpointReactor, CheckpointDiffQuery) | git plumbing in cwd | **Delegated** (`RemoteCheckpointStore`, cwd-keyed) | +| 🔥 `Normalizer` `project.create`/`meta.update` → `WorkspacePaths.normalizeWorkspaceRoot` | stat / mkdir -p | **Delegated** (`runner.workspace.normalizeRoot`); relative-path resolution stays pure on the hub | +| 🔥 `ProviderService` → adapters (`startSession` cwd, turns, approvals, interrupt, rollback, readThread) | spawn provider CLI in cwd | **Delegated** (remote driver) | +| 🔥 `ProviderCommandReactor` first-turn title / branch-name generation | provider CLI in cwd | **Delegated** (remote `textGeneration`); commit/PR/structured generation **fail** with "not delegated" | +| 🔥 `ProjectionSnapshotQuery` via `RepositoryIdentityResolver` (every shell snapshot, project lookups, reactors, watchdog seed) | `git rev-parse` + `git remote -v` | **Stubbed** (identity `null`). Production: persist identity on the project, reported by the runner | +| 🔥 `CheckpointReactor` → `WorkspaceEntries.refresh` after capture | stat + file-index rescan | **Stubbed** (no-op); search/list/browse return empty | +| 🔥 `CheckpointReactor.refreshLocalGitStatusFromTurnCompletion`, CommandDispatcher bootstrap, `vcs*` RPCs → `VcsStatusBroadcaster` | realpath + git status/fetch | **Stubbed** (fails with "not delegated"; logged once per turn). Branch-drift following is off. The sidebar has no git status | +| `ProviderCommandReactor.ensureThreadWorktree` (turn start when the thread has a worktree) | `fs.exists(worktreePath)`, `git worktree prune/add` | **Still broken** (not exercised; local-mode threads have no worktree). It would stat the hub's disk and then run git locally | +| `ProviderCommandReactor` worktree branch rename (`git branch -m`) + `refreshStatus` | git in worktree | **Still broken** (worktree mode only) | +| `CommandDispatcher` bootstrap: `git fetch`, `worktree add`, setup script → `ProjectSetupScriptRunner` → terminal PTY | git + PTY in the checkout | **Still broken** (the prototype uses `/api/orchestration/dispatch`, which bypasses bootstrap) | +| `ThreadSettlementReactor` sweep → `GitManager.branchPullRequest({cwd: workspaceRoot})` (every minute, threads with branches) | git remote/for-each-ref + gh | **Still broken** (not exercised; the test threads have `branch: null`) | +| `ws.ts`: file search/read/write, `filesystemBrowse`, terminals, VCS/git/PR actions, review diff preview, open in editor, source-control clone/publish | FS, PTY, git, gh in cwd | **Still local**, so they fail on a hub. Need routed runner RPCs | +| `http.ts` asset routes (project favicon, workspace/media files), `ProjectFaviconResolver`, `T3ProjectFileLoader` (`t3.json`) | stat/read files in the checkout | **Still local** (UI only). Proxy to the runner or persist on the hub | +| `workflowScriptQuery` (reads `~/.claude/projects/...`) | provider home on the machine | **Still local**; belongs on the runner | +| Server cwd fallbacks (`?? process.cwd()` for title cwd, OpenCode `directory: serverConfig.cwd`, probes) | provider work in the server cwd | Harmless on the hub because nothing spawns there. Runner probes use the runner cwd | + +## What breaks or is missing + +- **Terminals.** `TerminalManager` still runs on the hub. The RPCs are already keyed by + `threadId`, so they need a routed streaming runner service (PTY data, resize, restart, + port scanning for previews). +- **Workspace file RPCs.** Search, read and write, `@`-mention indexing (the hub stub + returns empty lists), and filesystem browse for the project picker. Browse must target + the machine that will own the project. +- **VCS status.** Stubbed, so the sidebar has no git status, branch-drift following is + off, and there is no PR and branch lookup. Production needs the runner to push status + into a hub cache (the design's "caches that keep machines asleep"), and routed RPCs for + pull, commit, push and create PR. +- **Attachments.** The hub writes uploads to its own `attachmentsDir`, but the runner's + adapter reads `/`. A turn with an image fails or loses the + image. Uploads need to be copied to the runner with each turn and the paths rewritten. + The `ChatAttachment` ids already carry through `sendTurn`. +- **MCP endpoint.** `ProviderService.prepareMcpSession` mints the credential in the hub + and stores it in the hub process's `McpProviderSession` global. The runner's + `ClaudeAdapter` reads its own global and finds nothing, so agents get no `t3-code` MCP + tools (browser, review). The credential and the hub's public MCP URL have to travel in + `startSession`, which means extending `ProviderSessionStartInput` or adding an MCP field + to the protocol. +- **Text generation.** Title and branch generation are delegated and work (14 s, same + path as local). Commit message, PR content and `generateStructured` fail; the schema is + not serializable, so the runner needs to own those callers (git actions) instead. +- **Auth connector and provider sign-in.** `authConnector` still runs CLI logins where the + hub is. It must run on the runner, or be replaced by the credential service. + Provider maintenance (update CLI) and usage scanning are runner or host concerns too. +- **Provider settings sync.** The runner reads its own `settings.json`. The hub's provider + instance config (binary path, env, custom instances, enabled flags) is not pushed; the + hub's remote instance only mirrors id, kind, display name and enabled. The runner hosts + only `claudeAgent` and `codex` (`T3CODE_RUNNER_DRIVERS`), and the other kinds show as + unavailable. The snapshot and model list are fetched once at hub boot and on `refresh`, + with no push from the runner. `continuationIdentity` uses the default key rather than + Claude's HOME-based group key. `sessionModelSwitch` is hard-coded to `in-session` on the + hub. `uploadFeedback` is not forwarded. +- **Runner crash (SIGKILL) mid-turn.** No exit events are written, so the hub turn stays + `running` until `TurnLivenessWatchdog` fires (10 min silence). Fix: when a reconnect + shows a new `bootId`, the hub diffs its active bindings against the runner's + `listSessions` and synthesizes `session.exited` and an interrupted `turn.completed` for + the missing sessions. The graceful path is covered by the shutdown drain (e3). +- **`hasSession`/`listSessions` go to the runner on every routed call**, and startup + reconcile calls every runner. With sleeping machines this wakes them. The hub should + answer from the session directory plus runner-pushed lifecycle events, and treat + "machine asleep" as "session resumable", as the design says. +- **Hub shutdown semantics.** `ProviderService`'s finalizer still marks every binding + `stopped` on hub exit, even though the runner keeps the session. It is harmless here + because routing asks the runner, but the directory is wrong until the next event. +- **Routing.** There is one runner URL per hub. Adapter calls are keyed by `threadId` and + route easily. `listSessions`, `stopAll` and the cwd-keyed `CheckpointStore` need to + become thread-keyed, or resolve thread by cwd, to pick a machine. +- **Security.** A static shared token sits in the query string. Production uses the + control-plane execution token over the machine's outbound tunnel. The hub never dials + machines directly. + +Provider behavior seen in these runs; the split did not cause it, and the control run +matches: + +- Claude interrupt stops the session, and T3 projects an interrupted turn as `completed`. +- The user's Claude settings (`--setting-sources=user,…`) auto-allow `Bash`, so only file + edits asked for approval. +- The model backgrounded a `sleep 45`. After resume, Claude reported the stopped + background task as a separate short turn. + +## Productionization plan (rough size) + +| Step | Work | Size (files / lines) | +| ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------- | +| 1 | Move `RunnerRpcGroup` into contracts with versioning. Hub and runner become separate compositions (`hub.ts`, `runner.ts`) instead of `byRunnerMode` switches, and host layers are dropped from the hub build | 6 / ~700 | +| 2 | Per-thread routing: a `ThreadMachineDirectory` (thread → machine, connection pool, wake through the control plane); thread-keyed `CheckpointStore`/`CheckpointDiffQuery` API | 8 / ~900 | +| 3 | Durable delivery: SQLite outbox on the runner, cursor stored in the same transaction as projection writes plus `(machine, sequence)` dedup, `bootId` crash reconciliation | 6 / ~800 + migration | +| 4 | Session state on the hub: `hasSession`/`listSessions` from the directory plus pushed lifecycle, "asleep = resumable", remote-aware `stopAll`/finalizer | 4 / ~350 | +| 5 | Hub caches: persisted repository identity, pushed VCS status, per-turn diffs in `checkpoint_diff_blobs`, so history and diffs never wake a machine | 8 / ~900 + migrations | +| 6 | Routed runner services for `ws.ts`: terminals (streaming), workspace files and search, VCS/git/PR/review, assets and favicon, worktree/bootstrap/setup scripts | 15 / ~2,500 | +| 7 | Attachments shipped with turns; MCP credential and URL in `startSession`; provider settings push and credential-service login; auth connector on the runner | 8 / ~1,000 | +| 8 | Contract tests that run the existing adapter/reactor suites through a loopback runner, plus a hub and runner e2e in CI | 6 / ~800 | + +That totals about 60 files and ~8,000 lines. Steps 1–4 are the milestone-1 core, about +2,700 lines. Steps 5–7 are needed before the product loses its always-on machine. + +**Risks:** + +- **Fork divergence.** `server.ts` composition, `ProviderService` session semantics, the + reactors and `ws.ts` routing are hot upstream files, so every upstream merge touches + them. Keep the seams narrow: driver, `CheckpointStore`, probes, routed services. +- **Resume fidelity.** Resume depends on the provider's own session files on the machine + (`~/.claude/projects//.jsonl`) and on the checkout living at an + identical absolute path. Recreate from snapshot must preserve both, and prototype 3 + should verify it. +- **Wake latency.** Every uncached hub read or `hasSession` call becomes a machine wake. + Missing a cache turns a fast UI into a spinner. +- **Exactly-once ingestion.** This needs the cursor and projection write in one + transaction. Without it, crash windows duplicate deltas and activities. +- **Security surface.** The runner executes whatever the hub sends. Its authentication + must be per-machine, scoped and short-lived. +- **Provider-specific behavior.** Background tasks and synthetic turns after resume show + up more often when machines restart freely. The UI must handle them gracefully. + +## Cleanup + +Every process started for this prototype has been stopped (hub, runner, local control). +`/tmp/proto-runner` holds the throwaway checkouts, state directories and logs. Claude +Code wrote its normal session files under +`~/.claude/projects/-tmp-proto-runner-{checkout,control-checkout}` (about 0.5 MB). No +credential file was read, copied or modified by this work. diff --git a/apps/server/src/bin.ts b/apps/server/src/bin.ts index 0a2e4091560b..101df9f3dc0c 100644 --- a/apps/server/src/bin.ts +++ b/apps/server/src/bin.ts @@ -16,6 +16,7 @@ import { sharedServerCommandFlags } from "./cli/config.ts"; import { isEntrypoint } from "./entrypoint.ts"; import { projectCommand } from "./cli/project.ts"; import { runServerCommand, serveCommand, startCommand } from "./cli/server.ts"; +import { runnerCommand } from "./cli/runner.ts"; import { serviceCommand } from "./cli/service.ts"; import { servicePreflightCommand } from "./cli/servicePreflight.ts"; import { themeCommand } from "./cli/theme.ts"; @@ -62,6 +63,7 @@ export const makeCli = ({ cloudEnabled = hasCloudPublicConfig } = {}) => servicePreflightCommand, themeCommand, triageCommand, + runnerCommand, cloudEnabled ? connectCommand : connectUnavailableCommand, ]), ); diff --git a/apps/server/src/cli/config.ts b/apps/server/src/cli/config.ts index 32bf5916b9f9..8b82924d3247 100644 --- a/apps/server/src/cli/config.ts +++ b/apps/server/src/cli/config.ts @@ -131,6 +131,14 @@ const EnvServerConfig = Config.all({ Config.option, Config.map(Option.getOrUndefined), ), + runnerUrl: Config.string("T3CODE_RUNNER_URL").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), + runnerToken: Config.string("T3CODE_RUNNER_TOKEN").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), bootstrapFd: Config.int("T3CODE_BOOTSTRAP_FD").pipe( Config.option, Config.map(Option.getOrUndefined), @@ -391,6 +399,8 @@ export const resolveServerConfig = ( museCodeEnabled: env.museCodeEnabled ?? !env.managedDevPc, primeAgentSubscriptionOAuthEnabled: env.primeAgentSubscriptionOAuthEnabled, managedGatewayToken: env.managedGatewayToken, + runnerUrl: env.runnerUrl, + runnerToken: env.runnerToken, startupPresentation, desktopBootstrapToken, desktopTelemetryFd, diff --git a/apps/server/src/cli/runner.ts b/apps/server/src/cli/runner.ts new file mode 100644 index 000000000000..e4b8ab496542 --- /dev/null +++ b/apps/server/src/cli/runner.ts @@ -0,0 +1,21 @@ +import * as Effect from "effect/Effect"; +import { Command, GlobalFlag } from "effect/unstable/cli"; + +import { ServerConfig } from "../config.ts"; +import { runRunner } from "../runner/RunnerServer.ts"; +import { resolveServerConfig, sharedServerCommandFlags } from "./config.ts"; + +/** Prototype: serve the runner protocol for this machine's checkout. */ +export const runnerCommand = Command.make("runner", { ...sharedServerCommandFlags }).pipe( + Command.withDescription("Run a T3 runner: provider drivers and checkpoints for a remote hub."), + Command.withHandler((flags) => + Effect.gen(function* () { + const logLevel = yield* GlobalFlag.LogLevel; + const config = yield* resolveServerConfig(flags, logLevel, { + startupPresentation: "headless", + forceAutoBootstrapProjectFromCwd: false, + }); + return yield* runRunner.pipe(Effect.provideService(ServerConfig, config)); + }), + ), +); diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index e89150ea2f17..ca7231182e05 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -98,6 +98,14 @@ export class ServerConfig extends Context.Service< * managed automation routes that are never exposed by standalone T3 Code. */ readonly managedGatewayToken?: string | undefined; + /** + * Hub mode (prototype): when set, this server owns no checkout. Provider + * sessions, checkpoints and workspace validation run on the runner at this + * WebSocket URL (see `runner/`). + */ + readonly runnerUrl?: string | undefined; + /** Shared bearer secret presented to the runner (hub) or required from hubs (runner). */ + readonly runnerToken?: string | undefined; readonly startupPresentation: StartupPresentation; readonly desktopBootstrapToken: string | undefined; readonly desktopTelemetryFd?: number | undefined; diff --git a/apps/server/src/git/CheckoutGitProbe.ts b/apps/server/src/git/CheckoutGitProbe.ts new file mode 100644 index 000000000000..fc77301f6c66 --- /dev/null +++ b/apps/server/src/git/CheckoutGitProbe.ts @@ -0,0 +1,18 @@ +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; + +import { isGitRepository } from "./Utils.ts"; + +export const localCheckoutGitProbe = (cwd: string): Effect.Effect => + Effect.sync(() => isGitRepository(cwd)); + +/** + * Answers "is this checkout a git repository?" for orchestration reactors. + * + * Defaults to the local `.git` existence check. A hub that owns no checkout + * overrides it with a runner-backed probe so reactors never stat the path. + */ +export class CheckoutGitProbe extends Context.Reference<(cwd: string) => Effect.Effect>( + "t3/git/CheckoutGitProbe", + { defaultValue: () => localCheckoutGitProbe }, +) {} diff --git a/apps/server/src/orchestration/Layers/CheckpointReactor.ts b/apps/server/src/orchestration/Layers/CheckpointReactor.ts index dd9d397200e7..0eb932643ef2 100644 --- a/apps/server/src/orchestration/Layers/CheckpointReactor.ts +++ b/apps/server/src/orchestration/Layers/CheckpointReactor.ts @@ -35,7 +35,7 @@ import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts" import { RuntimeReceiptBus } from "../Services/RuntimeReceiptBus.ts"; import type { CheckpointStoreError } from "../../checkpointing/Errors.ts"; import type { OrchestrationDispatchError } from "../Errors.ts"; -import { isGitRepository } from "../../git/Utils.ts"; +import { CheckoutGitProbe } from "../../git/CheckoutGitProbe.ts"; import { VcsStatusBroadcaster } from "../../vcs/VcsStatusBroadcaster.ts"; import * as WorkspaceEntries from "../../workspace/WorkspaceEntries.ts"; @@ -88,6 +88,7 @@ const make = Effect.gen(function* () { const receiptBus = yield* RuntimeReceiptBus; const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; const vcsStatusBroadcaster = yield* VcsStatusBroadcaster; + const isGitWorkspace = yield* CheckoutGitProbe; const appendRevertFailureActivity = (input: { readonly threadId: ThreadId; @@ -177,8 +178,6 @@ const make = Effect.gen(function* () { return project ? [project] : []; }); - const isGitWorkspace = (cwd: string) => isGitRepository(cwd); - // Resolves the workspace CWD for checkpoint operations, preferring the // active provider session CWD and falling back to the thread/project config. // Returns undefined when no CWD can be determined or the workspace is not @@ -209,7 +208,7 @@ const make = Effect.gen(function* () { if (!cwd) { return undefined; } - if (!isGitWorkspace(cwd)) { + if (!(yield* isGitWorkspace(cwd))) { return undefined; } return cwd; @@ -713,7 +712,7 @@ const make = Effect.gen(function* () { }).pipe(Effect.catch(() => Effect.void)); return; } - if (!isGitWorkspace(sessionRuntime.value.cwd)) { + if (!(yield* isGitWorkspace(sessionRuntime.value.cwd))) { yield* appendRevertFailureActivity({ threadId: event.payload.threadId, turnCount: event.payload.turnCount, diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index a90010f0b6e2..424a2f55f2b9 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -32,7 +32,7 @@ import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; import { ProviderService } from "../../provider/Services/ProviderService.ts"; import { ProjectionTurnRepository } from "../../persistence/Services/ProjectionTurns.ts"; import { ProjectionTurnRepositoryLive } from "../../persistence/Layers/ProjectionTurns.ts"; -import { isGitRepository } from "../../git/Utils.ts"; +import { CheckoutGitProbe } from "../../git/CheckoutGitProbe.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; import { ThreadBackgroundLivenessService } from "../ThreadBackgroundLiveness.ts"; import { ThreadPlanProgressService } from "../ThreadPlanProgress.ts"; @@ -897,6 +897,7 @@ const make = Effect.gen(function* () { const providerService = yield* ProviderService; const projectionTurnRepository = yield* ProjectionTurnRepository; const serverSettingsService = yield* ServerSettingsService; + const isGitRepository = yield* CheckoutGitProbe; const providerCommandId = (event: ProviderRuntimeEvent, tag: string) => crypto.randomUUIDv4.pipe( Effect.map((uuid) => CommandId.make(`provider:${event.eventId}:${tag}:${uuid}`)), @@ -1948,7 +1949,7 @@ const make = Effect.gen(function* () { : undefined; const workspaceCwd = checkpointContext?.worktreePath ?? checkpointContext?.workspaceRoot ?? undefined; - if (turnId && checkpointContext && workspaceCwd && isGitRepository(workspaceCwd)) { + if (turnId && checkpointContext && workspaceCwd && (yield* isGitRepository(workspaceCwd))) { // Skip if a checkpoint already exists for this turn. A real // (non-placeholder) capture from CheckpointReactor should not // be clobbered, and dispatching a duplicate placeholder for the diff --git a/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts b/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts index d960af5c66dc..89e5c40ecedf 100644 --- a/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts +++ b/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts @@ -62,6 +62,7 @@ import type { AnyProviderDriver } from "../ProviderDriver.ts"; import { ProviderInstanceRegistry } from "../Services/ProviderInstanceRegistry.ts"; import { ProviderInstanceRegistryMutator } from "../Services/ProviderInstanceRegistryMutator.ts"; import { ProviderInstanceRegistryMutableLayer } from "./ProviderInstanceRegistryLive.ts"; +import { makeRemoteProviderDriver } from "../../runner/hub/RemoteProviderDriver.ts"; /** * Synthesize a `ProviderInstanceConfigMap` from a `ServerSettings` snapshot. @@ -157,31 +158,50 @@ const makeSettingsWatcherLive = (drivers: ReadonlyArray( + drivers: ReadonlyArray>, +): Layer.Layer => + Layer.unwrap( + Effect.gen(function* () { + const serverSettings = yield* ServerSettingsService; + const initialSettings: ServerSettings | undefined = yield* serverSettings.getSettings.pipe( + Effect.orElseSucceed(() => undefined), + ); + // The legacy `providers.` mirror only depends on `driverKind`, so + // any driver list (built-in, remote, or a runner subset) derives the same map. + const configDrivers = drivers as unknown as ReadonlyArray< + AnyProviderDriver + >; + const initialConfigMap = + initialSettings === undefined + ? ({} as ProviderInstanceConfigMap) + : deriveProviderInstanceConfigMap(initialSettings, configDrivers); + + const mutableLayer = ProviderInstanceRegistryMutableLayer({ + drivers, + configMap: initialConfigMap, + }); + + return makeSettingsWatcherLive(configDrivers).pipe(Layer.provideMerge(mutableLayer)); + }), + ) as Layer.Layer; + export const ProviderInstanceRegistryHydrationLive: Layer.Layer< ProviderInstanceRegistry, never, BuiltInDriversEnv | ServerConfig | ServerSettingsService > = Layer.unwrap( Effect.gen(function* () { - const serverSettings = yield* ServerSettingsService; const serverConfig = yield* ServerConfig; const drivers = resolveBuiltInDrivers({ museCodeEnabled: serverConfig.museCodeEnabled, }); - const initialSettings: ServerSettings | undefined = yield* serverSettings.getSettings.pipe( - Effect.orElseSucceed(() => undefined), - ); - const initialConfigMap = - initialSettings === undefined - ? ({} as ProviderInstanceConfigMap) - : deriveProviderInstanceConfigMap(initialSettings, drivers); - - const mutableLayer = ProviderInstanceRegistryMutableLayer({ - drivers, - configMap: initialConfigMap, - }); - - return makeSettingsWatcherLive(drivers).pipe(Layer.provideMerge(mutableLayer)); + // Hub mode: every provider runs on the thread's runner. The hub keeps the + // same driver kinds and instance ids, but each adapter forwards over the + // runner protocol instead of spawning a provider CLI locally. + return serverConfig.runnerUrl + ? makeProviderInstanceRegistryHydration(drivers.map(makeRemoteProviderDriver)) + : makeProviderInstanceRegistryHydration(drivers); }), ) as Layer.Layer< ProviderInstanceRegistry, diff --git a/apps/server/src/runner/RunnerOutbox.test.ts b/apps/server/src/runner/RunnerOutbox.test.ts new file mode 100644 index 000000000000..7d2f2496e953 --- /dev/null +++ b/apps/server/src/runner/RunnerOutbox.test.ts @@ -0,0 +1,115 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { it } from "@effect/vitest"; +import { + EventId, + ProviderDriverKind, + type ProviderRuntimeEvent, + ThreadId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import { describe, expect } from "vite-plus/test"; + +import * as ServerConfig from "../config.ts"; +import { RunnerOutbox, layer as RunnerOutboxLayer } from "./RunnerOutbox.ts"; + +const event = (n: number): ProviderRuntimeEvent => ({ + type: "session.state.changed", + eventId: EventId.make(`evt-${n}`), + provider: ProviderDriverKind.make("claudeAgent"), + threadId: ThreadId.make("thread-outbox-test"), + createdAt: "2026-09-25T00:00:00.000Z", + payload: { state: "ready" }, +}); + +/** One runner process lifetime against a persistent state directory. */ +const boot = ( + baseDir: string, + body: (outbox: RunnerOutbox["Service"]) => Effect.Effect, +) => + Effect.gen(function* () { + const outbox = yield* RunnerOutbox; + return yield* body(outbox); + }).pipe( + Effect.provide( + RunnerOutboxLayer.pipe( + Layer.provide(ServerConfig.ServerConfig.layerTest(process.cwd(), baseDir)), + Layer.provide(NodeServices.layer), + ), + ), + ); + +const sequencesOf = (stream: Stream.Stream<{ readonly sequence: number }>, count: number) => + stream.pipe( + Stream.take(count), + Stream.runCollect, + Effect.map((entries) => Array.from(entries, (entry) => entry.sequence)), + ); + +describe("RunnerOutbox", () => { + it.effect("replays only unacknowledged events across restarts, in order", () => + Effect.gen(function* () { + const baseDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "runner-outbox-test-")); + + const firstBoot = yield* boot(baseDir, (outbox) => + Effect.gen(function* () { + yield* outbox.append(event(1)); + yield* outbox.append(event(2)); + yield* outbox.append(event(3)); + const replayed = yield* sequencesOf(outbox.subscribe(0), 3); + const acked = yield* outbox.ack(2); + return { replayed, acked, stats: yield* outbox.stats }; + }), + ); + expect(firstBoot.replayed).toEqual([1, 2, 3]); + expect(firstBoot.acked).toEqual({ ackedSequence: 2, retained: 1 }); + + // A crash can leave a torn final line; it must not block the next boot. + const outboxPath = NodePath.join(baseDir, "userdata", "runner", "outbox.ndjson"); + NodeFS.appendFileSync(outboxPath, '{"sequence":4,"bootId":"torn","ev'); + + const secondBoot = yield* boot(baseDir, (outbox) => + Effect.gen(function* () { + const before = yield* outbox.stats; + yield* outbox.append(event(4)); + const fromStart = yield* sequencesOf(outbox.subscribe(0), 2); + const fromCursor = yield* sequencesOf(outbox.subscribe(3), 1); + return { before, fromStart, fromCursor, after: yield* outbox.stats }; + }), + ); + expect(secondBoot.before.runnerId).toBe(firstBoot.stats.runnerId); + expect(secondBoot.before.bootId).not.toBe(firstBoot.stats.bootId); + expect(secondBoot.before).toMatchObject({ headSequence: 3, ackedSequence: 2, retained: 1 }); + expect(secondBoot.fromStart).toEqual([3, 4]); + expect(secondBoot.fromCursor).toEqual([4]); + expect(secondBoot.after.headSequence).toBe(4); + + NodeFS.rmSync(baseDir, { recursive: true, force: true }); + }), + ); + + it.effect("delivers events appended after subscribing exactly once", () => + Effect.gen(function* () { + const baseDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "runner-outbox-test-")); + const sequences = yield* boot(baseDir, (outbox) => + Effect.gen(function* () { + yield* outbox.append(event(1)); + const fiber = yield* sequencesOf(outbox.subscribe(0), 3).pipe(Effect.forkChild); + yield* Effect.yieldNow; + yield* outbox.append(event(2)); + yield* outbox.append(event(3)); + return yield* Fiber.join(fiber); + }), + ); + expect(sequences).toEqual([1, 2, 3]); + NodeFS.rmSync(baseDir, { recursive: true, force: true }); + }), + ); +}); diff --git a/apps/server/src/runner/RunnerOutbox.ts b/apps/server/src/runner/RunnerOutbox.ts new file mode 100644 index 000000000000..dd128769fc43 --- /dev/null +++ b/apps/server/src/runner/RunnerOutbox.ts @@ -0,0 +1,240 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Append order is the durability contract; synchronous appends keep it trivially ordered. +/** + * RunnerOutbox - durable, replayable provider runtime event log (prototype). + * + * Provider adapters publish runtime events to an in-memory PubSub. A hub on + * another machine can disconnect, restart, or lag, so the runner writes each + * event to an append-only NDJSON file with a monotonically increasing + * `sequence` before it is offered to subscribers. The hub subscribes from its + * last acknowledged sequence and acknowledges after ingestion; acknowledged + * events are compacted away. + * + * Files under `/runner/`: + * - `runner-id` stable runner identity (survives restarts) + * - `outbox.ndjson` one encoded `RunnerEventEnvelope` per line + * - `outbox-ack.json` `{ "ackedSequence": n }` + * + * Guarantees: at-least-once delivery of every event appended before a crash + * (a torn final line is discarded on load), strictly increasing sequences + * across restarts, and in-order replay. The hub deduplicates by sequence. + * + * @module runner/RunnerOutbox + */ +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + +import type { ProviderRuntimeEvent } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +import { ServerConfig } from "../config.ts"; +import { RunnerEventEnvelope } from "./RunnerProtocol.ts"; + +const EnvelopeLine = Schema.fromJsonString(RunnerEventEnvelope); +const encodeEnvelopeLine = Schema.encodeSync(EnvelopeLine); +const decodeEnvelopeLine = Schema.decodeUnknownSync(EnvelopeLine); +const AckFile = Schema.fromJsonString(Schema.Struct({ ackedSequence: Schema.Number })); +const encodeAckFile = Schema.encodeSync(AckFile); +const decodeAckFile = Schema.decodeUnknownSync(AckFile); + +/** Rewrite the file once this many acknowledged lines have accumulated. */ +const COMPACT_AFTER_ACKED_LINES = 500; + +export interface RunnerOutboxStats { + readonly runnerId: string; + readonly bootId: string; + readonly headSequence: number; + readonly ackedSequence: number; + readonly retained: number; + readonly rejected: number; +} + +export class RunnerOutbox extends Context.Service< + RunnerOutbox, + { + readonly append: (event: ProviderRuntimeEvent) => Effect.Effect; + readonly subscribe: (afterSequence: number) => Stream.Stream; + readonly ack: ( + throughSequence: number, + ) => Effect.Effect<{ readonly ackedSequence: number; readonly retained: number }>; + readonly stats: Effect.Effect; + } +>()("t3/runner/RunnerOutbox") {} + +interface LoadedOutbox { + readonly entries: Array; + readonly headSequence: number; + readonly ackedSequence: number; + readonly discardedLines: number; +} + +function readAckedSequence(ackPath: string): number { + try { + return decodeAckFile(NodeFS.readFileSync(ackPath, "utf8")).ackedSequence; + } catch { + return 0; + } +} + +function loadOutbox(outboxPath: string, ackPath: string): LoadedOutbox { + const ackedSequence = readAckedSequence(ackPath); + const entries: Array = []; + let headSequence = ackedSequence; + let discardedLines = 0; + let raw = ""; + try { + raw = NodeFS.readFileSync(outboxPath, "utf8"); + } catch { + raw = ""; + } + for (const line of raw.split("\n")) { + if (line.trim().length === 0) continue; + try { + const envelope = decodeEnvelopeLine(line); + headSequence = Math.max(headSequence, envelope.sequence); + if (envelope.sequence > ackedSequence) entries.push(envelope); + } catch { + // A crash can tear the final line; anything unparseable is dropped. + discardedLines += 1; + } + } + entries.sort((left, right) => left.sequence - right.sequence); + return { entries, headSequence, ackedSequence, discardedLines }; +} + +function readOrCreateRunnerId(idPath: string): string { + try { + const existing = NodeFS.readFileSync(idPath, "utf8").trim(); + if (existing.length > 0) return existing; + } catch { + // fall through + } + const created = `runner-${NodeCrypto.randomUUID()}`; + NodeFS.writeFileSync(idPath, `${created}\n`, { mode: 0o600 }); + return created; +} + +export const make = Effect.gen(function* () { + const config = yield* ServerConfig; + const dir = NodePath.join(config.stateDir, "runner"); + NodeFS.mkdirSync(dir, { recursive: true }); + const outboxPath = NodePath.join(dir, "outbox.ndjson"); + const ackPath = NodePath.join(dir, "outbox-ack.json"); + const runnerId = readOrCreateRunnerId(NodePath.join(dir, "runner-id")); + const bootId = `boot-${NodeCrypto.randomUUID()}`; + const fsyncEnabled = process.env.T3CODE_RUNNER_OUTBOX_FSYNC !== "0"; + // Evidence/debug switch: keep acknowledged lines instead of compacting. + const keepAcked = process.env.T3CODE_RUNNER_OUTBOX_KEEP === "1"; + + const loaded = loadOutbox(outboxPath, ackPath); + const entries = loaded.entries; + let headSequence = loaded.headSequence; + let ackedSequence = loaded.ackedSequence; + let ackedLinesSinceCompaction = 0; + let rejected = 0; + yield* Effect.logInfo("runner outbox loaded", { + runnerId, + bootId, + headSequence, + ackedSequence, + retained: entries.length, + discardedLines: loaded.discardedLines, + }); + + let fd = NodeFS.openSync(outboxPath, "a"); + yield* Effect.addFinalizer(() => Effect.sync(() => NodeFS.closeSync(fd))); + + const live = yield* PubSub.unbounded(); + + const compact = () => { + const tmpPath = `${outboxPath}.tmp`; + const body = entries.map((entry) => `${encodeEnvelopeLine(entry)}\n`).join(""); + NodeFS.writeFileSync(tmpPath, body); + NodeFS.closeSync(fd); + NodeFS.renameSync(tmpPath, outboxPath); + fd = NodeFS.openSync(outboxPath, "a"); + ackedLinesSinceCompaction = 0; + }; + + const append = (event: ProviderRuntimeEvent) => + Effect.suspend(() => { + const envelope: RunnerEventEnvelope = { sequence: headSequence + 1, bootId, event }; + let line: string; + try { + line = `${encodeEnvelopeLine(envelope)}\n`; + } catch (cause) { + // An event that does not satisfy the wire schema could never be + // delivered; count it instead of poisoning every subscription. + rejected += 1; + return Effect.logWarning("runner outbox rejected an event that fails the wire schema", { + eventType: event.type, + eventId: event.eventId, + cause: String(cause), + }); + } + headSequence = envelope.sequence; + NodeFS.writeSync(fd, line); + if (fsyncEnabled) NodeFS.fsyncSync(fd); + entries.push(envelope); + return PubSub.publish(live, envelope).pipe(Effect.asVoid); + }); + + const subscribe = (afterSequence: number) => + Stream.unwrap( + Effect.gen(function* () { + // Subscribe before snapshotting the backlog so no append can fall + // between replay and the live tail; overlap is filtered by sequence. + const subscription = yield* PubSub.subscribe(live); + const backlog = entries.filter((entry) => entry.sequence > afterSequence); + const replayedThrough = backlog.at(-1)?.sequence ?? afterSequence; + return Stream.concat( + Stream.fromIterable(backlog), + Stream.fromSubscription(subscription).pipe( + Stream.filter((entry) => entry.sequence > replayedThrough), + ), + ); + }), + ); + + const ack = (throughSequence: number) => + Effect.sync(() => { + const next = Math.min(Math.max(ackedSequence, throughSequence), headSequence); + if (next > ackedSequence) { + ackedSequence = next; + let dropped = 0; + while (entries.length > 0 && entries[0]!.sequence <= ackedSequence) { + entries.shift(); + dropped += 1; + } + ackedLinesSinceCompaction += dropped; + NodeFS.writeFileSync(ackPath, encodeAckFile({ ackedSequence })); + if ( + !keepAcked && + (ackedLinesSinceCompaction >= COMPACT_AFTER_ACKED_LINES || entries.length === 0) + ) { + compact(); + } + } + return { ackedSequence, retained: entries.length }; + }); + + const stats = Effect.sync( + (): RunnerOutboxStats => ({ + runnerId, + bootId, + headSequence, + ackedSequence, + retained: entries.length, + rejected, + }), + ); + + return RunnerOutbox.of({ append, subscribe, ack, stats }); +}); + +export const layer = Layer.effect(RunnerOutbox, make); diff --git a/apps/server/src/runner/RunnerProtocol.ts b/apps/server/src/runner/RunnerProtocol.ts new file mode 100644 index 000000000000..77f0e053a170 --- /dev/null +++ b/apps/server/src/runner/RunnerProtocol.ts @@ -0,0 +1,317 @@ +/** + * RunnerProtocol - the hub <-> runner wire contract (prototype). + * + * The hub is a T3 server with no local checkout: it owns orchestration, + * projections, settings and the provider session directory. A runner owns + * one machine's checkout and runs the real provider drivers, git and + * checkpoints for it. The hub reaches the runner through this Effect RPC + * group over one authenticated WebSocket. + * + * Every payload reuses a `@t3tools/contracts` schema where one exists, so + * the runner speaks exactly the shapes the in-process adapter already uses. + * + * Event delivery is the only stateful part: runners append every provider + * runtime event to a durable outbox with a monotonically increasing + * `sequence`; the hub subscribes from its last acknowledged sequence and + * acknowledges once the events are ingested. See `RunnerOutbox.ts`. + * + * @module runner/RunnerProtocol + */ +import { + ApprovalRequestId, + ChatAttachment, + CheckpointRef, + ModelSelection, + NonNegativeInt, + ProviderApprovalDecision, + ProviderInstanceId, + ProviderRuntimeEvent, + ProviderSendTurnInput, + ProviderSession, + ProviderSessionStartInput, + ProviderTurnStartResult, + ProviderUserInputAnswers, + ServerProvider, + TextGenerationError, + ThreadId, + TurnId, + VcsError, +} from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import * as Rpc from "effect/unstable/rpc/Rpc"; +import * as RpcGroup from "effect/unstable/rpc/RpcGroup"; + +import { + ProviderAdapterProcessError, + ProviderAdapterRequestError, + ProviderAdapterSessionClosedError, + ProviderAdapterSessionNotFoundError, + ProviderAdapterValidationError, +} from "../provider/Errors.ts"; + +export const RUNNER_PROTOCOL_VERSION = 1; +export const RUNNER_WS_PATH = "/runner/ws"; + +/** Adapter errors cross the wire with their original tags. */ +export const RunnerProviderError = Schema.Union([ + ProviderAdapterValidationError, + ProviderAdapterSessionNotFoundError, + ProviderAdapterSessionClosedError, + ProviderAdapterRequestError, + ProviderAdapterProcessError, +]); +export type RunnerProviderError = typeof RunnerProviderError.Type; + +export class RunnerWorkspaceError extends Schema.TaggedErrorClass()( + "RunnerWorkspaceError", + { + operation: Schema.String, + workspaceRoot: Schema.String, + reason: Schema.Literals(["not-exists", "not-directory", "create-failed", "stat-failed"]), + detail: Schema.String, + }, +) { + override get message(): string { + return `Runner workspace ${this.operation} failed for ${this.workspaceRoot}: ${this.detail}`; + } +} + +export const RunnerHello = Schema.Struct({ + protocolVersion: Schema.Number, + /** Stable for the runner's state directory; survives restarts. */ + runnerId: Schema.String, + /** Changes on every runner process start. */ + bootId: Schema.String, + /** Highest sequence ever appended to the outbox (0 when empty). */ + headSequence: NonNegativeInt, + /** Highest sequence the hub has acknowledged. */ + ackedSequence: NonNegativeInt, + instances: Schema.Array(ProviderInstanceId), +}); +export type RunnerHello = typeof RunnerHello.Type; + +export const RunnerEventEnvelope = Schema.Struct({ + sequence: NonNegativeInt, + bootId: Schema.String, + event: ProviderRuntimeEvent, +}); +export type RunnerEventEnvelope = typeof RunnerEventEnvelope.Type; + +const InstanceScoped = { instanceId: ProviderInstanceId } as const; + +export const RunnerThreadSnapshot = Schema.Struct({ + threadId: ThreadId, + turns: Schema.Array(Schema.Struct({ id: TurnId, items: Schema.Array(Schema.Unknown) })), +}); + +// ── Provider adapter group ─────────────────────────────────────────────── + +export const RunnerHelloRpc = Rpc.make("runner.hello", { + payload: Schema.Struct({}), + success: RunnerHello, +}); + +export const RunnerStartSessionRpc = Rpc.make("runner.provider.startSession", { + payload: Schema.Struct({ ...InstanceScoped, input: ProviderSessionStartInput }), + success: ProviderSession, + error: RunnerProviderError, +}); + +export const RunnerSendTurnRpc = Rpc.make("runner.provider.sendTurn", { + payload: Schema.Struct({ ...InstanceScoped, input: ProviderSendTurnInput }), + success: ProviderTurnStartResult, + error: RunnerProviderError, +}); + +export const RunnerInterruptTurnRpc = Rpc.make("runner.provider.interruptTurn", { + payload: Schema.Struct({ + ...InstanceScoped, + threadId: ThreadId, + turnId: Schema.optional(TurnId), + }), + success: Schema.Void, + error: RunnerProviderError, +}); + +export const RunnerRespondToRequestRpc = Rpc.make("runner.provider.respondToRequest", { + payload: Schema.Struct({ + ...InstanceScoped, + threadId: ThreadId, + requestId: ApprovalRequestId, + decision: ProviderApprovalDecision, + }), + success: Schema.Void, + error: RunnerProviderError, +}); + +export const RunnerRespondToUserInputRpc = Rpc.make("runner.provider.respondToUserInput", { + payload: Schema.Struct({ + ...InstanceScoped, + threadId: ThreadId, + requestId: ApprovalRequestId, + answers: ProviderUserInputAnswers, + }), + success: Schema.Void, + error: RunnerProviderError, +}); + +export const RunnerStopSessionRpc = Rpc.make("runner.provider.stopSession", { + payload: Schema.Struct({ ...InstanceScoped, threadId: ThreadId }), + success: Schema.Void, + error: RunnerProviderError, +}); + +export const RunnerListSessionsRpc = Rpc.make("runner.provider.listSessions", { + payload: Schema.Struct({ ...InstanceScoped }), + success: Schema.Array(ProviderSession), + error: RunnerProviderError, +}); + +export const RunnerReadThreadRpc = Rpc.make("runner.provider.readThread", { + payload: Schema.Struct({ ...InstanceScoped, threadId: ThreadId }), + success: RunnerThreadSnapshot, + error: RunnerProviderError, +}); + +export const RunnerRollbackThreadRpc = Rpc.make("runner.provider.rollbackThread", { + payload: Schema.Struct({ ...InstanceScoped, threadId: ThreadId, numTurns: NonNegativeInt }), + success: RunnerThreadSnapshot, + error: RunnerProviderError, +}); + +/** Provider status/models snapshot as the runner's real driver sees it. */ +export const RunnerGetCapabilitiesRpc = Rpc.make("runner.provider.getCapabilities", { + payload: Schema.Struct({ ...InstanceScoped, refresh: Schema.optional(Schema.Boolean) }), + success: Schema.Struct({ + snapshot: ServerProvider, + sessionModelSwitch: Schema.Literals(["in-session", "unsupported"]), + }), + error: RunnerProviderError, +}); + +export const RunnerGenerateThreadTitleRpc = Rpc.make("runner.text.generateThreadTitle", { + payload: Schema.Struct({ + ...InstanceScoped, + cwd: Schema.String, + message: Schema.String, + previousTitle: Schema.optional(Schema.String), + attachments: Schema.optional(Schema.Array(ChatAttachment)), + modelSelection: ModelSelection, + }), + success: Schema.Struct({ title: Schema.String }), + error: TextGenerationError, +}); + +export const RunnerGenerateBranchNameRpc = Rpc.make("runner.text.generateBranchName", { + payload: Schema.Struct({ + ...InstanceScoped, + cwd: Schema.String, + message: Schema.String, + attachments: Schema.optional(Schema.Array(ChatAttachment)), + modelSelection: ModelSelection, + }), + success: Schema.Struct({ branch: Schema.String }), + error: TextGenerationError, +}); + +// ── Event outbox ───────────────────────────────────────────────────────── + +/** + * Replays every outboxed event with `sequence > afterSequence`, then tails + * live events. Idempotent: the hub may resubscribe from any acked point. + */ +export const RunnerSubscribeEventsRpc = Rpc.make("runner.events.subscribe", { + payload: Schema.Struct({ afterSequence: NonNegativeInt }), + success: RunnerEventEnvelope, + stream: true, +}); + +/** Marks every event with `sequence <= throughSequence` durable on the hub. */ +export const RunnerAckEventsRpc = Rpc.make("runner.events.ack", { + payload: Schema.Struct({ throughSequence: NonNegativeInt }), + success: Schema.Struct({ ackedSequence: NonNegativeInt, retained: NonNegativeInt }), +}); + +// ── Checkpoints (hidden git refs in the runner's checkout) ─────────────── + +export const RunnerIsGitRepositoryRpc = Rpc.make("runner.checkpoint.isGitRepository", { + payload: Schema.Struct({ cwd: Schema.String }), + success: Schema.Boolean, + error: VcsError, +}); + +export const RunnerCaptureCheckpointRpc = Rpc.make("runner.checkpoint.capture", { + payload: Schema.Struct({ cwd: Schema.String, checkpointRef: CheckpointRef }), + success: Schema.Void, + error: VcsError, +}); + +export const RunnerHasCheckpointRefRpc = Rpc.make("runner.checkpoint.hasRef", { + payload: Schema.Struct({ cwd: Schema.String, checkpointRef: CheckpointRef }), + success: Schema.Boolean, + error: VcsError, +}); + +export const RunnerRestoreCheckpointRpc = Rpc.make("runner.checkpoint.restore", { + payload: Schema.Struct({ + cwd: Schema.String, + checkpointRef: CheckpointRef, + fallbackToHead: Schema.optional(Schema.Boolean), + }), + success: Schema.Boolean, + error: VcsError, +}); + +export const RunnerDiffCheckpointsRpc = Rpc.make("runner.checkpoint.diff", { + payload: Schema.Struct({ + cwd: Schema.String, + fromCheckpointRef: CheckpointRef, + toCheckpointRef: CheckpointRef, + fallbackFromToHead: Schema.optional(Schema.Boolean), + ignoreWhitespace: Schema.Boolean, + }), + success: Schema.String, + error: VcsError, +}); + +export const RunnerDeleteCheckpointRefsRpc = Rpc.make("runner.checkpoint.deleteRefs", { + payload: Schema.Struct({ cwd: Schema.String, checkpointRefs: Schema.Array(CheckpointRef) }), + success: Schema.Void, + error: VcsError, +}); + +// ── Workspace ──────────────────────────────────────────────────────────── + +export const RunnerNormalizeWorkspaceRootRpc = Rpc.make("runner.workspace.normalizeRoot", { + payload: Schema.Struct({ + workspaceRoot: Schema.String, + createIfMissing: Schema.optional(Schema.Boolean), + }), + success: Schema.String, + error: RunnerWorkspaceError, +}); + +export const RunnerRpcGroup = RpcGroup.make( + RunnerHelloRpc, + RunnerStartSessionRpc, + RunnerSendTurnRpc, + RunnerInterruptTurnRpc, + RunnerRespondToRequestRpc, + RunnerRespondToUserInputRpc, + RunnerStopSessionRpc, + RunnerListSessionsRpc, + RunnerReadThreadRpc, + RunnerRollbackThreadRpc, + RunnerGetCapabilitiesRpc, + RunnerGenerateThreadTitleRpc, + RunnerGenerateBranchNameRpc, + RunnerSubscribeEventsRpc, + RunnerAckEventsRpc, + RunnerIsGitRepositoryRpc, + RunnerCaptureCheckpointRpc, + RunnerHasCheckpointRefRpc, + RunnerRestoreCheckpointRpc, + RunnerDiffCheckpointsRpc, + RunnerDeleteCheckpointRefsRpc, + RunnerNormalizeWorkspaceRootRpc, +); diff --git a/apps/server/src/runner/RunnerServer.ts b/apps/server/src/runner/RunnerServer.ts new file mode 100644 index 000000000000..a059636fc2e0 --- /dev/null +++ b/apps/server/src/runner/RunnerServer.ts @@ -0,0 +1,360 @@ +/** + * RunnerServer - the runner build (prototype). + * + * A runner is the half of the T3 server that must live next to a checkout: + * real provider drivers, the checkpoint store, and workspace validation. It + * serves `RunnerRpcGroup` on `RUNNER_WS_PATH` and never runs orchestration, + * projections, or the client API; the hub does. + * + * Provider runtime events from every hosted instance are stamped with their + * instance id and appended to the durable `RunnerOutbox` before any hub sees + * them. + * + * @module runner/RunnerServer + */ +import * as NodeCrypto from "node:crypto"; + +import type { ProviderInstanceId } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import { FetchHttpClient, HttpRouter, HttpServerResponse } from "effect/unstable/http"; +import { RpcSerialization, RpcServer } from "effect/unstable/rpc"; + +import * as CheckpointStore from "../checkpointing/CheckpointStore.ts"; +import { ServerConfig } from "../config.ts"; +import { isGitRepository } from "../git/Utils.ts"; +import { ClaudeDriver, type ClaudeDriverEnv } from "../provider/Drivers/ClaudeDriver.ts"; +import { CodexDriver, type CodexDriverEnv } from "../provider/Drivers/CodexDriver.ts"; +import { ProviderAdapterValidationError } from "../provider/Errors.ts"; +import * as ModelManifest from "../provider/ModelManifest.ts"; +import type { AnyProviderDriver } from "../provider/ProviderDriver.ts"; +import * as ProviderEventLoggers from "../provider/Layers/ProviderEventLoggers.ts"; +import { makeProviderInstanceRegistryHydration } from "../provider/Layers/ProviderInstanceRegistryHydration.ts"; +import type { ProviderAdapterShape } from "../provider/Services/ProviderAdapter.ts"; +import { ProviderInstanceRegistry } from "../provider/Services/ProviderInstanceRegistry.ts"; +import type { ProviderAdapterError } from "../provider/Errors.ts"; +import { + ApplicationObservabilityLive, + BackgroundLayerLive, + HttpServerLive, + PlatformServicesLive, + VcsDriverRegistryLayerLive, +} from "../server.ts"; +import { TextGenerationError } from "@t3tools/contracts"; +import * as VcsProcess from "../vcs/VcsProcess.ts"; +import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; +import { RunnerOutbox, layer as RunnerOutboxLayer } from "./RunnerOutbox.ts"; +import { + RUNNER_PROTOCOL_VERSION, + RUNNER_WS_PATH, + RunnerRpcGroup, + RunnerWorkspaceError, +} from "./RunnerProtocol.ts"; + +type RunnerDriversEnv = ClaudeDriverEnv | CodexDriverEnv; + +const RUNNER_DRIVERS: ReadonlyArray> = [ + ClaudeDriver, + CodexDriver, +]; + +/** `T3CODE_RUNNER_DRIVERS=claudeAgent,codex` selects which drivers this runner hosts. */ +const selectRunnerDrivers = () => { + const wanted = new Set( + (process.env.T3CODE_RUNNER_DRIVERS ?? "claudeAgent,codex") + .split(",") + .map((value) => value.trim()) + .filter((value) => value.length > 0), + ); + return RUNNER_DRIVERS.filter((driver) => wanted.has(driver.driverKind)); +}; + +const RunnerRegistryLive = Layer.unwrap( + Effect.sync(() => makeProviderInstanceRegistryHydration(selectRunnerDrivers())), +); + +/** Appends every hosted adapter's runtime events to the outbox. */ +const RunnerEventPumpLive = Layer.effectDiscard( + Effect.gen(function* () { + const registry = yield* ProviderInstanceRegistry; + const outbox = yield* RunnerOutbox; + const subscribed = new Map>(); + const reconcile = Effect.gen(function* () { + const instances = yield* registry.listInstances; + for (const instance of instances) { + if (subscribed.get(instance.instanceId) === instance.adapter) continue; + subscribed.set(instance.instanceId, instance.adapter); + yield* Effect.logInfo("runner hosting provider instance", { + instanceId: instance.instanceId, + driver: instance.driverKind, + }); + yield* Stream.runForEach(instance.adapter.streamEvents, (event) => + outbox.append({ ...event, providerInstanceId: instance.instanceId }), + ).pipe(Effect.forkScoped); + } + }); + const changes = yield* registry.subscribeChanges; + yield* reconcile; + yield* Stream.runForEach(Stream.fromSubscription(changes), () => reconcile).pipe( + Effect.forkScoped, + ); + // Registered after the pumps, so it runs first on shutdown: stopping the + // sessions while the pumps still run outboxes their exit events, and the + // hub learns the sessions ended instead of waiting on a silent turn. + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + const instances = yield* registry.listInstances; + yield* Effect.forEach( + instances, + (instance) => instance.adapter.stopAll().pipe(Effect.ignore), + { discard: true }, + ); + yield* Effect.sleep("300 millis"); + yield* Effect.logInfo("runner drained provider sessions into the outbox", { + ...(yield* outbox.stats), + }); + }), + ); + }), +); + +const RunnerRpcHandlersLive = RunnerRpcGroup.toLayer( + Effect.gen(function* () { + const registry = yield* ProviderInstanceRegistry; + const checkpointStore = yield* CheckpointStore.CheckpointStore; + const workspacePaths = yield* WorkspacePaths.WorkspacePaths; + const outbox = yield* RunnerOutbox; + + const instanceOf = (instanceId: ProviderInstanceId) => + registry.getInstance(instanceId).pipe( + Effect.flatMap((instance) => + instance + ? Effect.succeed(instance) + : Effect.fail( + new ProviderAdapterValidationError({ + provider: String(instanceId), + operation: "runner.resolveInstance", + issue: `This runner does not host provider instance '${instanceId}'.`, + }), + ), + ), + ); + const adapterOf = (instanceId: ProviderInstanceId) => + instanceOf(instanceId).pipe(Effect.map((instance) => instance.adapter)); + /** Logs each state-changing call with its duration, for protocol evidence. */ + const logged = ( + method: string, + attributes: Record, + effect: Effect.Effect, + ) => + Effect.gen(function* () { + const startedAt = yield* Clock.currentTimeMillis; + const exit = yield* Effect.exit(effect); + yield* Effect.logInfo(`runner rpc ${method}`, { + ...attributes, + ok: exit._tag === "Success", + ms: (yield* Clock.currentTimeMillis) - startedAt, + }); + return yield* exit; + }); + const textGenerationOf = (instanceId: ProviderInstanceId, operation: string) => + instanceOf(instanceId).pipe( + Effect.map((instance) => instance.textGeneration), + Effect.mapError((error) => new TextGenerationError({ operation, detail: error.message })), + ); + + return RunnerRpcGroup.of({ + "runner.hello": () => + Effect.gen(function* () { + const stats = yield* outbox.stats; + const instances = yield* registry.listInstances; + return { + protocolVersion: RUNNER_PROTOCOL_VERSION, + runnerId: stats.runnerId, + bootId: stats.bootId, + headSequence: stats.headSequence, + ackedSequence: stats.ackedSequence, + instances: instances.map((instance) => instance.instanceId), + }; + }), + "runner.provider.startSession": ({ instanceId, input }) => + logged( + "startSession", + { + threadId: input.threadId, + cwd: input.cwd, + runtimeMode: input.runtimeMode, + resumeCursor: input.resumeCursor ?? null, + }, + adapterOf(instanceId).pipe(Effect.flatMap((adapter) => adapter.startSession(input))), + ), + "runner.provider.sendTurn": ({ instanceId, input }) => + logged( + "sendTurn", + { threadId: input.threadId }, + adapterOf(instanceId).pipe(Effect.flatMap((adapter) => adapter.sendTurn(input))), + ), + "runner.provider.interruptTurn": ({ instanceId, threadId, turnId }) => + logged( + "interruptTurn", + { threadId, turnId }, + adapterOf(instanceId).pipe( + Effect.flatMap((adapter) => adapter.interruptTurn(threadId, turnId)), + ), + ), + "runner.provider.respondToRequest": ({ instanceId, threadId, requestId, decision }) => + logged( + "respondToRequest", + { threadId, requestId, decision }, + adapterOf(instanceId).pipe( + Effect.flatMap((adapter) => adapter.respondToRequest(threadId, requestId, decision)), + ), + ), + "runner.provider.respondToUserInput": ({ instanceId, threadId, requestId, answers }) => + adapterOf(instanceId).pipe( + Effect.flatMap((adapter) => adapter.respondToUserInput(threadId, requestId, answers)), + ), + "runner.provider.stopSession": ({ instanceId, threadId }) => + logged( + "stopSession", + { threadId }, + adapterOf(instanceId).pipe(Effect.flatMap((adapter) => adapter.stopSession(threadId))), + ), + "runner.provider.listSessions": ({ instanceId }) => + adapterOf(instanceId).pipe(Effect.flatMap((adapter) => adapter.listSessions())), + "runner.provider.readThread": ({ instanceId, threadId }) => + adapterOf(instanceId).pipe(Effect.flatMap((adapter) => adapter.readThread(threadId))), + "runner.provider.rollbackThread": ({ instanceId, threadId, numTurns }) => + adapterOf(instanceId).pipe( + Effect.flatMap((adapter) => adapter.rollbackThread(threadId, numTurns)), + ), + "runner.provider.getCapabilities": ({ instanceId, refresh }) => + instanceOf(instanceId).pipe( + Effect.flatMap((instance) => + (refresh ? instance.snapshot.refresh : instance.snapshot.getSnapshot).pipe( + Effect.map((snapshot) => ({ + snapshot, + sessionModelSwitch: instance.adapter.capabilities.sessionModelSwitch, + })), + ), + ), + ), + "runner.text.generateThreadTitle": ({ instanceId, ...request }) => + logged( + "text.generateThreadTitle", + { cwd: request.cwd, model: request.modelSelection.model }, + textGenerationOf(instanceId, "generateThreadTitle").pipe( + Effect.flatMap((textGeneration) => textGeneration.generateThreadTitle(request)), + ), + ), + "runner.text.generateBranchName": ({ instanceId, ...request }) => + textGenerationOf(instanceId, "generateBranchName").pipe( + Effect.flatMap((textGeneration) => textGeneration.generateBranchName(request)), + ), + "runner.events.subscribe": ({ afterSequence }) => outbox.subscribe(afterSequence), + "runner.events.ack": ({ throughSequence }) => outbox.ack(throughSequence), + // Same `.git` presence check the in-process reactors used, so hub mode + // keeps the exact checkpoint eligibility rule. + "runner.checkpoint.isGitRepository": ({ cwd }) => Effect.sync(() => isGitRepository(cwd)), + "runner.checkpoint.capture": (input) => + logged( + "checkpoint.capture", + { checkpointRef: input.checkpointRef }, + checkpointStore.captureCheckpoint(input), + ), + "runner.checkpoint.hasRef": (input) => checkpointStore.hasCheckpointRef(input), + "runner.checkpoint.restore": ({ cwd, checkpointRef, fallbackToHead }) => + checkpointStore.restoreCheckpoint({ + cwd, + checkpointRef, + ...(fallbackToHead !== undefined ? { fallbackToHead } : {}), + }), + "runner.checkpoint.diff": ({ fallbackFromToHead, ...input }) => + checkpointStore.diffCheckpoints({ + ...input, + ...(fallbackFromToHead !== undefined ? { fallbackFromToHead } : {}), + }), + "runner.checkpoint.deleteRefs": (input) => checkpointStore.deleteCheckpointRefs(input), + "runner.workspace.normalizeRoot": ({ workspaceRoot, createIfMissing }) => + workspacePaths + .normalizeWorkspaceRoot( + workspaceRoot, + createIfMissing !== undefined ? { createIfMissing } : undefined, + ) + .pipe( + Effect.mapError( + (error) => + new RunnerWorkspaceError({ + operation: "normalizeRoot", + workspaceRoot, + reason: + error._tag === "WorkspaceRootNotExistsError" + ? "not-exists" + : error._tag === "WorkspaceRootNotDirectoryError" + ? "not-directory" + : error._tag === "WorkspaceRootCreateFailedError" + ? "create-failed" + : "stat-failed", + detail: error.message, + }), + ), + ), + }); + }), +); + +function tokenMatches(presented: string | null, expected: string): boolean { + if (presented === null) return false; + const left = Buffer.from(presented); + const right = Buffer.from(expected); + return left.length === right.length && NodeCrypto.timingSafeEqual(left, right); +} + +const RunnerRouteLive = Layer.unwrap( + Effect.gen(function* () { + const config = yield* ServerConfig; + const expectedToken = config.runnerToken; + const handlers = yield* Layer.build( + RunnerRpcHandlersLive.pipe(Layer.provideMerge(RpcSerialization.layerJson)), + ); + const rpcHttpEffect = yield* RpcServer.toHttpEffectWebsocket(RunnerRpcGroup, { + disableTracing: true, + }).pipe(Effect.provide(handlers)); + return HttpRouter.add("GET", RUNNER_WS_PATH, (request) => + Effect.gen(function* () { + const token = new URL(request.url, "http://runner.local").searchParams.get("token"); + if (expectedToken && !tokenMatches(token, expectedToken)) { + return HttpServerResponse.text("unauthorized", { status: 401 }); + } + return yield* rpcHttpEffect; + }), + ); + }), +); + +const RunnerServicesLive = Layer.mergeAll(RunnerEventPumpLive, RunnerRouteLive).pipe( + Layer.provideMerge(RunnerOutboxLayer), + Layer.provideMerge(RunnerRegistryLive), + Layer.provideMerge( + Layer.mergeAll( + CheckpointStore.layer.pipe(Layer.provide(VcsDriverRegistryLayerLive)), + WorkspacePaths.layer, + ), + ), + Layer.provideMerge(Layer.mergeAll(ProviderEventLoggers.layer, ModelManifest.layer)), + Layer.provideMerge(BackgroundLayerLive), +); + +export const makeRunnerLayer = HttpRouter.serve(RunnerServicesLive, { + disableLogger: true, +}).pipe( + Layer.provideMerge(HttpServerLive), + Layer.provide(ApplicationObservabilityLive), + Layer.provideMerge(FetchHttpClient.layer), + Layer.provideMerge(VcsProcess.layer), + Layer.provideMerge(PlatformServicesLive), +); + +export const runRunner = Layer.launch(makeRunnerLayer); diff --git a/apps/server/src/runner/hub/HubLayers.ts b/apps/server/src/runner/hub/HubLayers.ts new file mode 100644 index 000000000000..03a5dfcbe1a3 --- /dev/null +++ b/apps/server/src/runner/hub/HubLayers.ts @@ -0,0 +1,225 @@ +/** + * HubLayers - services a checkout-less hub substitutes for host-bound ones. + * + * Each export replaces one service whose local implementation would stat, + * read, or run git against a thread checkout that only exists on the runner: + * + * - delegated: CheckpointStore, CheckoutGitProbe, WorkspacePaths root + * validation (runner RPCs, same cwd strings). + * - stubbed: RepositoryIdentityResolver (null identity), + * WorkspaceEntries (empty index, refresh is a no-op), VcsStatusBroadcaster + * (fails; the runner should push status to a hub cache in production). + * + * @module runner/hub/HubLayers + */ +import { GitManagerError, VcsRepositoryDetectionError, type VcsError } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; + +import { CheckpointStore } from "../../checkpointing/CheckpointStore.ts"; +import { CheckoutGitProbe } from "../../git/CheckoutGitProbe.ts"; +import { RepositoryIdentityResolver } from "../../project/RepositoryIdentityResolver.ts"; +import { VcsStatusBroadcaster } from "../../vcs/VcsStatusBroadcaster.ts"; +import { WorkspaceEntries } from "../../workspace/WorkspaceEntries.ts"; +import * as WorkspacePaths from "../../workspace/WorkspacePaths.ts"; +import { RunnerClient } from "./RunnerClient.ts"; + +const describe = (cause: unknown): string => + cause && typeof cause === "object" && "message" in cause + ? String((cause as { message: unknown }).message) + : String(cause); + +const VCS_ERROR_TAGS = new Set([ + "VcsProcessSpawnError", + "VcsProcessExitError", + "VcsProcessTimeoutError", + "VcsProcessStdinWriteError", + "VcsProcessOutputReadError", + "VcsProcessOutputLimitError", + "VcsProcessMissingExitCodeError", + "VcsRepositoryDetectionError", + "VcsUnsupportedOperationError", +]); + +const toVcsError = + (operation: string, cwd: string) => + (error: { readonly _tag: string }): VcsError => + VCS_ERROR_TAGS.has(error._tag) + ? (error as VcsError) + : new VcsRepositoryDetectionError({ + operation, + cwd, + detail: `runner call failed: ${describe(error)}`, + }); + +export const remoteCheckpointStoreLayer = Layer.effect( + CheckpointStore, + Effect.gen(function* () { + const runner = yield* RunnerClient; + return CheckpointStore.of({ + isGitRepository: (cwd) => + runner + .use((client) => client["runner.checkpoint.isGitRepository"]({ cwd })) + .pipe(Effect.mapError(toVcsError("CheckpointStore.isGitRepository", cwd))), + captureCheckpoint: (input) => + runner + .use((client) => + client["runner.checkpoint.capture"]({ + cwd: input.cwd, + checkpointRef: input.checkpointRef, + }), + ) + .pipe(Effect.mapError(toVcsError("CheckpointStore.captureCheckpoint", input.cwd))), + hasCheckpointRef: (input) => + runner + .use((client) => + client["runner.checkpoint.hasRef"]({ + cwd: input.cwd, + checkpointRef: input.checkpointRef, + }), + ) + .pipe(Effect.mapError(toVcsError("CheckpointStore.hasCheckpointRef", input.cwd))), + restoreCheckpoint: (input) => + runner + .use((client) => + client["runner.checkpoint.restore"]({ + cwd: input.cwd, + checkpointRef: input.checkpointRef, + ...(input.fallbackToHead !== undefined + ? { fallbackToHead: input.fallbackToHead } + : {}), + }), + ) + .pipe(Effect.mapError(toVcsError("CheckpointStore.restoreCheckpoint", input.cwd))), + diffCheckpoints: (input) => + runner + .use((client) => + client["runner.checkpoint.diff"]({ + cwd: input.cwd, + fromCheckpointRef: input.fromCheckpointRef, + toCheckpointRef: input.toCheckpointRef, + ...(input.fallbackFromToHead !== undefined + ? { fallbackFromToHead: input.fallbackFromToHead } + : {}), + ignoreWhitespace: input.ignoreWhitespace, + }), + ) + .pipe(Effect.mapError(toVcsError("CheckpointStore.diffCheckpoints", input.cwd))), + deleteCheckpointRefs: (input) => + runner + .use((client) => + client["runner.checkpoint.deleteRefs"]({ + cwd: input.cwd, + checkpointRefs: input.checkpointRefs, + }), + ) + .pipe(Effect.mapError(toVcsError("CheckpointStore.deleteCheckpointRefs", input.cwd))), + }); + }), +); + +export const remoteCheckoutGitProbeLayer = Layer.effect( + CheckoutGitProbe, + Effect.gen(function* () { + const runner = yield* RunnerClient; + return (cwd: string) => + runner + .use((client) => client["runner.checkpoint.isGitRepository"]({ cwd })) + .pipe( + Effect.catch((error) => + Effect.logWarning("runner git probe failed; treating checkout as non-git", { + cwd, + detail: describe(error), + }).pipe(Effect.as(false)), + ), + ); + }), +); + +export const remoteWorkspacePathsLayer = Layer.effect( + WorkspacePaths.WorkspacePaths, + Effect.gen(function* () { + const runner = yield* RunnerClient; + // Relative-path resolution is pure; only root validation touches disk. + const local = yield* WorkspacePaths.make; + return WorkspacePaths.WorkspacePaths.of({ + resolveRelativePathWithinRoot: local.resolveRelativePathWithinRoot, + normalizeWorkspaceRoot: (workspaceRoot, options) => + runner + .use((client) => + client["runner.workspace.normalizeRoot"]({ + workspaceRoot, + ...(options?.createIfMissing !== undefined + ? { createIfMissing: options.createIfMissing } + : {}), + }), + ) + .pipe( + Effect.mapError((error) => { + if (error._tag === "RunnerWorkspaceError") { + switch (error.reason) { + case "not-exists": + return new WorkspacePaths.WorkspaceRootNotExistsError({ + workspaceRoot, + normalizedWorkspaceRoot: workspaceRoot, + }); + case "not-directory": + return new WorkspacePaths.WorkspaceRootNotDirectoryError({ + workspaceRoot, + normalizedWorkspaceRoot: workspaceRoot, + }); + case "create-failed": + return new WorkspacePaths.WorkspaceRootCreateFailedError({ + workspaceRoot, + normalizedWorkspaceRoot: workspaceRoot, + cause: error, + }); + default: + break; + } + } + return new WorkspacePaths.WorkspaceRootStatFailedError({ + workspaceRoot, + normalizedWorkspaceRoot: workspaceRoot, + phase: "validate-existing", + cause: error, + }); + }), + ), + }); + }), +); + +export const hubWorkspaceEntriesLayer = Layer.succeed( + WorkspaceEntries, + WorkspaceEntries.of({ + browse: (input) => Effect.succeed({ parentPath: input.partialPath, entries: [] }), + list: () => Effect.succeed({ entries: [], truncated: false }), + search: () => Effect.succeed({ entries: [], truncated: false }), + searchContents: () => Effect.succeed({ matches: [], truncated: false }), + refresh: () => Effect.void, + }), +); + +export const hubRepositoryIdentityResolverLayer = Layer.succeed( + RepositoryIdentityResolver, + RepositoryIdentityResolver.of({ resolve: () => Effect.succeed(null) }), +); + +const vcsStatusUnavailable = (operation: string, cwd: string) => + new GitManagerError({ + operation, + cwd, + detail: "VCS status lives on the thread's runner and is not delegated in prototype 2.", + }); + +export const hubVcsStatusBroadcasterLayer = Layer.succeed( + VcsStatusBroadcaster, + VcsStatusBroadcaster.of({ + getStatus: (input) => Effect.fail(vcsStatusUnavailable("getStatus", input.cwd)), + refreshLocalStatus: (cwd) => Effect.fail(vcsStatusUnavailable("refreshLocalStatus", cwd)), + refreshStatus: (cwd) => Effect.fail(vcsStatusUnavailable("refreshStatus", cwd)), + streamStatus: (input) => Stream.fail(vcsStatusUnavailable("streamStatus", input.cwd)), + }), +); diff --git a/apps/server/src/runner/hub/RemoteProviderDriver.ts b/apps/server/src/runner/hub/RemoteProviderDriver.ts new file mode 100644 index 000000000000..18093491149e --- /dev/null +++ b/apps/server/src/runner/hub/RemoteProviderDriver.ts @@ -0,0 +1,314 @@ +/** + * RemoteProviderDriver - hub-side provider driver that runs nothing locally. + * + * Wraps a built-in driver's identity (driver kind, config schema, defaults) so + * the hub keeps the same instance ids, model selections and session directory + * rows, while the adapter forwards every call to the runner that owns the + * checkout. `ProviderService` above it is unchanged: it still persists resume + * cursors, recovers sessions and serializes per-thread lifecycle work. + * + * `listSessions`/`hasSession` ask the runner so that a hub restart adopts the + * runner's still-running sessions instead of marking them orphaned; when the + * runner is unreachable they report nothing and `ProviderService` recovers + * through the persisted resume cursor on the next turn. + * + * @module runner/hub/RemoteProviderDriver + */ +import { + type ProviderDriverKind, + type ProviderInstanceId, + type ServerProvider, + TextGenerationError, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; + +import { + type ProviderAdapterError, + ProviderAdapterRequestError, + ProviderDriverError, +} from "../../provider/Errors.ts"; +import { + defaultProviderContinuationIdentity, + type AnyProviderDriver, + type ProviderInstance, +} from "../../provider/ProviderDriver.ts"; +import type { + ProviderAdapterShape, + ProviderThreadSnapshot, +} from "../../provider/Services/ProviderAdapter.ts"; +import type { ServerProviderShape } from "../../provider/Services/ServerProvider.ts"; +import type { TextGenerationShape } from "../../textGeneration/TextGeneration.ts"; +import { buildUnavailableProviderSnapshot } from "../../provider/unavailableProviderSnapshot.ts"; +import { RunnerClient, type RunnerRpcClient } from "./RunnerClient.ts"; + +const describe = (cause: unknown): string => { + if (cause && typeof cause === "object" && "message" in cause) { + return String((cause as { message: unknown }).message); + } + return String(cause); +}; + +export function makeRemoteProviderDriver( + base: AnyProviderDriver, +): AnyProviderDriver { + const driverKind: ProviderDriverKind = base.driverKind; + return { + driverKind, + metadata: base.metadata, + configSchema: base.configSchema, + defaultConfig: base.defaultConfig as () => Config, + create: ({ instanceId, displayName, accentColor, enabled }) => + Effect.gen(function* () { + const runnerOption = yield* Effect.serviceOption(RunnerClient); + if (Option.isNone(runnerOption) || !runnerOption.value.enabled) { + return yield* new ProviderDriverError({ + driver: driverKind, + instanceId, + detail: "Hub mode requires a configured runner.", + }); + } + const runner = runnerOption.value; + const adapter = makeRemoteAdapter({ runner, driverKind, instanceId }); + const snapshot = yield* makeRemoteSnapshot({ + runner, + driverKind, + instanceId, + displayName, + accentColor, + }); + const textGeneration = makeRemoteTextGeneration({ runner, instanceId }); + return { + instanceId, + driverKind, + continuationIdentity: defaultProviderContinuationIdentity({ driverKind, instanceId }), + displayName, + accentColor, + enabled, + snapshot, + adapter, + textGeneration, + } satisfies ProviderInstance; + }), + }; +} + +function makeRemoteAdapter(input: { + readonly runner: RunnerClient["Service"]; + readonly driverKind: ProviderDriverKind; + readonly instanceId: ProviderInstanceId; +}): ProviderAdapterShape { + const { runner, driverKind, instanceId } = input; + + const call = ( + method: string, + f: (client: RunnerRpcClient) => Effect.Effect, + ): Effect.Effect => + runner.use(f).pipe( + Effect.mapError((error): ProviderAdapterError => { + switch (error._tag) { + case "ProviderAdapterValidationError": + case "ProviderAdapterSessionNotFoundError": + case "ProviderAdapterSessionClosedError": + case "ProviderAdapterRequestError": + case "ProviderAdapterProcessError": + return error as ProviderAdapterError; + default: + return new ProviderAdapterRequestError({ + provider: driverKind, + method, + detail: `runner call failed: ${describe(error)}`, + cause: error, + }); + } + }), + ); + + const listSessions = () => + runner.hello + .pipe( + Effect.map((hello) => hello.instances.includes(instanceId)), + Effect.orElseSucceed(() => true), + Effect.flatMap((hosted) => + hosted + ? call("listSessions", (client) => + client["runner.provider.listSessions"]({ instanceId }), + ) + : Effect.succeed([]), + ), + ) + .pipe( + Effect.catch((error) => + Effect.logWarning("remote listSessions unavailable; reporting none", { + instanceId, + detail: error.message, + }).pipe(Effect.as([])), + ), + ); + + const toThreadSnapshot = (snapshot: { + readonly threadId: ProviderThreadSnapshot["threadId"]; + readonly turns: ReadonlyArray<{ readonly id: string; readonly items: ReadonlyArray }>; + }): ProviderThreadSnapshot => snapshot as ProviderThreadSnapshot; + + return { + provider: driverKind, + // Model switching is a property of the runner's real adapter; the hub + // treats it as in-session and lets the runner reject when unsupported. + capabilities: { sessionModelSwitch: "in-session" }, + startSession: (sessionInput) => + call("startSession", (client) => + client["runner.provider.startSession"]({ instanceId, input: sessionInput }), + ), + sendTurn: (turnInput) => + call("sendTurn", (client) => + client["runner.provider.sendTurn"]({ instanceId, input: turnInput }), + ), + interruptTurn: (threadId, turnId) => + call("interruptTurn", (client) => + client["runner.provider.interruptTurn"]({ + instanceId, + threadId, + ...(turnId !== undefined ? { turnId } : {}), + }), + ), + respondToRequest: (threadId, requestId, decision) => + call("respondToRequest", (client) => + client["runner.provider.respondToRequest"]({ instanceId, threadId, requestId, decision }), + ), + respondToUserInput: (threadId, requestId, answers) => + call("respondToUserInput", (client) => + client["runner.provider.respondToUserInput"]({ instanceId, threadId, requestId, answers }), + ), + stopSession: (threadId) => + call("stopSession", (client) => + client["runner.provider.stopSession"]({ instanceId, threadId }), + ), + listSessions, + hasSession: (threadId) => + listSessions().pipe( + Effect.map((sessions) => sessions.some((session) => session.threadId === threadId)), + ), + readThread: (threadId) => + call("readThread", (client) => + client["runner.provider.readThread"]({ instanceId, threadId }), + ).pipe(Effect.map(toThreadSnapshot)), + rollbackThread: (threadId, numTurns) => + call("rollbackThread", (client) => + client["runner.provider.rollbackThread"]({ instanceId, threadId, numTurns }), + ).pipe(Effect.map(toThreadSnapshot)), + // A hub shutting down must not stop sessions it does not host; the runner + // keeps them running and outboxes their events until the hub returns. + stopAll: () => Effect.void, + streamEvents: runner.eventsFor(instanceId), + }; +} + +const makeRemoteSnapshot = (input: { + readonly runner: RunnerClient["Service"]; + readonly driverKind: ProviderDriverKind; + readonly instanceId: ProviderInstanceId; + readonly displayName: string | undefined; + readonly accentColor: string | undefined; +}) => + Effect.gen(function* () { + const { runner, driverKind, instanceId } = input; + const pending = yield* buildUnavailableProviderSnapshot({ + driverKind, + instanceId, + displayName: input.displayName, + accentColor: input.accentColor, + reason: "Waiting for the thread machine to report this provider.", + }); + const current = yield* Ref.make(pending); + const changes = yield* PubSub.unbounded(); + + const fetch = (refresh: boolean) => + runner + .use((client) => + client["runner.provider.getCapabilities"]({ + instanceId, + ...(refresh ? { refresh } : {}), + }), + ) + .pipe( + Effect.map((result) => result.snapshot), + Effect.catch((error) => + buildUnavailableProviderSnapshot({ + driverKind, + instanceId, + displayName: input.displayName, + accentColor: input.accentColor, + reason: `Runner did not report this provider: ${describe(error)}`, + }), + ), + Effect.tap((snapshot) => Ref.set(current, snapshot)), + Effect.tap((snapshot) => PubSub.publish(changes, snapshot)), + ); + + // Populate in the background so hub startup never waits on a machine. + yield* fetch(false).pipe(Effect.forkScoped); + + return { + maintenanceCapabilities: { provider: driverKind, packageName: null, update: null }, + getSnapshot: Ref.get(current), + refresh: fetch(true), + streamChanges: Stream.fromPubSub(changes), + } satisfies ServerProviderShape; + }); + +function makeRemoteTextGeneration(input: { + readonly runner: RunnerClient["Service"]; + readonly instanceId: ProviderInstanceId; +}): TextGenerationShape { + const { runner, instanceId } = input; + const toTextGenerationError = (operation: string) => (error: { readonly _tag: string }) => + error._tag === "TextGenerationError" + ? (error as TextGenerationError) + : new TextGenerationError({ + operation, + detail: `runner call failed: ${describe(error)}`, + }); + const unsupported = (operation: string) => + Effect.fail( + new TextGenerationError({ + operation, + detail: "Not delegated to the runner in prototype 2.", + }), + ); + return { + generateThreadTitle: (request) => + runner + .use((client) => + client["runner.text.generateThreadTitle"]({ + instanceId, + cwd: request.cwd, + message: request.message, + ...(request.previousTitle !== undefined + ? { previousTitle: request.previousTitle } + : {}), + ...(request.attachments !== undefined ? { attachments: request.attachments } : {}), + modelSelection: request.modelSelection, + }), + ) + .pipe(Effect.mapError(toTextGenerationError("generateThreadTitle"))), + generateBranchName: (request) => + runner + .use((client) => + client["runner.text.generateBranchName"]({ + instanceId, + cwd: request.cwd, + message: request.message, + ...(request.attachments !== undefined ? { attachments: request.attachments } : {}), + modelSelection: request.modelSelection, + }), + ) + .pipe(Effect.mapError(toTextGenerationError("generateBranchName"))), + generateCommitMessage: () => unsupported("generateCommitMessage"), + generatePrContent: () => unsupported("generatePrContent"), + generateStructured: () => unsupported("generateStructured"), + }; +} diff --git a/apps/server/src/runner/hub/RunnerClient.ts b/apps/server/src/runner/hub/RunnerClient.ts new file mode 100644 index 000000000000..aeecb7ca79ff --- /dev/null +++ b/apps/server/src/runner/hub/RunnerClient.ts @@ -0,0 +1,313 @@ +// @effect-diagnostics nodeBuiltinImport:off -- The delivery cursor is a tiny synchronous state file. +/** + * RunnerClient - the hub's connection to a runner (prototype). + * + * Owns one supervised WebSocket RPC connection to the configured runner, + * reconnecting with backoff. Calls wait (bounded) for a live connection. + * + * Event delivery: once the hub's ingestion is running (`startDelivery`), the + * client subscribes to the runner outbox from its durable cursor, drops any + * sequence it already delivered, and fans events out to the remote adapters. + * `ackDelivered` persists the cursor and lets the runner compact; the caller + * invokes it only after ingestion has drained, so an acknowledged event has + * been projected. + * + * In production the hub would route by thread to that thread's machine; the + * prototype has one runner URL for the whole hub. + * + * @module runner/hub/RunnerClient + */ +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + +import type { ProviderInstanceId, ProviderRuntimeEvent } from "@t3tools/contracts"; +import * as NodeSocket from "@effect/platform-node/NodeSocket"; +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import { RpcClient, RpcSerialization } from "effect/unstable/rpc"; +import type { RpcClientError } from "effect/unstable/rpc/RpcClientError"; +import * as Socket from "effect/unstable/socket/Socket"; + +import { ServerConfig } from "../../config.ts"; +import { type RunnerHello, RunnerRpcGroup } from "../RunnerProtocol.ts"; + +export type RunnerRpcClient = RpcClient.FromGroup; + +export class RunnerUnavailableError extends Schema.TaggedErrorClass()( + "RunnerUnavailableError", + { detail: Schema.String }, +) { + override get message(): string { + return `Runner unavailable: ${this.detail}`; + } +} + +export interface RunnerDeliveryState { + readonly runnerId: string | null; + readonly bootId: string | null; + readonly deliveredSequence: number; + readonly ackedSequence: number; + readonly deliveredCount: number; + readonly duplicateCount: number; + readonly connectCount: number; +} + +export class RunnerClient extends Context.Service< + RunnerClient, + { + readonly enabled: boolean; + /** Run one call against the live connection, waiting briefly for one. */ + readonly use: ( + f: (client: RunnerRpcClient) => Effect.Effect, + ) => Effect.Effect; + readonly hello: Effect.Effect; + /** Provider runtime events for one instance, in runner sequence order. */ + readonly eventsFor: (instanceId: ProviderInstanceId) => Stream.Stream; + /** Opens the outbox subscription. Call once hub ingestion is consuming. */ + readonly startDelivery: Effect.Effect; + readonly deliveryState: Effect.Effect; + /** Persist the cursor and acknowledge everything delivered so far. */ + readonly ackDelivered: (throughSequence: number) => Effect.Effect; + } +>()("t3/runner/hub/RunnerClient") {} + +const CONNECT_WAIT = Duration.seconds(20); +const RECONNECT_DELAY = Duration.seconds(1); + +interface Connection { + readonly client: RunnerRpcClient; + readonly hello: RunnerHello; +} + +const PersistedCursorJson = Schema.fromJsonString( + Schema.Struct({ runnerId: Schema.NullOr(Schema.String), ackedSequence: Schema.Number }), +); +type PersistedCursor = typeof PersistedCursorJson.Type; +const decodeCursor = Schema.decodeUnknownSync(PersistedCursorJson); +const encodeCursor = Schema.encodeSync(PersistedCursorJson); + +function readCursor(path: string): PersistedCursor { + try { + return decodeCursor(NodeFS.readFileSync(path, "utf8")); + } catch { + return { runnerId: null, ackedSequence: 0 }; + } +} + +function writeCursor(path: string, cursor: PersistedCursor): void { + const tmp = `${path}.tmp`; + NodeFS.writeFileSync(tmp, encodeCursor(cursor)); + NodeFS.renameSync(tmp, path); +} + +const makeLive = (runnerUrl: string) => + Effect.gen(function* () { + const config = yield* ServerConfig; + const cursorPath = NodePath.join(config.stateDir, "runner-client-cursor.json"); + const persisted = readCursor(cursorPath); + const url = new URL(runnerUrl); + if (config.runnerToken) url.searchParams.set("token", config.runnerToken); + const socketUrl = url.toString(); + + const events = yield* PubSub.unbounded(); + const readyRef = yield* Ref.make(yield* Deferred.make()); + const deliveryEnabled = yield* Deferred.make(); + const state = yield* Ref.make({ + runnerId: persisted.runnerId, + bootId: null, + deliveredSequence: persisted.ackedSequence, + ackedSequence: persisted.ackedSequence, + deliveredCount: 0, + duplicateCount: 0, + connectCount: 0, + }); + + const awaitConnection = Ref.get(readyRef).pipe( + Effect.flatMap(Deferred.await), + Effect.timeoutOrElse({ + duration: CONNECT_WAIT, + orElse: () => + Effect.fail( + new RunnerUnavailableError({ + detail: `no connection to ${runnerUrl} within ${Duration.format(CONNECT_WAIT)}`, + }), + ), + }), + ); + + const pumpEvents = (client: RunnerRpcClient) => + Effect.gen(function* () { + yield* Deferred.await(deliveryEnabled); + const { deliveredSequence } = yield* Ref.get(state); + yield* Effect.logInfo("runner event subscription opened", { + afterSequence: deliveredSequence, + }); + yield* client["runner.events.subscribe"]({ afterSequence: deliveredSequence }).pipe( + Stream.runForEach((envelope) => + Ref.modify(state, (current) => { + if (envelope.sequence <= current.deliveredSequence) { + return [false, { ...current, duplicateCount: current.duplicateCount + 1 }] as const; + } + return [ + true, + { + ...current, + deliveredSequence: envelope.sequence, + deliveredCount: current.deliveredCount + 1, + }, + ] as const; + }).pipe( + Effect.flatMap((fresh) => + fresh ? PubSub.publish(events, envelope.event).pipe(Effect.asVoid) : Effect.void, + ), + ), + ), + ); + }); + + const connectOnce = Effect.scoped( + Effect.gen(function* () { + const disconnected = yield* Deferred.make(); + const hooks = RpcClient.ConnectionHooks.of({ + onConnect: Effect.void, + onDisconnect: Deferred.succeed(disconnected, undefined).pipe(Effect.asVoid), + }); + const protocol = yield* Layer.build( + Layer.effect( + RpcClient.Protocol, + RpcClient.makeProtocolSocket({ retryTransientErrors: false }), + ).pipe( + Layer.provide( + Layer.mergeAll( + Socket.layerWebSocket(socketUrl, { openTimeout: Duration.seconds(5) }).pipe( + Layer.provide(NodeSocket.layerWebSocketConstructor), + ), + RpcSerialization.layerJson, + Layer.succeed(RpcClient.ConnectionHooks, hooks), + ), + ), + ), + ); + const client = yield* RpcClient.make(RunnerRpcGroup, { disableTracing: true }).pipe( + Effect.provide(protocol), + ); + const hello = yield* client["runner.hello"]({}).pipe(Effect.timeout("10 seconds")); + yield* Ref.update(state, (current) => { + // A different runner identity means a different outbox; its + // sequences restart, so the cursor restarts with it. + const sameRunner = current.runnerId === null || current.runnerId === hello.runnerId; + return { + ...current, + runnerId: hello.runnerId, + bootId: hello.bootId, + connectCount: current.connectCount + 1, + ...(sameRunner ? {} : { deliveredSequence: 0, ackedSequence: 0 }), + }; + }); + yield* Effect.logInfo("runner connected", { + runnerId: hello.runnerId, + bootId: hello.bootId, + headSequence: hello.headSequence, + runnerAckedSequence: hello.ackedSequence, + instances: hello.instances, + }); + const ready = yield* Ref.get(readyRef); + yield* Deferred.succeed(ready, { client, hello }); + yield* Effect.raceFirst(Deferred.await(disconnected), pumpEvents(client)); + }), + ).pipe( + Effect.ensuring( + Effect.gen(function* () { + const current = yield* Ref.get(readyRef); + if (yield* Deferred.isDone(current)) { + yield* Ref.set(readyRef, yield* Deferred.make()); + } + }), + ), + ); + + yield* connectOnce.pipe( + Effect.catchCause((cause) => + Effect.logWarning("runner connection ended", { cause: String(cause).slice(0, 400) }), + ), + Effect.andThen(Effect.sleep(RECONNECT_DELAY)), + Effect.forever, + Effect.forkScoped, + ); + + const use = (f: (client: RunnerRpcClient) => Effect.Effect) => + awaitConnection.pipe(Effect.flatMap((connection) => f(connection.client))); + + const ackDelivered = (throughSequence: number) => + Effect.gen(function* () { + const current = yield* Ref.get(state); + if (throughSequence <= current.ackedSequence) return; + writeCursor(cursorPath, { runnerId: current.runnerId, ackedSequence: throughSequence }); + const latest = yield* Ref.updateAndGet(state, (value) => ({ + ...value, + ackedSequence: throughSequence, + })); + yield* Effect.logInfo("runner events acknowledged", { + throughSequence, + deliveredCount: latest.deliveredCount, + duplicateCount: latest.duplicateCount, + connectCount: latest.connectCount, + }); + yield* use((client) => client["runner.events.ack"]({ throughSequence })).pipe( + Effect.catchCause((cause) => + Effect.logWarning("runner ack failed; it is retried with the next ack", { + throughSequence, + cause: String(cause).slice(0, 200), + }), + ), + ); + }); + + return RunnerClient.of({ + enabled: true, + use, + hello: awaitConnection.pipe(Effect.map((connection) => connection.hello)), + eventsFor: (instanceId) => + Stream.fromPubSub(events).pipe( + Stream.filter((event) => event.providerInstanceId === instanceId), + ), + startDelivery: Deferred.succeed(deliveryEnabled, undefined).pipe(Effect.asVoid), + deliveryState: Ref.get(state), + ackDelivered, + }); + }); + +const disabled = RunnerClient.of({ + enabled: false, + use: () => Effect.fail(new RunnerUnavailableError({ detail: "no runner configured" })), + hello: Effect.fail(new RunnerUnavailableError({ detail: "no runner configured" })), + eventsFor: () => Stream.empty, + startDelivery: Effect.void, + deliveryState: Effect.succeed({ + runnerId: null, + bootId: null, + deliveredSequence: 0, + ackedSequence: 0, + deliveredCount: 0, + duplicateCount: 0, + connectCount: 0, + }), + ackDelivered: () => Effect.void, +}); + +/** Live client in hub mode (`T3CODE_RUNNER_URL`), an inert one otherwise. */ +export const layer = Layer.unwrap( + Effect.gen(function* () { + const config = yield* ServerConfig; + return config.runnerUrl + ? Layer.effect(RunnerClient, makeLive(config.runnerUrl)) + : Layer.succeed(RunnerClient, disabled); + }), +); diff --git a/apps/server/src/runner/hub/RunnerDelivery.ts b/apps/server/src/runner/hub/RunnerDelivery.ts new file mode 100644 index 000000000000..0216364a8e0c --- /dev/null +++ b/apps/server/src/runner/hub/RunnerDelivery.ts @@ -0,0 +1,50 @@ +/** + * RunnerDelivery - starts runner event delivery once hub ingestion runs, and + * acknowledges delivered events only after ingestion has drained them. + * + * Prototype semantics: at-least-once across hub restarts with a window of one + * ack interval (an event ingested but not yet acked is re-delivered after a + * crash). Production should persist the runner cursor in the same database + * transaction as the projection writes the event produced. + * + * @module runner/hub/RunnerDelivery + */ +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import { ProviderRuntimeIngestionService } from "../../orchestration/Services/ProviderRuntimeIngestion.ts"; +import { forkParked } from "../../serverActivation.ts"; +import { RunnerClient } from "./RunnerClient.ts"; + +const ACK_INTERVAL = "500 millis"; + +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + const runner = yield* RunnerClient; + if (!runner.enabled) return; + const ingestion = yield* ProviderRuntimeIngestionService; + yield* forkParked( + Effect.gen(function* () { + // Ingestion's stream consumers are parked at the same activation + // gate; give them a moment to subscribe before replay starts. + yield* Effect.sleep("500 millis"); + yield* runner.startDelivery; + return yield* Effect.gen(function* () { + const { deliveredSequence, ackedSequence } = yield* runner.deliveryState; + if (deliveredSequence <= ackedSequence) return; + // Two async hops separate delivery from the ingestion queue + // (adapter stream -> ProviderService PubSub -> ingestion worker). + yield* Effect.sleep("50 millis"); + yield* ingestion.drain; + yield* runner.ackDelivered(deliveredSequence); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("runner delivery ack failed", { cause: String(cause) }), + ), + Effect.andThen(Effect.sleep(ACK_INTERVAL)), + Effect.forever, + ); + }), + ); + }), +); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 92b0020dc4c1..d99d9f2cd4d0 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -134,6 +134,10 @@ import * as NetService from "@t3tools/shared/Net"; import * as RelayClient from "@t3tools/shared/relayClient"; import { disableTailscaleServe, ensureTailscaleServe } from "@t3tools/tailscale"; import { forkParked, ServerActivation } from "./serverActivation.ts"; +import { CheckoutGitProbe, localCheckoutGitProbe } from "./git/CheckoutGitProbe.ts"; +import * as HubLayers from "./runner/hub/HubLayers.ts"; +import * as RunnerClient from "./runner/hub/RunnerClient.ts"; +import * as RunnerDelivery from "./runner/hub/RunnerDelivery.ts"; // MCP handoff thread IDs include escaped provenance and can exceed find-my-way's // 100-character default for one path segment. @@ -147,7 +151,7 @@ export const HTTP_ROUTER_CONFIG = { // those finalizers get a chance to run. const HTTP_PREEMPTIVE_SHUTDOWN_GRACE_MS = 0; const ResourceAttributionLayerLive = ResourceAttribution.layer; -const ApplicationObservabilityLive = ObservabilityLive.pipe( +export const ApplicationObservabilityLive = ObservabilityLive.pipe( Layer.provideMerge(ResourceAttributionLayerLive), ); @@ -163,7 +167,22 @@ const PtyAdapterLive = Layer.unwrap( }), ); -const ServerSettingsLayerLive = ServerSettings.layer.pipe( +/** + * Picks the checkout-bound implementation (`local`) or the hub substitute + * (`hub`) depending on whether this server delegates to a runner. + */ +const byRunnerMode = ( + local: Layer.Layer, + hub: Layer.Layer, +): Layer.Layer => + Layer.unwrap( + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + return (config.runnerUrl ? hub : local) as Layer.Layer; + }), + ); + +export const ServerSettingsLayerLive = ServerSettings.layer.pipe( Layer.provide(ServerSecretStore.layer), Layer.provideMerge(SqlitePersistenceLayerLive), ); @@ -184,7 +203,7 @@ const HostPowerMonitorLayerLive = HostPowerMonitor.layer.pipe( Layer.provide(DesktopTelemetryReceiverLayerLive), ); -const BackgroundLayerLive = BackgroundPolicy.layer.pipe( +export const BackgroundLayerLive = BackgroundPolicy.layer.pipe( Layer.provide(HostPowerMonitorLayerLive), Layer.provideMerge(ServerSettingsLayerLive), ); @@ -213,7 +232,7 @@ const AgentAwarenessRelayLayerLive = Layer.unwrap( }), ); -const HttpServerLive = Layer.unwrap( +export const HttpServerLive = Layer.unwrap( Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; if (typeof Bun !== "undefined") { @@ -260,7 +279,7 @@ const HttpServerLive = Layer.unwrap( }), ); -const PlatformServicesLive = Layer.unwrap( +export const PlatformServicesLive = Layer.unwrap( Effect.gen(function* () { if (typeof Bun !== "undefined") { const { layer } = yield* Effect.promise(() => import("@effect/platform-bun/BunServices")); @@ -272,7 +291,13 @@ const PlatformServicesLive = Layer.unwrap( }), ); +const CheckoutGitProbeLayerLive = byRunnerMode( + Layer.succeed(CheckoutGitProbe, localCheckoutGitProbe), + HubLayers.remoteCheckoutGitProbeLayer, +); + const ReactorLayerLive = Layer.empty.pipe( + Layer.provideMerge(RunnerDelivery.layer), Layer.provideMerge(OrchestrationReactorLive), Layer.provideMerge(ProviderRuntimeIngestionLive), Layer.provideMerge(ProviderCommandReactorLive), @@ -281,6 +306,7 @@ const ReactorLayerLive = Layer.empty.pipe( Layer.provideMerge(ThreadSettlementReactor.layer), Layer.provideMerge(AgentAwarenessRelayLayerLive), Layer.provideMerge(RuntimeReceiptBusLive), + Layer.provideMerge(CheckoutGitProbeLayerLive), ); const ProviderSessionDirectoryLayerLive = ProviderSessionDirectoryLive.pipe( @@ -300,7 +326,7 @@ const ProviderLayerLive = ProviderServiceLive.pipe( const PersistenceLayerLive = Layer.empty.pipe(Layer.provideMerge(SqlitePersistenceLayerLive)); -const VcsDriverRegistryLayerLive = VcsDriverRegistry.layer.pipe( +export const VcsDriverRegistryLayerLive = VcsDriverRegistry.layer.pipe( Layer.provide(VcsProjectConfig.layer), ); @@ -356,12 +382,22 @@ const VcsLayerLive = Layer.empty.pipe( Layer.provideMerge(ReviewLayerLive), Layer.provideMerge(GrokReviewLayerLive), Layer.provideMerge(SourceControlRepositoryServiceLayerLive), - Layer.provideMerge(VcsStatusBroadcaster.layer.pipe(Layer.provide(GitWorkflowLayerLive))), + Layer.provideMerge( + byRunnerMode( + VcsStatusBroadcaster.layer.pipe(Layer.provide(GitWorkflowLayerLive)), + HubLayers.hubVcsStatusBroadcasterLayer, + ), + ), ); const CheckpointingLayerLive = Layer.empty.pipe( Layer.provideMerge(CheckpointDiffQuery.layer), - Layer.provideMerge(CheckpointStore.layer.pipe(Layer.provide(VcsDriverRegistryLayerLive))), + Layer.provideMerge( + byRunnerMode( + CheckpointStore.layer.pipe(Layer.provide(VcsDriverRegistryLayerLive)), + HubLayers.remoteCheckpointStoreLayer, + ), + ), ); const PortScannerLayerLive = PortScanner.layer.pipe(Layer.provide(ProcessRunner.layer)); @@ -384,8 +420,8 @@ const WorkspaceFileSystemLayerLive = WorkspaceFileSystem.layer.pipe( ); const WorkspaceLayerLive = Layer.mergeAll( - WorkspacePaths.layer, - WorkspaceEntriesLayerLive, + byRunnerMode(WorkspacePaths.layer, HubLayers.remoteWorkspacePathsLayer), + byRunnerMode(WorkspaceEntriesLayerLive, HubLayers.hubWorkspaceEntriesLayer), WorkspaceFileSystemLayerLive, ); @@ -456,7 +492,9 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( Layer.provideMerge(OpenCodeRuntime.OpenCodeRuntimeLive), Layer.provideMerge(WorkspaceLayerLive), Layer.provideMerge(ProjectFaviconResolverLayerLive), - Layer.provideMerge(RepositoryIdentityResolver.layer), + Layer.provideMerge( + byRunnerMode(RepositoryIdentityResolver.layer, HubLayers.hubRepositoryIdentityResolverLayer), + ), Layer.provideMerge(ServerEnvironmentLayerLive), Layer.provideMerge(AuthLayerLive), Layer.provideMerge(ServerSecretStore.layer), @@ -735,6 +773,7 @@ export const makeServerLayer = Layer.unwrap( return serverApplicationLayer.pipe( Layer.provideMerge(runtimeServicesLive), + Layer.provideMerge(RunnerClient.layer), Layer.provide(activationLayer), Layer.provideMerge(serverRelayBrokerTracingLayer), Layer.provideMerge(HttpServerLive), diff --git a/prototype-2/ctl.sh b/prototype-2/ctl.sh new file mode 100755 index 000000000000..9727858da18a --- /dev/null +++ b/prototype-2/ctl.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# ctl.sh start-hub|stop-hub|start-runner|stop-runner|status +# Stops only the process listening on our port after confirming its cwd. +set -uo pipefail +cd /tmp/proto-runner +ms() { echo $(( $(date +%s%N) / 1000000 )); } +port_pid() { ss -H -ltnp "sport = :$1" | grep -o 'pid=[0-9]*' | head -1 | cut -d= -f2; } +wait_http() { # port path + local t0=$(ms) + for i in $(seq 1 240); do + code=$(curl -s --max-time 2 -o /dev/null -w '%{http_code}' -H "authorization: Bearer $(cat hub-token 2>/dev/null)" "http://127.0.0.1:$1$2" 2>/dev/null) + [ "$code" = "200" ] && { echo "ready in $(( $(ms) - t0 )) ms"; return 0; } + sleep 0.25 + done + echo "not ready"; return 1 +} +stop_port() { # port expected-cwd + local pid=$(port_pid $1) + [ -z "$pid" ] && { echo "nothing on :$1"; return 0; } + [ "$(readlink /proc/$pid/cwd)" = "$2" ] || { echo "refusing: pid $pid cwd $(readlink /proc/$pid/cwd)"; return 1; } + local t0=$(ms); kill -TERM "$pid" + for i in $(seq 1 100); do kill -0 "$pid" 2>/dev/null || break; sleep 0.1; done + echo "stopped pid $pid on :$1 in $(( $(ms) - t0 )) ms" +} +case "$1" in + start-runner) setsid -f ./start-runner.sh /dev/null 2>&1; for i in $(seq 1 120); do [ -n "$(port_pid 4412)" ] && break; sleep 0.25; done; echo "runner pid $(port_pid 4412)";; + start-hub) t0=$(ms); setsid -f ./start-hub.sh /dev/null 2>&1; wait_http 4411 /api/orchestration/shell; echo "hub pid $(port_pid 4411) (start->ready $(( $(ms) - t0 )) ms)";; + stop-hub) stop_port 4411 /tmp/proto-runner/hub;; + stop-runner) stop_port 4412 /tmp/proto-runner/checkout;; + status) ss -H -ltnp | grep -E ":(4411|4412) " || echo "none";; +esac diff --git a/prototype-2/evidence/control-interrupt.out b/prototype-2/evidence/control-interrupt.out new file mode 100644 index 000000000000..2eedc409bc95 --- /dev/null +++ b/prototype-2/evidence/control-interrupt.out @@ -0,0 +1,42 @@ +[19:28:23.900] thread.turn.start accepted in 35 ms +[19:28:23.937] message user: "Count from 1 to 1500, writing each number on its own line as plain text. Do not use any tools." +[19:28:28.041] interrupting turn f0ad36d1-d1e6-464b-b924-f808ba4eeb96 +[19:28:28.471] activity context-window.updated: Context window updated +[19:28:28.471] activity checkpoint.captured: Checkpoint captured +[19:28:28.471] activity checkpoint.captured: Checkpoint captured +[19:28:28.471] message assistant: "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n21\n22\n23\n24\n25\n26\n27\n28\n29\n30\n31\n32\n33\n34\n35\n36\n37\n38\n39\n40\n41\n42\n43\n44\n45\n46\n47\n48\n49\n50\n51\n52\n53\n54\n55\n56\n5" +[19:28:28.471] turn f0ad36d1-d1e6-464b-b924-f808ba4eeb96 settled: completed after 4570 ms (first assistant text after 4570 ms) +{ + "latestTurn": { + "turnId": "f0ad36d1-d1e6-464b-b924-f808ba4eeb96", + "state": "completed", + "requestedAt": "2026-09-25T19:28:23.872Z", + "startedAt": "2026-09-25T19:28:23.872Z", + "completedAt": "2026-09-25T19:28:28.062Z", + "assistantMessageId": "assistant:f0ad36d1-d1e6-464b-b924-f808ba4eeb96" + }, + "session": { + "status": "stopped", + "activeTurnId": null, + "lastError": null + }, + "messageCount": 4, + "activityCount": 14, + "checkpoints": [ + { + "turnCount": 1, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWVhYWU4ZjVhLWFhOGUtNGY0ZC1iYzdhLTdiNmExNTNhNjEyNw/turn/1", + "files": [ + "notes.txt +1/-0" + ] + }, + { + "turnCount": 2, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWVhYWU4ZjVhLWFhOGUtNGY0ZC1iYzdhLTdiNmExNTNhNjEyNw/turn/2", + "files": [] + } + ], + "pendingApprovals": [] +} diff --git a/prototype-2/evidence/control-turn1.out b/prototype-2/evidence/control-turn1.out new file mode 100644 index 000000000000..796f70c365b0 --- /dev/null +++ b/prototype-2/evidence/control-turn1.out @@ -0,0 +1,49 @@ +[19:28:14.372] thread.turn.start accepted in 32 ms +[19:28:14.397] message user: "Append a new line 'edited by the remote runner' to the end of notes.txt in the current directory. Do not touch other files. Reply with just: done" +[19:28:16.883] activity tool.started: Tool call started +[19:28:17.295] activity tool.completed: Tool call +[19:28:17.295] activity tool.updated: Tool call +[19:28:17.295] activity context-window.updated: Context window updated +[19:28:18.527] activity tool.started: File change started +[19:28:19.346] activity tool.completed: File change +[19:28:19.347] activity context-window.updated: Context window updated +[19:28:20.164] activity tool.started: File change started +[19:28:20.983] activity tool.updated: File change +[19:28:20.983] activity approval.requested: File-change approval requested +[19:28:20.983] activity context-window.updated: Context window updated +[19:28:20.983] approving ac2b0e64-0b8a-49f5-a365-0802d8d4c90a (File-change approval requested: Edit: {"file_path":"/tmp/proto-runner/control-checkout/notes.txt","old_string":") +[19:28:20.996] approval dispatched in 13 ms +[19:28:21.406] activity approval.resolved: Approval resolved +[19:28:21.406] activity tool.completed: File change +[19:28:22.261] activity context-window.updated: Context window updated +[19:28:22.261] activity checkpoint.captured: Checkpoint captured +[19:28:22.261] message assistant: "done" +[19:28:22.261] turn 4e8d8ef0-ec2c-4dc9-a221-fe36aaacc587 settled: completed after 7889 ms (first assistant text after 7889 ms) +{ + "latestTurn": { + "turnId": "4e8d8ef0-ec2c-4dc9-a221-fe36aaacc587", + "state": "completed", + "requestedAt": "2026-09-25T19:28:14.346Z", + "startedAt": "2026-09-25T19:28:14.346Z", + "completedAt": "2026-09-25T19:28:21.959Z", + "assistantMessageId": "assistant:12925a11-744f-4c46-875f-b3e3a101cad6" + }, + "session": { + "status": "ready", + "activeTurnId": null, + "lastError": null + }, + "messageCount": 2, + "activityCount": 11, + "checkpoints": [ + { + "turnCount": 1, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWVhYWU4ZjVhLWFhOGUtNGY0ZC1iYzdhLTdiNmExNTNhNjEyNw/turn/1", + "files": [ + "notes.txt +1/-0" + ] + } + ], + "pendingApprovals": [] +} diff --git a/prototype-2/evidence/diff-0-1.out b/prototype-2/evidence/diff-0-1.out new file mode 100644 index 000000000000..98e1dc6693a3 --- /dev/null +++ b/prototype-2/evidence/diff-0-1.out @@ -0,0 +1,7 @@ +[19:17:31.083] getTurnDiff via hub in 121 ms +{ + "fromTurnCount": 0, + "toTurnCount": 1, + "threadId": "thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967", + "diff": "diff --git a/notes.txt b/notes.txt\nindex f31bc06..77957df 100644\n--- a/notes.txt\n+++ b/notes.txt\n@@ -1 +1,2 @@\n hello from the runner\n+edited by the remote runner\n" +} diff --git a/prototype-2/evidence/hub-delivery-log.txt b/prototype-2/evidence/hub-delivery-log.txt new file mode 100644 index 000000000000..fb5ccb390aed --- /dev/null +++ b/prototype-2/evidence/hub-delivery-log.txt @@ -0,0 +1,350 @@ +[19:15:29.359] INFO (#45): runner connected + bootId: 'boot-87901cd8-0c08-4338-9996-f7b99f3e14c9', + headSequence: 0, + runnerAckedSequence: 0, +[19:15:30.356] INFO (#203): runner event subscription opened + { afterSequence: 0 } +[19:20:05.283] INFO (#46): runner connected + bootId: 'boot-4e155bbc-f7b5-484c-8009-4922a3ae942f', + headSequence: 28, + runnerAckedSequence: 28, +[19:20:06.058] INFO (#90): runner event subscription opened + { afterSequence: 28 } +[19:22:29.856] INFO (#243): runner events acknowledged + throughSequence: 31, + deliveredCount: 3, + duplicateCount: 0, + connectCount: 1 +[19:22:30.414] INFO (#243): runner events acknowledged + throughSequence: 35, + deliveredCount: 7, + duplicateCount: 0, + connectCount: 1 +[19:22:32.993] INFO (#243): runner events acknowledged + throughSequence: 40, + deliveredCount: 12, + duplicateCount: 0, + connectCount: 1 +[19:23:22.101] INFO (#243): runner events acknowledged + throughSequence: 43, + deliveredCount: 15, + duplicateCount: 0, + connectCount: 1 +[19:23:22.673] INFO (#243): runner events acknowledged + throughSequence: 50, + deliveredCount: 22, + duplicateCount: 0, + connectCount: 1 +[19:23:24.226] INFO (#243): runner events acknowledged + throughSequence: 52, + deliveredCount: 24, + duplicateCount: 0, + connectCount: 1 +[19:23:24.782] INFO (#243): runner events acknowledged + throughSequence: 60, + deliveredCount: 32, + duplicateCount: 0, + connectCount: 1 +[19:23:25.836] INFO (#243): runner events acknowledged + throughSequence: 68, + deliveredCount: 40, + duplicateCount: 0, + connectCount: 1 +[19:23:26.390] INFO (#243): runner events acknowledged + throughSequence: 70, + deliveredCount: 42, + duplicateCount: 0, + connectCount: 1 +[19:23:54.470] INFO (#243): runner events acknowledged + throughSequence: 73, + deliveredCount: 45, + duplicateCount: 0, + connectCount: 1 +[19:23:55.524] INFO (#243): runner events acknowledged + throughSequence: 78, + deliveredCount: 52, + duplicateCount: 0, + connectCount: 1 +[19:23:56.077] INFO (#243): runner events acknowledged + throughSequence: 99, + deliveredCount: 73, + duplicateCount: 0, + connectCount: 1 +[19:23:56.733] INFO (#243): runner events acknowledged + throughSequence: 120, + deliveredCount: 93, + duplicateCount: 0, + connectCount: 1 +[19:23:57.387] INFO (#243): runner events acknowledged + throughSequence: 148, + deliveredCount: 122, + duplicateCount: 0, + connectCount: 1 +[19:23:57.943] INFO (#243): runner events acknowledged + throughSequence: 170, + deliveredCount: 143, + duplicateCount: 0, + connectCount: 1 +[19:23:58.498] INFO (#243): runner events acknowledged + throughSequence: 191, + deliveredCount: 164, + duplicateCount: 0, + connectCount: 1 +[19:23:59.054] INFO (#243): runner events acknowledged + throughSequence: 212, + deliveredCount: 186, + duplicateCount: 0, + connectCount: 1 +[19:23:59.607] INFO (#243): runner events acknowledged + throughSequence: 228, + deliveredCount: 200, + duplicateCount: 0, + connectCount: 1 +[19:24:09.684] INFO (#243): runner events acknowledged + throughSequence: 232, + deliveredCount: 204, + duplicateCount: 0, + connectCount: 1 +[19:24:11.239] INFO (#243): runner events acknowledged + throughSequence: 234, + deliveredCount: 206, + duplicateCount: 0, + connectCount: 1 +[19:24:11.793] INFO (#243): runner events acknowledged + throughSequence: 239, + deliveredCount: 211, + duplicateCount: 0, + connectCount: 1 +[19:24:12.848] INFO (#243): runner events acknowledged + throughSequence: 244, + deliveredCount: 216, + duplicateCount: 0, + connectCount: 1 +[19:24:23.417] INFO (#243): runner events acknowledged + throughSequence: 247, + deliveredCount: 219, + duplicateCount: 0, + connectCount: 1 +[19:24:24.470] INFO (#243): runner events acknowledged + throughSequence: 256, + deliveredCount: 230, + duplicateCount: 0, + connectCount: 1 +[19:24:25.022] INFO (#243): runner events acknowledged + throughSequence: 277, + deliveredCount: 251, + duplicateCount: 0, + connectCount: 1 +[19:24:25.576] INFO (#243): runner events acknowledged + throughSequence: 298, + deliveredCount: 272, + duplicateCount: 0, + connectCount: 1 +[19:24:26.128] INFO (#243): runner events acknowledged + throughSequence: 319, + deliveredCount: 293, + duplicateCount: 0, + connectCount: 1 +[19:24:26.682] INFO (#243): runner events acknowledged + throughSequence: 340, + deliveredCount: 314, + duplicateCount: 0, + connectCount: 1 +[19:24:27.234] INFO (#243): runner events acknowledged + throughSequence: 361, + deliveredCount: 335, + duplicateCount: 0, + connectCount: 1 +[19:24:27.787] INFO (#243): runner events acknowledged + throughSequence: 368, + deliveredCount: 340, + duplicateCount: 0, + connectCount: 1 +[19:24:52.865] INFO (#243): runner events acknowledged + throughSequence: 371, + deliveredCount: 343, + duplicateCount: 0, + connectCount: 1 +[19:24:53.920] INFO (#243): runner events acknowledged + throughSequence: 375, + deliveredCount: 347, + duplicateCount: 0, + connectCount: 1 +[19:24:56.975] INFO (#243): runner events acknowledged + throughSequence: 379, + deliveredCount: 351, + duplicateCount: 0, + connectCount: 1 +[19:25:26.826] INFO (#46): runner connected + bootId: 'boot-4e155bbc-f7b5-484c-8009-4922a3ae942f', + headSequence: 395, + runnerAckedSequence: 379, +[19:25:27.725] INFO (#90): runner event subscription opened + { afterSequence: 379 } +[19:25:28.285] INFO (#240): runner events acknowledged + throughSequence: 395, + deliveredCount: 16, + duplicateCount: 0, + connectCount: 1 +[19:25:28.865] INFO (#240): runner events acknowledged + throughSequence: 400, + deliveredCount: 21, + duplicateCount: 0, + connectCount: 1 +[19:25:29.927] INFO (#240): runner events acknowledged + throughSequence: 402, + deliveredCount: 23, + duplicateCount: 0, + connectCount: 1 +[19:25:30.555] INFO (#240): runner events acknowledged + throughSequence: 403, + deliveredCount: 26, + duplicateCount: 0, + connectCount: 1 +[19:25:31.117] INFO (#240): runner events acknowledged + throughSequence: 405, + deliveredCount: 26, + duplicateCount: 0, + connectCount: 1 +[19:25:53.280] WARN (#46): runner connection ended +[19:25:54.284] WARN (#46): runner connection ended +[19:25:55.288] WARN (#46): runner connection ended +[19:25:56.292] WARN (#46): runner connection ended +[19:25:57.311] INFO (#46): runner connected + bootId: 'boot-1b34a7ce-30b4-418a-8d97-7adb8eecf074', + headSequence: 405, + runnerAckedSequence: 405, +[19:25:57.311] INFO (#756): runner event subscription opened + { afterSequence: 405 } +[19:26:05.761] INFO (#240): runner events acknowledged + throughSequence: 408, + deliveredCount: 29, + duplicateCount: 0, + connectCount: 2 +[19:26:06.819] INFO (#240): runner events acknowledged + throughSequence: 412, + deliveredCount: 33, + duplicateCount: 0, + connectCount: 2 +[19:26:08.403] INFO (#240): runner events acknowledged + throughSequence: 418, + deliveredCount: 39, + duplicateCount: 0, + connectCount: 2 +[19:26:08.963] INFO (#240): runner events acknowledged + throughSequence: 420, + deliveredCount: 41, + duplicateCount: 0, + connectCount: 2 +[19:27:11.155] WARN (#46): runner connection ended +[19:27:12.178] INFO (#46): runner connected + bootId: 'boot-3d1e2e6c-b8d2-496f-8cdf-cb189cf657d2', + headSequence: 420, + runnerAckedSequence: 420, +[19:27:12.178] INFO (#1047): runner event subscription opened + { afterSequence: 420 } +[19:27:27.230] INFO (#240): runner events acknowledged + throughSequence: 423, + deliveredCount: 44, + duplicateCount: 0, + connectCount: 3 +[19:27:27.786] INFO (#240): runner events acknowledged + throughSequence: 427, + deliveredCount: 48, + duplicateCount: 0, + connectCount: 3 +[19:27:29.342] INFO (#240): runner events acknowledged + throughSequence: 431, + deliveredCount: 52, + duplicateCount: 0, + connectCount: 3 +[19:27:31.416] WARN (#46): runner connection ended +[19:27:32.419] WARN (#46): runner connection ended +[19:27:33.422] WARN (#46): runner connection ended +[19:27:34.441] INFO (#46): runner connected + bootId: 'boot-0d427cb0-d9a9-4c7a-9578-d5ca7b9d0bf4', + headSequence: 435, + runnerAckedSequence: 431, +[19:27:34.441] INFO (#1312): runner event subscription opened + { afterSequence: 431 } +[19:27:34.901] INFO (#240): runner events acknowledged + throughSequence: 435, + deliveredCount: 56, + duplicateCount: 0, + connectCount: 4 +[19:27:36.458] INFO (#240): runner events acknowledged + throughSequence: 438, + deliveredCount: 59, + duplicateCount: 0, + connectCount: 4 +[19:27:37.025] INFO (#240): runner events acknowledged + throughSequence: 440, + deliveredCount: 63, + duplicateCount: 0, + connectCount: 4 +[19:27:37.579] INFO (#240): runner events acknowledged + throughSequence: 442, + deliveredCount: 63, + duplicateCount: 0, + connectCount: 4 +[19:27:38.636] INFO (#240): runner events acknowledged + throughSequence: 449, + deliveredCount: 70, + duplicateCount: 0, + connectCount: 4 +[19:27:39.690] INFO (#240): runner events acknowledged + throughSequence: 450, + deliveredCount: 71, + duplicateCount: 0, + connectCount: 4 +[19:27:40.258] INFO (#240): runner events acknowledged + throughSequence: 451, + deliveredCount: 74, + duplicateCount: 0, + connectCount: 4 +[19:27:40.814] INFO (#240): runner events acknowledged + throughSequence: 458, + deliveredCount: 79, + duplicateCount: 0, + connectCount: 4 +[19:27:41.381] INFO (#240): runner events acknowledged + throughSequence: 460, + deliveredCount: 82, + duplicateCount: 0, + connectCount: 4 +[19:27:41.937] INFO (#240): runner events acknowledged + throughSequence: 463, + deliveredCount: 84, + duplicateCount: 0, + connectCount: 4 +[19:31:41.091] INFO (#46): runner connected + bootId: 'boot-e6ca177c-997c-49a2-aecf-1acdba8c445f', + headSequence: 464, + runnerAckedSequence: 463, +[19:31:41.895] INFO (#90): runner event subscription opened + { afterSequence: 463 } +[19:31:42.452] INFO (#242): runner events acknowledged + throughSequence: 464, + deliveredCount: 1, + duplicateCount: 0, + connectCount: 1 +[19:31:50.521] INFO (#242): runner events acknowledged + throughSequence: 467, + deliveredCount: 4, + duplicateCount: 0, + connectCount: 1 +[19:31:51.076] INFO (#242): runner events acknowledged + throughSequence: 471, + deliveredCount: 8, + duplicateCount: 0, + connectCount: 1 +[19:31:51.632] INFO (#242): runner events acknowledged + throughSequence: 472, + deliveredCount: 9, + duplicateCount: 0, + connectCount: 1 +[19:31:52.686] INFO (#242): runner events acknowledged + throughSequence: 477, + deliveredCount: 14, + duplicateCount: 0, + connectCount: 1 diff --git a/prototype-2/evidence/hub-restart-test.out b/prototype-2/evidence/hub-restart-test.out new file mode 100644 index 000000000000..3aac3b705790 --- /dev/null +++ b/prototype-2/evidence/hub-restart-test.out @@ -0,0 +1,11 @@ +outbox head before turn: 368 +[19:24:56.968] command started on runner; stopping hub +stopped pid 3600897 on :4411 in 107 ms +[19:24:57.126] hub down; outbox head 379 +[19:25:20.363] approval requested while hub down: +395 19:25:20.148 request.opened file_change_approval +[19:25:20.590] outbox head 395; restarting hub +ready in 8574 ms +hub pid 3609589 (start->ready 8675 ms) +scenario exit 0 +events produced by the runner while the hub was down: 379..405 diff --git a/prototype-2/evidence/hub-strace-all-lifetimes.txt b/prototype-2/evidence/hub-strace-all-lifetimes.txt new file mode 100644 index 000000000000..aab015070205 --- /dev/null +++ b/prototype-2/evidence/hub-strace-all-lifetimes.txt @@ -0,0 +1,32 @@ +3594265 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594266, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3594265 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594267, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3594265 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594268, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3594265 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594269, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3594265 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594270, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3594186 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594265, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3594186 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594279, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3594186 --- SIGTERM {si_signo=SIGTERM, si_code=SI_USER, si_pid=3600359, si_uid=1000} --- +3601072 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601073, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3601072 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601074, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3601072 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601075, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3601072 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601076, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3601072 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601077, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3600897 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601072, si_uid=1000, si_status=0, si_utime=0, si_stime=1 /* 0.01 s */} --- +3600897 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601082, si_uid=1000, si_status=0, si_utime=0, si_stime=1 /* 0.01 s */} --- +3600897 --- SIGTERM {si_signo=SIGTERM, si_code=SI_USER, si_pid=3608476, si_uid=1000} --- +3609858 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609860, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3609858 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609862, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3609858 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609864, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3609858 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609865, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3609858 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609866, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3609589 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609858, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3609589 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609885, si_uid=1000, si_status=0, si_utime=0, si_stime=1 /* 0.01 s */} --- +3609589 --- SIGTERM {si_signo=SIGTERM, si_code=SI_USER, si_pid=3616263, si_uid=1000} --- +3622473 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622476, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3622473 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622478, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3622473 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622480, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3622473 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622481, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3622473 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622483, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3622218 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622473, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- +3622218 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622493, si_uid=1000, si_status=0, si_utime=0, si_stime=1 /* 0.01 s */} --- +3622218 --- SIGTERM {si_signo=SIGTERM, si_code=SI_USER, si_pid=3623549, si_uid=1000} --- diff --git a/prototype-2/evidence/outbox-summary.txt b/prototype-2/evidence/outbox-summary.txt new file mode 100644 index 000000000000..edd189c15a6a --- /dev/null +++ b/prototype-2/evidence/outbox-summary.txt @@ -0,0 +1,14 @@ +Runner outbox (T3CODE_RUNNER_OUTBOX_KEEP=1 from the second runner boot on) +sequences 29 .. 478 = 450 events; runner boots: 5 +(session seq 29-478 boot#2 {"session.started":7,"session.configured":9,"session.state.changed":9,"thread.started":7,"session.exited":5,"task.completed":2,"task.updated":1} +646f7959 seq 32-39 boot#2 {"turn.started":1,"session.configured":1,"session.state.changed":1,"item.started":1,"item.completed":1,"thread.token-usage.updated":1,"turn.completed":1} +f4f2e0ef seq 46-48 boot#2 {"turn.started":1,"session.configured":1,"turn.completed":1} +a1aa7576 seq 51-70 boot#2 {"turn.started":1,"item.started":1,"item.updated":2,"thread.token-usage.updated":3,"account.rate-limits.updated":1,"task.started":1,"content.delta":7,"item.completed":2,"session.state.changed":1,"turn.completed":1} +91f83731 seq 71-228 boot#2 {"turn.started":1,"session.configured":1,"session.state.changed":1,"content.delta":151,"item.completed":1,"thread.token-usage.updated":2,"turn.completed":1} +505bb185 seq 233-244 boot#2 {"turn.started":1,"item.started":1,"item.updated":2,"thread.token-usage.updated":3,"item.completed":2,"session.state.changed":1,"content.delta":1,"turn.completed":1} +01b724b4 seq 245-367 boot#2 {"turn.started":1,"session.configured":1,"session.state.changed":1,"content.delta":117,"item.completed":1,"thread.token-usage.updated":1,"turn.completed":1} +e3457ec4 seq 372-405 boot#2 {"turn.started":1,"session.configured":1,"session.state.changed":4,"item.started":3,"item.updated":6,"thread.token-usage.updated":5,"account.rate-limits.updated":1,"task.started":1,"task.completed":1,"content.delta":3,"item.completed":4,"request.opened":1,"request.resolved":1,"turn.completed":1} +3aec0919 seq 409-420 boot#3 {"turn.started":1,"session.configured":1,"session.state.changed":1,"content.delta":3,"item.completed":1,"thread.token-usage.updated":2,"account.rate-limits.updated":1,"turn.completed":1} +b53e0583 seq 424-434 boot#4 {"turn.started":1,"session.configured":1,"session.state.changed":1,"item.started":1,"item.updated":1,"thread.token-usage.updated":2,"account.rate-limits.updated":1,"item.completed":1,"turn.completed":1} +5361dcda seq 439-463 boot#5 {"turn.started":1,"session.configured":1,"session.state.changed":3,"item.started":2,"item.updated":4,"thread.token-usage.updated":4,"account.rate-limits.updated":1,"item.completed":3,"request.opened":1,"request.resolved":1,"content.delta":2,"turn.completed":1} +dcc58fc1 seq 468-477 boot#6 {"turn.started":1,"session.configured":1,"session.state.changed":1,"account.rate-limits.updated":1,"content.delta":1,"item.completed":1,"thread.token-usage.updated":2,"turn.completed":1} diff --git a/prototype-2/evidence/runner-restart-midturn-test.out b/prototype-2/evidence/runner-restart-midturn-test.out new file mode 100644 index 000000000000..341188389793 --- /dev/null +++ b/prototype-2/evidence/runner-restart-midturn-test.out @@ -0,0 +1,17 @@ +outbox head before turn: 420 +[19:27:29.383] command running on runner; stopping runner +stopped pid 3613078 on :4412 in 412 ms +[19:27:29.812] events outboxed during drain: +432 19:27:29.407 item.completed command_execution +433 19:27:29.407 thread.token-usage.updated +434 19:27:29.407 turn.completed interrupted +435 19:27:29.408 session.exited +[19:27:31.860] starting runner +runner pid 3613921 +scenario exit 0 +[19:27:36.184] follow-up turn +follow-up exit 0 +hello from the runner +edited by the remote runner +after hub restart +after runner restart diff --git a/prototype-2/evidence/runner-restart-test.out b/prototype-2/evidence/runner-restart-test.out new file mode 100644 index 000000000000..849b18cd2f1c --- /dev/null +++ b/prototype-2/evidence/runner-restart-test.out @@ -0,0 +1,16 @@ +[19:25:51.248] stopping runner +stopped pid 3600783 on :4412 in 107 ms +[19:25:54.376] starting runner +runner pid 3610610 +791- } +792:[19:25:56.292] WARN (#46): runner connection ended +793- { +794- cause: 'Cause([Fail(~effect/rpc/RpcClientError: SocketOpenError: An error occurred during Open)])' +795- } +796:[19:25:57.311] INFO (#46): runner connected +797- { +798- runnerId: 'runner-f44d1a25-700d-4f1d-859c-85956c39170f', +799- bootId: 'boot-1b34a7ce-30b4-418a-8d97-7adb8eecf074', +-- +804:[19:25:57.311] INFO (#756): runner event subscription opened +805- { afterSequence: 405 } diff --git a/prototype-2/evidence/runner-rpc-log.txt b/prototype-2/evidence/runner-rpc-log.txt new file mode 100644 index 000000000000..1e0cd9fbb118 --- /dev/null +++ b/prototype-2/evidence/runner-rpc-log.txt @@ -0,0 +1,258 @@ +75:[19:22:29.478] INFO (#149): runner rpc startSession +76- { +77- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +78- cwd: '/tmp/proto-runner/checkout', +79- runtimeMode: 'approval-required', +80- resumeCursor: { +81- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +82- resume: '202a75b8-d54d-4c76-8507-3cfe0c931870', +83- resumeSessionAt: '1c0876b2-ef23-4bde-bd5c-f78bc34e859c', +89:[19:22:30.060] INFO (#188): runner rpc sendTurn +90- { +91- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +92- ok: true, +93- ms: 464 +94- } +95:[19:22:32.715] INFO (#236): runner rpc interruptTurn +96- { +97- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +98- turnId: undefined, +99- ok: true, +100- ms: 3 +101- } +102:[19:22:32.881] INFO (#270): runner rpc checkpoint.capture +103- { +104- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2', +105- ok: true, +106- ms: 42 +107- } +108:[19:22:33.015] INFO (#316): runner rpc checkpoint.capture +109- { +110- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2', +111- ok: true, +112- ms: 57 +113- } +114:[19:23:21.853] INFO (#371): runner rpc startSession +115- { +116- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +117- cwd: '/tmp/proto-runner/checkout', +118- runtimeMode: 'approval-required', +119- resumeCursor: { +120- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +121- resume: '202a75b8-d54d-4c76-8507-3cfe0c931870', +122- resumeSessionAt: '1c0876b2-ef23-4bde-bd5c-f78bc34e859c', +128:[19:23:22.380] INFO (#408): runner rpc sendTurn +129- { +130- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +131- ok: true, +132- ms: 457 +133- } +134:[19:23:22.588] INFO (#462): runner rpc checkpoint.capture +135- { +136- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/3', +137- ok: true, +138- ms: 57 +139- } +140:[19:23:25.992] INFO (#596): runner rpc checkpoint.capture +141- { +142- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/4', +143- ok: true, +144- ms: 34 +145- } +146:[19:23:53.948] INFO (#659): runner rpc sendTurn +147- { +148- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +149- ok: true, +150- ms: 20 +151- } +152:[19:23:59.614] INFO (#1057): runner rpc checkpoint.capture +153- { +154- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/5', +155- ok: true, +156- ms: 58 +157- } +158:[19:24:12.740] INFO (#1199): runner rpc checkpoint.capture +159- { +160- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/6', +161- ok: true, +162- ms: 36 +163- } +164:[19:24:23.191] INFO (#1259): runner rpc sendTurn +165- { +166- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +167- ok: true, +168- ms: 25 +169- } +170:[19:24:27.277] INFO (#1560): runner rpc interruptTurn +171- { +172- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +173- turnId: undefined, +174- ok: true, +175- ms: 2 +176- } +177:[19:24:27.443] INFO (#1594): runner rpc checkpoint.capture +178- { +179- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7', +180- ok: true, +181- ms: 34 +182- } +183:[19:24:27.518] INFO (#1640): runner rpc checkpoint.capture +184- { +185- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7', +186- ok: true, +187- ms: 34 +188- } +189:[19:24:52.741] INFO (#1690): runner rpc startSession +190- { +191- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +192- cwd: '/tmp/proto-runner/checkout', +193- runtimeMode: 'approval-required', +194- resumeCursor: { +195- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +196- resume: '202a75b8-d54d-4c76-8507-3cfe0c931870', +197- resumeSessionAt: '0d126ad2-ba4e-4ccb-962f-ce368ec2a5dc', +203:[19:24:53.474] INFO (#1727): runner rpc sendTurn +204- { +205- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +206- ok: true, +207- ms: 660 +208- } +209:[19:25:28.372] INFO (#1878): runner rpc respondToRequest +210- { +211- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +212- requestId: '1ec080d3-2abc-4a2a-948a-d0d9f4afe95e', +213- decision: 'accept', +214- ok: true, +215- ms: 0 +216- } +217:[19:25:30.821] INFO (#1936): runner rpc checkpoint.capture +218- { +219- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/8', +220- ok: true, +221- ms: 164 +222- } +223-(node:3610610) ExperimentalWarning: SQLite is an experimental feature and might change at any time +224-(Use `node --trace-warnings ...` to show where the warning was created) +225-[19:25:56.633] INFO (#7): runner outbox loaded +237:[19:26:05.065] INFO (#477): runner rpc startSession +238- { +239- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +240- cwd: '/tmp/proto-runner/checkout', +241- runtimeMode: 'approval-required', +242- resumeCursor: { +243- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +244- resume: '202a75b8-d54d-4c76-8507-3cfe0c931870', +245- resumeSessionAt: '3c54100f-d025-4ed4-9525-5f507a67c8bf', +251:[19:26:06.398] INFO (#514): runner rpc sendTurn +252- { +253- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +254- ok: true, +255- ms: 1045 +256- } +257:[19:26:08.915] INFO (#598): runner rpc checkpoint.capture +258- { +259- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/9', +260- ok: true, +261- ms: 74 +262- } +263-(node:3613078) ExperimentalWarning: SQLite is an experimental feature and might change at any time +264-(Use `node --trace-warnings ...` to show where the warning was created) +265-[19:27:11.790] INFO (#7): runner outbox loaded +277:[19:27:26.794] INFO (#493): runner rpc startSession +278- { +279- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +280- cwd: '/tmp/proto-runner/checkout', +281- runtimeMode: 'approval-required', +282- resumeCursor: { +283- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +284- resume: '202a75b8-d54d-4c76-8507-3cfe0c931870', +285- resumeSessionAt: '3c54100f-d025-4ed4-9525-5f507a67c8bf', +291:[19:27:27.344] INFO (#528): runner rpc sendTurn +292- { +293- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +294- ok: true, +295- ms: 489 +296- } +297-[19:27:29.708] INFO (#593): runner drained provider sessions into the outbox +298- { +299- runnerId: 'runner-f44d1a25-700d-4f1d-859c-85956c39170f', +320:[19:27:34.611] INFO (#525): runner rpc checkpoint.capture +321- { +322- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/10', +323- ok: true, +324- ms: 45 +325- } +326:[19:27:34.730] INFO (#572): runner rpc checkpoint.capture +327- { +328- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/10', +329- ok: true, +330- ms: 54 +331- } +332:[19:27:36.326] INFO (#617): runner rpc startSession +333- { +334- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +335- cwd: '/tmp/proto-runner/checkout', +336- runtimeMode: 'approval-required', +337- resumeCursor: { +338- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +339- resume: '202a75b8-d54d-4c76-8507-3cfe0c931870', +340- resumeSessionAt: '3c54100f-d025-4ed4-9525-5f507a67c8bf', +346:[19:27:36.868] INFO (#643): runner rpc sendTurn +347- { +348- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +349- ok: true, +350- ms: 485 +351- } +352:[19:27:40.470] INFO (#763): runner rpc respondToRequest +353- { +354- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', +355- requestId: 'fed8ec9c-d62b-4f8b-97dc-a1cf6c869b7f', +356- decision: 'accept', +357- ok: true, +358- ms: 0 +359- } +360:[19:27:41.549] INFO (#815): runner rpc checkpoint.capture +361- { +362- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/11', +363- ok: true, +364- ms: 32 +365- } +366-[19:28:57.900] INFO (#876): runner drained provider sessions into the outbox +367- { +368- runnerId: 'runner-f44d1a25-700d-4f1d-859c-85956c39170f', +389:[19:31:50.070] INFO (#571): runner rpc startSession +390- { +391- threadId: 'thread-title-f0b10a47-12ec-40e2-b958-683f1b38d6fa', +392- cwd: '/tmp/proto-runner/checkout', +393- runtimeMode: 'approval-required', +394- resumeCursor: null, +395- ok: true, +396- ms: 9 +397- } +398:[19:31:50.188] INFO (#606): runner rpc checkpoint.capture +399- { +400- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLXRpdGxlLWYwYjEwYTQ3LTEyZWMtNDBlMi1iOTU4LTY4M2YxYjM4ZDZmYQ/turn/0', +401- ok: true, +402- ms: 45 +403- } +404:[19:31:50.596] INFO (#616): runner rpc sendTurn +405- { +406- threadId: 'thread-title-f0b10a47-12ec-40e2-b958-683f1b38d6fa', +407- ok: true, +408- ms: 438 +409- } +410:[19:31:52.642] INFO (#718): runner rpc checkpoint.capture +411- { +412- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLXRpdGxlLWYwYjEwYTQ3LTEyZWMtNDBlMi1iOTU4LTY4M2YxYjM4ZDZmYQ/turn/1', +413- ok: true, +414- ms: 33 +415- } +416:[19:32:04.201] INFO (#560): runner rpc text.generateThreadTitle +417- { +418- cwd: '/tmp/proto-runner/checkout', +419- model: 'claude-haiku-4-5', +420- ok: true, +421- ms: 14161 +422- } +423-[19:32:21.384] INFO (#769): runner drained provider sessions into the outbox +424- { diff --git a/prototype-2/evidence/turn1.out b/prototype-2/evidence/turn1.out new file mode 100644 index 000000000000..57d0e2da5803 --- /dev/null +++ b/prototype-2/evidence/turn1.out @@ -0,0 +1,45 @@ +[19:17:17.733] thread.turn.start accepted in 40 ms +[19:17:17.754] message user: "Append a new line 'edited by the remote runner' to the end of notes.txt in the current directory. Do not touch other files. Reply with just: done" +[19:17:21.057] activity tool.started: Tool call started +[19:17:21.468] activity context-window.updated: Context window updated +[19:17:21.468] activity tool.completed: Tool call +[19:17:22.695] activity tool.started: File change started +[19:17:23.513] activity tool.updated: File change +[19:17:23.513] activity approval.requested: File-change approval requested +[19:17:23.513] activity context-window.updated: Context window updated +[19:17:23.513] approving de4acf9b-62b1-42ef-8302-ff2109bf7f7e (File-change approval requested: Edit: {"file_path":"/tmp/proto-runner/checkout/notes.txt","old_string":"hello fr) +[19:17:23.532] approval dispatched in 19 ms +[19:17:23.941] activity approval.resolved: Approval resolved +[19:17:23.941] activity tool.completed: File change +[19:17:24.782] activity context-window.updated: Context window updated +[19:17:24.782] message assistant: "done" +[19:17:25.194] activity checkpoint.captured: Checkpoint captured +[19:17:25.194] turn 8ff8671b-3ab7-4f43-8bc4-5b56d1983e2a settled: completed after 7460 ms (first assistant text after 7048 ms) +{ + "latestTurn": { + "turnId": "8ff8671b-3ab7-4f43-8bc4-5b56d1983e2a", + "state": "completed", + "requestedAt": "2026-09-25T19:17:17.702Z", + "startedAt": "2026-09-25T19:17:17.702Z", + "completedAt": "2026-09-25T19:17:24.588Z", + "assistantMessageId": "assistant:80ac5a42-e09e-42b6-b17d-cd7ce029b03b" + }, + "session": { + "status": "ready", + "activeTurnId": null, + "lastError": null + }, + "messageCount": 2, + "activityCount": 8, + "checkpoints": [ + { + "turnCount": 1, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/1", + "files": [ + "notes.txt +1/-0" + ] + } + ], + "pendingApprovals": [] +} diff --git a/prototype-2/evidence/turn10-continue.out b/prototype-2/evidence/turn10-continue.out new file mode 100644 index 000000000000..965fac76080c --- /dev/null +++ b/prototype-2/evidence/turn10-continue.out @@ -0,0 +1,110 @@ +[19:27:36.288] thread.turn.start accepted in 24 ms +[19:27:36.318] message user: "Your previous task was cut off by a machine restart. Finish it now: make sure notes.txt ends with the line 'after runner restart' (add it only if missing). Repl" +[19:27:38.385] activity tool.started: Tool call started +[19:27:38.798] activity context-window.updated: Context window updated +[19:27:38.799] activity tool.completed: Tool call +[19:27:39.622] activity tool.started: File change started +[19:27:40.446] activity tool.updated: File change +[19:27:40.446] activity approval.requested: File-change approval requested +[19:27:40.446] activity context-window.updated: Context window updated +[19:27:40.447] approving fed8ec9c-d62b-4f8b-97dc-a1cf6c869b7f (File-change approval requested: Edit: {"file_path":"/tmp/proto-runner/checkout/notes.txt","old_string":"hello fr) +[19:27:40.464] approval dispatched in 17 ms +[19:27:40.878] activity approval.resolved: Approval resolved +[19:27:40.878] activity tool.completed: File change +[19:27:40.878] activity tool.updated: File change +[19:27:41.704] activity checkpoint.captured: Checkpoint captured +[19:27:41.704] activity context-window.updated: Context window updated +[19:27:41.704] message assistant: "done" +[19:27:41.704] turn 5361dcda-bf7d-45da-9a15-a3a4347b117e settled: completed after 5415 ms (first assistant text after 5415 ms) +{ + "latestTurn": { + "turnId": "5361dcda-bf7d-45da-9a15-a3a4347b117e", + "state": "completed", + "requestedAt": "2026-09-25T19:27:36.271Z", + "startedAt": "2026-09-25T19:27:36.271Z", + "completedAt": "2026-09-25T19:27:41.453Z", + "assistantMessageId": "assistant:4a8ac405-4358-44b2-bc48-e438e8d9e683" + }, + "session": { + "status": "ready", + "activeTurnId": null, + "lastError": null + }, + "messageCount": 17, + "activityCount": 61, + "checkpoints": [ + { + "turnCount": 1, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/1", + "files": [ + "notes.txt +1/-0" + ] + }, + { + "turnCount": 2, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2", + "files": [] + }, + { + "turnCount": 3, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/3", + "files": [] + }, + { + "turnCount": 4, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/4", + "files": [] + }, + { + "turnCount": 5, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/5", + "files": [] + }, + { + "turnCount": 6, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/6", + "files": [] + }, + { + "turnCount": 7, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7", + "files": [] + }, + { + "turnCount": 8, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/8", + "files": [ + "notes.txt +1/-0" + ] + }, + { + "turnCount": 9, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/9", + "files": [] + }, + { + "turnCount": 10, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/10", + "files": [] + }, + { + "turnCount": 11, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/11", + "files": [ + "notes.txt +1/-0" + ] + } + ], + "pendingApprovals": [] +} diff --git a/prototype-2/evidence/turn11-title.out b/prototype-2/evidence/turn11-title.out new file mode 100644 index 000000000000..890037894e39 --- /dev/null +++ b/prototype-2/evidence/turn11-title.out @@ -0,0 +1,33 @@ +[19:31:50.022] thread.turn.start accepted in 32 ms +[19:31:50.037] message user: "What is the capital of France? Reply with one word." +[19:31:52.517] activity context-window.updated: Context window updated +[19:31:52.517] message assistant: "Paris." +[19:31:52.929] activity checkpoint.captured: Checkpoint captured +[19:31:52.929] activity context-window.updated: Context window updated +[19:31:52.930] turn dcc58fc1-9dd8-4e48-9db9-a2ea5c469161 settled: completed after 2907 ms (first assistant text after 2494 ms) +{ + "latestTurn": { + "turnId": "dcc58fc1-9dd8-4e48-9db9-a2ea5c469161", + "state": "completed", + "requestedAt": "2026-09-25T19:31:49.996Z", + "startedAt": "2026-09-25T19:31:49.996Z", + "completedAt": "2026-09-25T19:31:52.541Z", + "assistantMessageId": "assistant:fdb5731a-3fff-4a3e-b7b5-eda167439390" + }, + "session": { + "status": "ready", + "activeTurnId": null, + "lastError": null + }, + "messageCount": 2, + "activityCount": 2, + "checkpoints": [ + { + "turnCount": 1, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLXRpdGxlLWYwYjEwYTQ3LTEyZWMtNDBlMi1iOTU4LTY4M2YxYjM4ZDZmYQ/turn/1", + "files": [] + } + ], + "pendingApprovals": [] +} diff --git a/prototype-2/evidence/turn6-interrupt.out b/prototype-2/evidence/turn6-interrupt.out new file mode 100644 index 000000000000..2d6dec25f2b3 --- /dev/null +++ b/prototype-2/evidence/turn6-interrupt.out @@ -0,0 +1,72 @@ +[19:24:23.128] thread.turn.start accepted in 26 ms +[19:24:23.147] message user: "Count from 1 to 1500, writing each number on its own line as plain text. Do not use any tools." +[19:24:27.260] interrupting turn 01b724b4-5435-4a80-ae29-49dbcacbabb0 +[19:24:27.685] activity context-window.updated: Context window updated +[19:24:27.685] activity checkpoint.captured: Checkpoint captured +[19:24:27.685] activity checkpoint.captured: Checkpoint captured +[19:24:27.685] message assistant: "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n21\n22\n23\n24\n25\n26\n27\n28\n29\n30\n31\n32\n33\n34\n35\n36\n37\n38\n39\n40\n41\n42\n43\n44\n45\n46\n47\n48\n49\n50\n51\n52\n53\n54\n55\n56\n5" +[19:24:27.685] turn 01b724b4-5435-4a80-ae29-49dbcacbabb0 settled: completed after 4556 ms (first assistant text after 4556 ms) +{ + "latestTurn": { + "turnId": "01b724b4-5435-4a80-ae29-49dbcacbabb0", + "state": "completed", + "requestedAt": "2026-09-25T19:24:23.108Z", + "startedAt": "2026-09-25T19:24:23.108Z", + "completedAt": "2026-09-25T19:24:27.276Z", + "assistantMessageId": "assistant:fd4dfdf7-ef55-4530-a742-4b1be6390a7c" + }, + "session": { + "status": "stopped", + "activeTurnId": null, + "lastError": null + }, + "messageCount": 10, + "activityCount": 33, + "checkpoints": [ + { + "turnCount": 1, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/1", + "files": [ + "notes.txt +1/-0" + ] + }, + { + "turnCount": 2, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2", + "files": [] + }, + { + "turnCount": 3, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/3", + "files": [] + }, + { + "turnCount": 4, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/4", + "files": [] + }, + { + "turnCount": 5, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/5", + "files": [] + }, + { + "turnCount": 6, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/6", + "files": [] + }, + { + "turnCount": 7, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7", + "files": [] + } + ], + "pendingApprovals": [] +} diff --git a/prototype-2/evidence/turn7-hub-restart.out b/prototype-2/evidence/turn7-hub-restart.out new file mode 100644 index 000000000000..5a1c784ffcdc --- /dev/null +++ b/prototype-2/evidence/turn7-hub-restart.out @@ -0,0 +1,124 @@ +[19:24:52.710] thread.turn.start accepted in 25 ms +[19:24:52.746] message user: "Run the Bash command 'sleep 20' in the foreground (do not run it in the background). When it has finished, append the line 'after hub restart' to notes.txt. Rep" +[19:24:56.878] activity tool.started: Command run started +[19:24:57.281] hub unreachable: fetch failed +[19:24:58.283] hub unreachable: fetch failed +[19:24:59.285] hub unreachable: fetch failed +[19:25:00.286] hub unreachable: fetch failed +[19:25:01.291] hub unreachable: fetch failed +[19:25:02.293] hub unreachable: fetch failed +[19:25:03.296] hub unreachable: fetch failed +[19:25:04.298] hub unreachable: fetch failed +[19:25:05.300] hub unreachable: fetch failed +[19:25:06.302] hub unreachable: fetch failed +[19:25:07.303] hub unreachable: fetch failed +[19:25:08.304] hub unreachable: fetch failed +[19:25:09.306] hub unreachable: fetch failed +[19:25:10.308] hub unreachable: fetch failed +[19:25:11.310] hub unreachable: fetch failed +[19:25:12.312] hub unreachable: fetch failed +[19:25:13.314] hub unreachable: fetch failed +[19:25:14.316] hub unreachable: fetch failed +[19:25:15.318] hub unreachable: fetch failed +[19:25:16.320] hub unreachable: fetch failed +[19:25:17.321] hub unreachable: fetch failed +[19:25:18.323] hub unreachable: fetch failed +[19:25:19.325] hub unreachable: fetch failed +[19:25:20.327] hub unreachable: fetch failed +[19:25:21.328] hub unreachable: fetch failed +[19:25:22.334] hub unreachable: fetch failed +[19:25:23.336] hub unreachable: fetch failed +[19:25:24.338] hub unreachable: fetch failed +[19:25:25.340] hub unreachable: fetch failed +[19:25:26.342] hub unreachable: fetch failed +[19:25:27.398] activity tool.updated: Command run +[19:25:27.398] activity context-window.updated: Context window updated +[19:25:27.873] activity task.started: local_bash task started +[19:25:27.873] activity task.completed: Task completed +[19:25:28.306] activity tool.completed: Command run +[19:25:28.306] activity tool.started: Tool call started +[19:25:28.306] activity tool.completed: Tool call +[19:25:28.306] activity tool.started: File change started +[19:25:28.306] activity tool.updated: File change +[19:25:28.306] activity context-window.updated: Context window updated +[19:25:28.306] activity approval.requested: File-change approval requested +[19:25:28.306] approving 1ec080d3-2abc-4a2a-948a-d0d9f4afe95e (File-change approval requested: Edit: {"file_path":"/tmp/proto-runner/checkout/notes.txt","old_string":"hello fr) +[19:25:28.352] approval dispatched in 46 ms +[19:25:28.765] activity approval.resolved: Approval resolved +[19:25:28.766] activity tool.completed: File change +[19:25:30.046] activity context-window.updated: Context window updated +[19:25:30.046] message assistant: "done" +[19:25:30.957] activity context-window.updated: Context window updated +[19:25:30.957] turn e3457ec4-8d3f-493b-ae23-3d56fae7b69d settled: completed after 38246 ms (first assistant text after 37335 ms) +{ + "latestTurn": { + "turnId": "e3457ec4-8d3f-493b-ae23-3d56fae7b69d", + "state": "completed", + "requestedAt": "2026-09-25T19:24:52.691Z", + "startedAt": "2026-09-25T19:24:52.691Z", + "completedAt": "2026-09-25T19:25:30.482Z", + "assistantMessageId": "assistant:fc0426b0-ff22-4fb1-a3d6-aaa2e4d8f2a3" + }, + "session": { + "status": "ready", + "activeTurnId": null, + "lastError": null + }, + "messageCount": 12, + "activityCount": 45, + "checkpoints": [ + { + "turnCount": 1, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/1", + "files": [ + "notes.txt +1/-0" + ] + }, + { + "turnCount": 2, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2", + "files": [] + }, + { + "turnCount": 3, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/3", + "files": [] + }, + { + "turnCount": 4, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/4", + "files": [] + }, + { + "turnCount": 5, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/5", + "files": [] + }, + { + "turnCount": 6, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/6", + "files": [] + }, + { + "turnCount": 7, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7", + "files": [] + }, + { + "turnCount": 8, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/8", + "files": [ + "notes.txt +1/-0" + ] + } + ], + "pendingApprovals": [] +} diff --git a/prototype-2/evidence/turn8-after-runner-restart.out b/prototype-2/evidence/turn8-after-runner-restart.out new file mode 100644 index 000000000000..0e0cd1e26f56 --- /dev/null +++ b/prototype-2/evidence/turn8-after-runner-restart.out @@ -0,0 +1,85 @@ +[19:26:04.742] thread.turn.start accepted in 38 ms +[19:26:04.833] message user: "Without using any tools: which lines did you add to notes.txt earlier in this conversation? Reply with the exact lines only, one per line." +[19:26:08.271] activity context-window.updated: Context window updated +[19:26:08.271] message assistant: "edited by the remote runner\nafter hub restart" +[19:26:09.103] activity context-window.updated: Context window updated +[19:26:09.103] activity checkpoint.captured: Checkpoint captured +[19:26:09.103] turn 3aec0919-df29-4679-8037-0d3deaf7fea3 settled: completed after 4360 ms (first assistant text after 3528 ms) +{ + "latestTurn": { + "turnId": "3aec0919-df29-4679-8037-0d3deaf7fea3", + "state": "completed", + "requestedAt": "2026-09-25T19:26:04.710Z", + "startedAt": "2026-09-25T19:26:04.710Z", + "completedAt": "2026-09-25T19:26:08.669Z", + "assistantMessageId": "assistant:b663b4d0-215b-4770-8878-3a1d64c12568" + }, + "session": { + "status": "ready", + "activeTurnId": null, + "lastError": null + }, + "messageCount": 14, + "activityCount": 47, + "checkpoints": [ + { + "turnCount": 1, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/1", + "files": [ + "notes.txt +1/-0" + ] + }, + { + "turnCount": 2, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2", + "files": [] + }, + { + "turnCount": 3, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/3", + "files": [] + }, + { + "turnCount": 4, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/4", + "files": [] + }, + { + "turnCount": 5, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/5", + "files": [] + }, + { + "turnCount": 6, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/6", + "files": [] + }, + { + "turnCount": 7, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7", + "files": [] + }, + { + "turnCount": 8, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/8", + "files": [ + "notes.txt +1/-0" + ] + }, + { + "turnCount": 9, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/9", + "files": [] + } + ], + "pendingApprovals": [] +} diff --git a/prototype-2/evidence/turn9-runner-restart-midturn.out b/prototype-2/evidence/turn9-runner-restart-midturn.out new file mode 100644 index 000000000000..3af00f2cc718 --- /dev/null +++ b/prototype-2/evidence/turn9-runner-restart-midturn.out @@ -0,0 +1,92 @@ +[19:27:26.747] thread.turn.start accepted in 31 ms +[19:27:26.776] message user: "Run the Bash command 'sleep 15' in the foreground (do not run it in the background). When it has finished, append the line 'after runner restart' to notes.txt. " +[19:27:29.277] activity tool.started: Command run started +[19:27:29.278] activity tool.updated: Command run +[19:27:29.278] activity context-window.updated: Context window updated +[19:27:34.662] activity context-window.updated: Context window updated +[19:27:34.662] activity tool.completed: Command run +[19:27:34.662] turn b53e0583-1332-4761-b85c-163aedc3c1db settled: completed after 7914 ms (first assistant text after n/a ms) +{ + "latestTurn": { + "turnId": "b53e0583-1332-4761-b85c-163aedc3c1db", + "state": "completed", + "requestedAt": "2026-09-25T19:27:26.722Z", + "startedAt": "2026-09-25T19:27:26.722Z", + "completedAt": "2026-09-25T19:27:29.407Z", + "assistantMessageId": "assistant:b53e0583-1332-4761-b85c-163aedc3c1db" + }, + "session": { + "status": "stopped", + "activeTurnId": null, + "lastError": null + }, + "messageCount": 15, + "activityCount": 52, + "checkpoints": [ + { + "turnCount": 1, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/1", + "files": [ + "notes.txt +1/-0" + ] + }, + { + "turnCount": 2, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2", + "files": [] + }, + { + "turnCount": 3, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/3", + "files": [] + }, + { + "turnCount": 4, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/4", + "files": [] + }, + { + "turnCount": 5, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/5", + "files": [] + }, + { + "turnCount": 6, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/6", + "files": [] + }, + { + "turnCount": 7, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7", + "files": [] + }, + { + "turnCount": 8, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/8", + "files": [ + "notes.txt +1/-0" + ] + }, + { + "turnCount": 9, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/9", + "files": [] + }, + { + "turnCount": 10, + "status": "ready", + "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/10", + "files": [] + } + ], + "pendingApprovals": [] +} diff --git a/prototype-2/hub-restart-test.sh b/prototype-2/hub-restart-test.sh new file mode 100755 index 000000000000..ff30c6f4120d --- /dev/null +++ b/prototype-2/hub-restart-test.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# (e1) Restart the hub while a turn runs on the runner. The approval request +# for the turn's file edit is produced while the hub is down. +set -uo pipefail +P2=/workspace/repos/t3code-proto-runner/prototype-2 +OUTBOX=/tmp/proto-runner/runner/userdata/runner/outbox.ndjson +head_seq() { tail -1 "$OUTBOX" | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>console.log(s.trim()?JSON.parse(s).sequence:0))'; } +START=$(head_seq); echo "outbox head before turn: $START" +node $P2/scenario.mjs turn "Run the Bash command 'sleep 20' in the foreground (do not run it in the background). When it has finished, append the line 'after hub restart' to notes.txt. Reply with just: done" --approve --timeout 240 > /tmp/proto-runner/logs/turn7-hub-restart.out 2>&1 & +SCEN=$! +for i in $(seq 1 120); do node $P2/outbox-tail.mjs $START item.started | grep -q command && break; sleep 0.5; done +echo "[$(date +%T.%3N)] command started on runner; stopping hub"; /tmp/proto-runner/ctl.sh stop-hub +DOWN=$(head_seq); echo "[$(date +%T.%3N)] hub down; outbox head $DOWN" +for i in $(seq 1 80); do node $P2/outbox-tail.mjs $DOWN request.opened | grep -q . && break; sleep 0.5; done +echo "[$(date +%T.%3N)] approval requested while hub down:"; node $P2/outbox-tail.mjs $DOWN request +echo "[$(date +%T.%3N)] outbox head $(head_seq); restarting hub"; /tmp/proto-runner/ctl.sh start-hub +wait $SCEN; echo "scenario exit $?" +echo "events produced by the runner while the hub was down: $DOWN..$(head_seq)" diff --git a/prototype-2/outbox-tail.mjs b/prototype-2/outbox-tail.mjs new file mode 100644 index 000000000000..96f1fcaa2d35 --- /dev/null +++ b/prototype-2/outbox-tail.mjs @@ -0,0 +1,23 @@ +#!/usr/bin/env node +// Prints outbox events after a sequence: node outbox-tail.mjs [typeFilter] +import * as fs from "node:fs"; +const path = process.env.OUTBOX ?? "/tmp/proto-runner/runner/userdata/runner/outbox.ndjson"; +const after = Number(process.argv[2] ?? 0); +const filter = process.argv[3]; +const lines = fs + .readFileSync(path, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); +for (const entry of lines) { + if (entry.sequence <= after) continue; + if (filter && !entry.event.type.startsWith(filter)) continue; + const payload = entry.event.payload ?? {}; + console.log( + entry.sequence, + entry.event.createdAt.slice(11, 23), + entry.event.type, + payload.itemType ?? payload.requestType ?? payload.state ?? "", + ); +} diff --git a/prototype-2/runner-restart-midturn-test.sh b/prototype-2/runner-restart-midturn-test.sh new file mode 100755 index 000000000000..7ea7e9557fee --- /dev/null +++ b/prototype-2/runner-restart-midturn-test.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Restart the runner while a turn is executing on it (graceful stop drains the +# provider session into the outbox), then continue the work in a new turn. +set -uo pipefail +P2=/workspace/repos/t3code-proto-runner/prototype-2 +OUTBOX=/tmp/proto-runner/runner/userdata/runner/outbox.ndjson +head_seq() { tail -1 "$OUTBOX" | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>console.log(s.trim()?JSON.parse(s).sequence:0))'; } +START=$(head_seq); echo "outbox head before turn: $START" +node $P2/scenario.mjs turn "Run the Bash command 'sleep 15' in the foreground (do not run it in the background). When it has finished, append the line 'after runner restart' to notes.txt. Reply with just: done" --approve --timeout 120 > /tmp/proto-runner/logs/turn9-runner-restart-midturn.out 2>&1 & +SCEN=$! +for i in $(seq 1 120); do node $P2/outbox-tail.mjs $START item.started | grep -q command && break; sleep 0.5; done +echo "[$(date +%T.%3N)] command running on runner; stopping runner"; /tmp/proto-runner/ctl.sh stop-runner +echo "[$(date +%T.%3N)] events outboxed during drain:"; node $P2/outbox-tail.mjs $START | tail -4 +sleep 2; echo "[$(date +%T.%3N)] starting runner"; /tmp/proto-runner/ctl.sh start-runner +wait $SCEN; echo "scenario exit $?" +echo "[$(date +%T.%3N)] follow-up turn" +node $P2/scenario.mjs turn "Your previous task was cut off by a machine restart. Finish it now: make sure notes.txt ends with the line 'after runner restart' (add it only if missing). Reply with just: done" --approve --timeout 120 > /tmp/proto-runner/logs/turn10-continue.out 2>&1 +echo "follow-up exit $?"; cat /tmp/proto-runner/checkout/notes.txt diff --git a/prototype-2/scenario.mjs b/prototype-2/scenario.mjs new file mode 100644 index 000000000000..35479c021c53 --- /dev/null +++ b/prototype-2/scenario.mjs @@ -0,0 +1,335 @@ +#!/usr/bin/env node +// Prototype 2 scenario driver. Talks only to the hub (:4411) over its public +// HTTP API and WebSocket RPC, exactly as a client would. +// +// node scenario.mjs setup +// node scenario.mjs turn "" [--approve] [--interrupt-after-tool] [--timeout 180] +// node scenario.mjs watch [--timeout 180] # follow the running turn without dispatching +// node scenario.mjs diff +// node scenario.mjs show +import * as fs from "node:fs"; +import * as crypto from "node:crypto"; + +const HUB = process.env.HUB_URL ?? "http://127.0.0.1:4411"; +const STATE_PATH = process.env.SCENARIO_STATE ?? "/tmp/proto-runner/scenario-state.json"; +const TOKEN = fs + .readFileSync(process.env.HUB_TOKEN_PATH ?? "/tmp/proto-runner/hub-token", "utf8") + .trim(); +const MODEL = { instanceId: "claudeAgent", model: "claude-haiku-4-5" }; + +const now = () => new Date().toISOString(); +const id = () => crypto.randomUUID(); +const stamp = () => new Date().toISOString().slice(11, 23); +const log = (...args) => console.log(`[${stamp()}]`, ...args); +const loadState = () => + fs.existsSync(STATE_PATH) ? JSON.parse(fs.readFileSync(STATE_PATH, "utf8")) : {}; +const saveState = (state) => fs.writeFileSync(STATE_PATH, JSON.stringify(state, null, 2)); +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function http(method, path, body) { + const response = await fetch(`${HUB}${path}`, { + method, + headers: { + authorization: `Bearer ${TOKEN}`, + ...(body !== undefined ? { "content-type": "application/json" } : {}), + }, + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), + }); + const text = await response.text(); + if (!response.ok) + throw new Error(`${method} ${path} -> ${response.status}: ${text.slice(0, 400)}`); + return text.length > 0 ? JSON.parse(text) : null; +} + +const dispatch = (command) => http("POST", "/api/orchestration/dispatch", command); +const threadSnapshot = (threadId) => http("GET", `/api/orchestration/threads/${threadId}`); + +/** One Effect RPC call over the hub's /ws endpoint using the JSON wire format. */ +async function rpc(tag, payload) { + const { ticket } = await http("POST", "/api/auth/websocket-ticket"); + const url = `${HUB.replace(/^http/, "ws")}/ws?wsTicket=${encodeURIComponent(ticket)}`; + const socket = new WebSocket(url); + return await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`rpc ${tag} timed out`)), 30_000); + socket.addEventListener("open", () => { + socket.send(JSON.stringify({ _tag: "Request", id: "1", tag, payload, headers: [] })); + }); + socket.addEventListener("message", (message) => { + const decoded = JSON.parse(String(message.data)); + const messages = Array.isArray(decoded) ? decoded : [decoded]; + for (const entry of messages) { + if (entry._tag === "Ping") socket.send(JSON.stringify({ _tag: "Pong" })); + if (entry._tag === "Exit" && String(entry.requestId) === "1") { + clearTimeout(timer); + socket.close(); + if (entry.exit._tag === "Success") resolve(entry.exit.value); + else reject(new Error(`rpc ${tag} failed: ${JSON.stringify(entry.exit).slice(0, 600)}`)); + } + } + }); + socket.addEventListener("error", (error) => reject(error.error ?? new Error("socket error"))); + }); +} + +function summarize(thread) { + const pending = new Map(); + for (const activity of thread.activities) { + const requestId = activity.payload?.requestId; + if (activity.kind === "approval.requested") pending.set(requestId, activity); + if (activity.kind === "approval.resolved") pending.delete(requestId); + } + return { + latestTurn: thread.latestTurn, + session: thread.session + ? { + status: thread.session.status, + activeTurnId: thread.session.activeTurnId, + lastError: thread.session.lastError, + } + : null, + messageCount: thread.messages.length, + activityCount: thread.activities.length, + checkpoints: thread.checkpoints.map((c) => ({ + turnCount: c.checkpointTurnCount, + status: c.status, + ref: c.checkpointRef, + files: c.files.map((f) => `${f.path} +${f.additions}/-${f.deletions}`), + })), + pendingApprovals: [...pending.values()].map((a) => ({ + requestId: a.payload.requestId, + summary: a.summary, + detail: a.payload.detail, + })), + }; +} + +async function follow(state, options) { + const deadline = Date.now() + (options.timeoutSec ?? 180) * 1000; + const seenActivities = new Set(options.preSeenActivities ?? []); + const seenMessages = new Map(options.preSeenMessages ?? []); + let commandApprovedAt = null; + const approved = new Set(); + let interrupted = false; + let toolSeenAt = null; + const startedAt = Date.now(); + let firstAssistantAt = null; + let lastSummary = null; + while (Date.now() < deadline) { + let snapshot; + try { + snapshot = await threadSnapshot(state.threadId); + } catch (error) { + log("hub unreachable:", String(error.message ?? error).slice(0, 120)); + await sleep(1000); + continue; + } + const thread = snapshot.thread; + for (const activity of thread.activities) { + if (seenActivities.has(activity.id)) continue; + seenActivities.add(activity.id); + log(`activity ${activity.kind}: ${activity.summary}`); + if (activity.tone === "tool" && toolSeenAt === null) toolSeenAt = Date.now(); + } + for (const message of thread.messages) { + const previous = seenMessages.get(message.id); + if (previous !== message.text) { + seenMessages.set(message.id, message.text); + if (message.role === "assistant" && firstAssistantAt === null) + firstAssistantAt = Date.now(); + if ( + previous === undefined || + !message.streaming || + message.text.length - (previous?.length ?? 0) > 200 + ) { + log( + `message ${message.role}${message.streaming ? " (streaming)" : ""}: ${JSON.stringify(message.text.slice(0, 160))}`, + ); + } + } + } + const summary = summarize(thread); + lastSummary = summary; + if (options.approve) { + for (const pending of summary.pendingApprovals) { + if (approved.has(pending.requestId)) continue; + approved.add(pending.requestId); + log( + `approving ${pending.requestId} (${pending.summary}: ${String(pending.detail ?? "").slice(0, 80)})`, + ); + const approvalStart = Date.now(); + const sent = await dispatch({ + type: "thread.approval.respond", + commandId: id(), + threadId: state.threadId, + requestId: pending.requestId, + decision: "accept", + createdAt: now(), + }).then( + () => true, + (error) => { + log( + "approval dispatch failed, will retry:", + String(error.message ?? error).slice(0, 120), + ); + approved.delete(pending.requestId); + return false; + }, + ); + if (!sent) continue; + log(`approval dispatched in ${Date.now() - approvalStart} ms`); + if (/command/i.test(pending.summary)) commandApprovedAt = Date.now(); + } + } + // Interrupt once the approved command has been running for 3 s (or 15 s + // after the tool call streamed if no approval was needed). + const commandRunning = + (commandApprovedAt !== null && Date.now() - commandApprovedAt > 3000) || + (toolSeenAt !== null && Date.now() - toolSeenAt > 15000); + const textStreaming = + (firstAssistantAt !== null && Date.now() - firstAssistantAt > 2000) || + (options.interruptAfterMs !== undefined && Date.now() - startedAt > options.interruptAfterMs); + if ( + !interrupted && + ((options.interruptAfterTool && commandRunning) || + (options.interruptAfterText && textStreaming)) + ) { + interrupted = true; + const turnId = summary.latestTurn?.turnId; + log(`interrupting turn ${turnId}`); + await dispatch({ + type: "thread.turn.interrupt", + commandId: id(), + threadId: state.threadId, + ...(turnId ? { turnId } : {}), + createdAt: now(), + }); + } + const latest = summary.latestTurn; + if (latest && options.turnMessageId !== undefined) { + // wait until the turn for this message exists + } + if ( + latest && + latest.state !== "running" && + (options.expectNewTurn === undefined || latest.turnId !== options.expectNewTurn) + ) { + if ( + summary.pendingApprovals.length === 0 && + (thread.session?.activeTurnId ?? null) === null + ) { + log( + `turn ${latest.turnId} settled: ${latest.state} after ${Date.now() - startedAt} ms (first assistant text after ${firstAssistantAt ? firstAssistantAt - startedAt : "n/a"} ms)`, + ); + // Give the checkpoint reactor a moment, then report. + await sleep(1500); + const final = summarize((await threadSnapshot(state.threadId)).thread); + console.log(JSON.stringify(final, null, 2)); + return final; + } + } + await sleep(400); + } + log("timed out following the turn"); + console.log(JSON.stringify(lastSummary, null, 2)); + return lastSummary; +} + +const [command, ...rest] = process.argv.slice(2); +const flag = (name) => rest.includes(name); +const flagValue = (name, fallback) => { + const index = rest.indexOf(name); + return index >= 0 ? rest[index + 1] : fallback; +}; + +if (command === "setup") { + const state = { projectId: `proj-${id()}`, threadId: `thread-${id()}` }; + const t0 = Date.now(); + await dispatch({ + type: "project.create", + commandId: id(), + projectId: state.projectId, + title: "Proto runner checkout", + workspaceRoot: process.env.WORKSPACE_ROOT ?? "/tmp/proto-runner/checkout", + defaultModelSelection: MODEL, + createdAt: now(), + }); + log(`project.create accepted in ${Date.now() - t0} ms`); + const t1 = Date.now(); + await dispatch({ + type: "thread.create", + commandId: id(), + threadId: state.threadId, + projectId: state.projectId, + title: "Remote runner thread", + modelSelection: MODEL, + runtimeMode: "approval-required", + interactionMode: "default", + branch: null, + worktreePath: null, + createdAt: now(), + }); + log(`thread.create accepted in ${Date.now() - t1} ms`); + saveState(state); + console.log(JSON.stringify(state)); +} else if (command === "turn") { + const state = loadState(); + const prompt = rest[0]; + const beforeThread = (await threadSnapshot(state.threadId)).thread; + const before = beforeThread.latestTurn?.turnId; + const t0 = Date.now(); + await dispatch({ + type: "thread.turn.start", + commandId: id(), + threadId: state.threadId, + message: { messageId: `msg-${id()}`, role: "user", text: prompt, attachments: [] }, + runtimeMode: "approval-required", + interactionMode: "default", + createdAt: now(), + }); + log(`thread.turn.start accepted in ${Date.now() - t0} ms`); + await follow(state, { + approve: flag("--approve"), + interruptAfterTool: flag("--interrupt-after-tool"), + interruptAfterText: + flag("--interrupt-after-text") || flagValue("--interrupt-after-ms") !== undefined, + interruptAfterMs: + flagValue("--interrupt-after-ms") !== undefined + ? Number(flagValue("--interrupt-after-ms")) + : undefined, + timeoutSec: Number(flagValue("--timeout", "180")), + expectNewTurn: before, + preSeenActivities: beforeThread.activities.map((activity) => activity.id), + preSeenMessages: beforeThread.messages.map((message) => [message.id, message.text]), + }); +} else if (command === "watch") { + const state = loadState(); + const beforeThread = (await threadSnapshot(state.threadId)).thread; + await follow(state, { + approve: flag("--approve"), + interruptAfterTool: flag("--interrupt-after-tool"), + timeoutSec: Number(flagValue("--timeout", "180")), + preSeenActivities: flag("--all") ? [] : beforeThread.activities.map((activity) => activity.id), + preSeenMessages: flag("--all") + ? [] + : beforeThread.messages.map((message) => [message.id, message.text]), + }); +} else if (command === "diff") { + const state = loadState(); + const t0 = Date.now(); + const result = await rpc("orchestration.getTurnDiff", { + threadId: state.threadId, + fromTurnCount: Number(rest[0]), + toTurnCount: Number(rest[1]), + }); + log(`getTurnDiff via hub in ${Date.now() - t0} ms`); + console.log(JSON.stringify(result, null, 2)); +} else if (command === "show") { + const state = loadState(); + const snapshot = await threadSnapshot(state.threadId); + console.log(JSON.stringify(summarize(snapshot.thread), null, 2)); + for (const message of snapshot.thread.messages) + console.log(`${message.role}: ${JSON.stringify(message.text.slice(0, 300))}`); +} else { + console.error("usage: scenario.mjs setup|turn|watch|diff|show"); + process.exit(2); +} diff --git a/prototype-2/start-control.sh b/prototype-2/start-control.sh new file mode 100755 index 000000000000..420487d4c7c7 --- /dev/null +++ b/prototype-2/start-control.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Control: the same server build in ordinary local mode (no runner), :4413. +set -euo pipefail +cd /tmp/proto-runner/control +export T3CODE_TELEMETRY_ENABLED=false T3CODE_NO_BROWSER=true T3CODE_AUTO_BOOTSTRAP_PROJECT_FROM_CWD=false +exec node /workspace/repos/t3code-proto-runner/apps/server/src/bin.ts \ + --base-dir /tmp/proto-runner/control --port 4413 >>/tmp/proto-runner/logs/control.log 2>&1 diff --git a/prototype-2/start-hub.sh b/prototype-2/start-hub.sh new file mode 100755 index 000000000000..341ef9b45fca --- /dev/null +++ b/prototype-2/start-hub.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Starts the prototype hub on :4411. The hub runs inside a mount namespace in +# which /tmp/proto-runner/checkout is an empty, mode-000 tmpfs, under strace +# recording every syscall that names that path. +set -euo pipefail +cd /tmp/proto-runner/hub +export T3CODE_RUNNER_URL=ws://127.0.0.1:4412/runner/ws +export T3CODE_RUNNER_TOKEN="$(cat /tmp/proto-runner/runner-token)" +export T3CODE_TELEMETRY_ENABLED=false T3CODE_NO_BROWSER=true T3CODE_AUTO_BOOTSTRAP_PROJECT_FROM_CWD=false +exec strace -f -qq --seccomp-bpf -e trace=%file,%process,chdir,fchdir -P /tmp/proto-runner/checkout \ + -o "/tmp/proto-runner/logs/hub.strace.$(date +%s)" \ + bwrap --dev-bind / / --tmpfs /tmp/proto-runner/checkout --chmod 0000 /tmp/proto-runner/checkout --die-with-parent \ + node /workspace/repos/t3code-proto-runner/apps/server/src/bin.ts \ + --base-dir /tmp/proto-runner/hub --port 4411 >>/tmp/proto-runner/logs/hub.log 2>&1 diff --git a/prototype-2/start-runner.sh b/prototype-2/start-runner.sh new file mode 100755 index 000000000000..f78d86c0684e --- /dev/null +++ b/prototype-2/start-runner.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Starts the prototype runner on :4412 with its own state dir; hosts real Claude. +set -euo pipefail +cd /tmp/proto-runner/checkout +export T3CODE_RUNNER_TOKEN="$(cat /tmp/proto-runner/runner-token)" +export T3CODE_TELEMETRY_ENABLED=false T3CODE_NO_BROWSER=true T3CODE_RUNNER_DRIVERS=claudeAgent T3CODE_RUNNER_OUTBOX_KEEP=1 +exec node /workspace/repos/t3code-proto-runner/apps/server/src/bin.ts runner \ + --base-dir /tmp/proto-runner/runner --port 4412 >>/tmp/proto-runner/logs/runner.log 2>&1 From efe158bee0a410f07fd2c01279e19158500845c8 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 19:31:14 +0000 Subject: [PATCH 02/58] feat(server): prototype multi-user Postgres persistence for the hub Run the unchanged orchestration engine and projection pipeline on PostgreSQL through tenant-scoped repository layers. Every table carries a user_id that leads its keys and indexes, event sequences are per user via a transactional counter row, and all engines in a process share one pool. Port the 12 repository layers and ProjectionSnapshotQuery, fixing the SQLite-specific semantics: NULL ordering, bytewise collation, ILIKE for search, int8 parsing, repeatable-read snapshot reads, and NUL-escape-safe JSON extraction. Export makeOrchestrationProjectionPipeline so the pipeline can be composed with the Postgres repositories. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/package.json | 1 + .../Layers/ProjectionPipeline.ts | 2 +- .../server/src/persistence/Layers/Postgres.ts | 72 + .../persistence/Postgres/HubEngine.test.ts | 429 +++ .../src/persistence/Postgres/HubEngine.ts | 92 + .../src/persistence/Postgres/HubTenant.ts | 18 + .../Postgres/OrchestrationCommandReceipts.ts | 95 + .../Postgres/OrchestrationEventStore.ts | 427 +++ .../Postgres/ProjectionCheckpoints.ts | 225 ++ .../Postgres/ProjectionPendingApprovals.ts | 162 + .../Postgres/ProjectionProjects.ts | 160 + .../Postgres/ProjectionSnapshotQuery.ts | 3193 +++++++++++++++++ .../persistence/Postgres/ProjectionState.ts | 125 + .../Postgres/ProjectionThreadActivities.ts | 212 ++ .../Postgres/ProjectionThreadMessages.ts | 251 ++ .../Postgres/ProjectionThreadProposedPlans.ts | 117 + .../Postgres/ProjectionThreadSessions.ts | 118 + .../persistence/Postgres/ProjectionThreads.ts | 247 ++ .../persistence/Postgres/ProjectionTurns.ts | 400 +++ .../server/src/persistence/Postgres/Schema.ts | 382 ++ pnpm-lock.yaml | 3 + 21 files changed, 6730 insertions(+), 1 deletion(-) create mode 100644 apps/server/src/persistence/Layers/Postgres.ts create mode 100644 apps/server/src/persistence/Postgres/HubEngine.test.ts create mode 100644 apps/server/src/persistence/Postgres/HubEngine.ts create mode 100644 apps/server/src/persistence/Postgres/HubTenant.ts create mode 100644 apps/server/src/persistence/Postgres/OrchestrationCommandReceipts.ts create mode 100644 apps/server/src/persistence/Postgres/OrchestrationEventStore.ts create mode 100644 apps/server/src/persistence/Postgres/ProjectionCheckpoints.ts create mode 100644 apps/server/src/persistence/Postgres/ProjectionPendingApprovals.ts create mode 100644 apps/server/src/persistence/Postgres/ProjectionProjects.ts create mode 100644 apps/server/src/persistence/Postgres/ProjectionSnapshotQuery.ts create mode 100644 apps/server/src/persistence/Postgres/ProjectionState.ts create mode 100644 apps/server/src/persistence/Postgres/ProjectionThreadActivities.ts create mode 100644 apps/server/src/persistence/Postgres/ProjectionThreadMessages.ts create mode 100644 apps/server/src/persistence/Postgres/ProjectionThreadProposedPlans.ts create mode 100644 apps/server/src/persistence/Postgres/ProjectionThreadSessions.ts create mode 100644 apps/server/src/persistence/Postgres/ProjectionThreads.ts create mode 100644 apps/server/src/persistence/Postgres/ProjectionTurns.ts create mode 100644 apps/server/src/persistence/Postgres/Schema.ts diff --git a/apps/server/package.json b/apps/server/package.json index 073008d917a5..d3ad9d5b1edd 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -26,6 +26,7 @@ "@effect/platform-bun": "catalog:", "@effect/platform-node": "catalog:", "@effect/platform-node-shared": "catalog:", + "@effect/sql-pg": "catalog:", "@effect/sql-sqlite-bun": "catalog:", "@ff-labs/fff-node": "0.9.4", "@opencode-ai/sdk": "^1.3.15", diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts index eef5c76641ed..1a8afcac8520 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts @@ -490,7 +490,7 @@ const runAttachmentSideEffects = Effect.fn("runAttachmentSideEffects")(function* ); }); -const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjectionPipeline")( +export const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjectionPipeline")( function* () { const sql = yield* SqlClient.SqlClient; const eventStore = yield* OrchestrationEventStore; diff --git a/apps/server/src/persistence/Layers/Postgres.ts b/apps/server/src/persistence/Layers/Postgres.ts new file mode 100644 index 000000000000..b4cb13e10874 --- /dev/null +++ b/apps/server/src/persistence/Layers/Postgres.ts @@ -0,0 +1,72 @@ +import * as PgClient from "@effect/sql-pg/PgClient"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { HUB_BASELINE_STATEMENTS, HUB_SCHEMA_VERSION } from "../Postgres/Schema.ts"; + +export interface HubPostgresConfig { + readonly url: string; + readonly maxConnections?: number; + readonly applicationName?: string; +} + +// node-postgres returns int8/numeric as strings. Hub sequences, COUNT(*) and +// MAX(sequence) are int8, and the shared row schemas decode numbers, so parse +// them here. Values stay far below 2^53 (sequences are per user). +const identity = (value: string) => value; +const toNumber = (value: string) => Number(value); +const HUB_TYPE_PARSERS: Record unknown> = { + 16: (value) => value === "t", // bool + 20: toNumber, // int8 + 21: toNumber, // int2 + 23: toNumber, // int4 + 26: toNumber, // oid + 700: toNumber, // float4 + 701: toNumber, // float8 + 1700: toNumber, // numeric (SUM of int8) + 114: (value) => JSON.parse(value), // json + 3802: (value) => JSON.parse(value), // jsonb +}; + +export const hubPgTypes = { + getTypeParser: (oid: number, format?: string) => + format === "binary" ? identity : (HUB_TYPE_PARSERS[oid] ?? identity), +} as unknown as NonNullable; + +export const makeHubPgClientLayer = (config: HubPostgresConfig) => + PgClient.layer({ + url: Redacted.make(config.url), + maxConnections: config.maxConnections ?? 10, + applicationName: config.applicationName ?? "t3-hub", + types: hubPgTypes, + }); + +// Arbitrary constant; serializes concurrent hub processes applying the baseline. +const HUB_SCHEMA_LOCK_KEY = 7_330_001; + +/** + * Applies the baseline idempotently. Transaction-scoped advisory locks are safe + * behind PgBouncer transaction pooling; session-level locks are not. + */ +export const ensureHubSchema = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql.withTransaction( + Effect.gen(function* () { + yield* sql`SELECT pg_advisory_xact_lock(${HUB_SCHEMA_LOCK_KEY})`; + for (const statement of HUB_BASELINE_STATEMENTS) { + yield* sql.unsafe(statement); + } + yield* sql` + INSERT INTO hub_schema_version (version) + VALUES (${HUB_SCHEMA_VERSION}) + ON CONFLICT (version) DO NOTHING + `; + }), + ); +}); + +/** Shared pool plus schema, provided once per hub process. */ +export const makeHubPostgresPersistenceLive = (config: HubPostgresConfig) => + Layer.provideMerge(Layer.effectDiscard(ensureHubSchema), makeHubPgClientLayer(config)); diff --git a/apps/server/src/persistence/Postgres/HubEngine.test.ts b/apps/server/src/persistence/Postgres/HubEngine.test.ts new file mode 100644 index 000000000000..ba2771b81ec2 --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubEngine.test.ts @@ -0,0 +1,429 @@ +/** + * Hub prototype: two users' orchestration engines in one process over one + * Postgres database. Runs only when T3_HUB_TEST_DATABASE_URL points at a + * disposable database, e.g. + * T3_HUB_TEST_DATABASE_URL=postgres://postgres@127.0.0.1:54339/hub + */ +import { + CommandId, + DEFAULT_PROVIDER_INTERACTION_MODE, + EventId, + MessageId, + ProjectId, + ProviderInstanceId, + ThreadId, + type OrchestrationCommand, + type OrchestrationEvent, +} from "@t3tools/contracts"; +import * as NodeCrypto from "node:crypto"; + +import * as PgClient from "@effect/sql-pg/PgClient"; +import { assert, expect, it } from "@effect/vitest"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Redacted from "effect/Redacted"; +import * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { OrchestrationEngineService } from "../../orchestration/Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../../orchestration/Services/ProjectionSnapshotQuery.ts"; +import { hubPgTypes } from "../Layers/Postgres.ts"; +import { makeHubSharedLayer, makeHubUserEngineLayer, type HubSharedServices } from "./HubEngine.ts"; +import { HUB_TENANT_TABLES, hubRowLevelSecurityStatements } from "./Schema.ts"; + +const databaseUrl = process.env.T3_HUB_TEST_DATABASE_URL; +const runId = NodeCrypto.randomUUID().slice(0, 8); +const userA = `test-user-a-${runId}`; +const userB = `test-user-b-${runId}`; +const createdAt = "2026-09-25T00:00:00.000Z"; +const at = (second: number) => `2026-09-25T00:00:${String(second).padStart(2, "0")}.000Z`; +const modelSelection = { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5-codex" }; + +// Both users deliberately reuse the same project, thread, message, and command +// ids: isolation must come from the tenant key, not from id uniqueness. +const commandsFor = (label: string): ReadonlyArray => [ + { + type: "project.create", + commandId: CommandId.make("cmd-project-create"), + projectId: ProjectId.make("project-shared"), + title: `Project of ${label}`, + workspaceRoot: `/workspace/${label}`, + defaultModelSelection: modelSelection, + createdAt: at(1), + }, + { + type: "thread.create", + commandId: CommandId.make("cmd-thread-create"), + threadId: ThreadId.make("thread-shared"), + projectId: ProjectId.make("project-shared"), + title: `Thread of ${label}`, + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: null, + createdAt: at(2), + }, + { + type: "thread.turn.start", + commandId: CommandId.make("cmd-turn-start"), + threadId: ThreadId.make("thread-shared"), + message: { + messageId: MessageId.make("msg-user-1"), + role: "user", + text: `hello from ${label}`, + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: at(3), + }, + { + type: "thread.activity.append", + commandId: CommandId.make("cmd-activity-1"), + threadId: ThreadId.make("thread-shared"), + activity: { + id: EventId.make("activity-1"), + tone: "tool", + kind: "tool.completed", + summary: `tool ran for ${label}`, + payload: { owner: label }, + turnId: null, + createdAt: at(4), + }, + createdAt: at(5), + }, + { + type: "thread.message.assistant.delta", + commandId: CommandId.make("cmd-assistant-delta-1"), + threadId: ThreadId.make("thread-shared"), + messageId: MessageId.make("msg-assistant-1"), + delta: `reply to ${label}`, + createdAt: at(6), + }, + { + type: "thread.message.assistant.complete", + commandId: CommandId.make("cmd-assistant-complete-1"), + threadId: ThreadId.make("thread-shared"), + messageId: MessageId.make("msg-assistant-1"), + createdAt: at(7), + }, +]; + +const extraThreadFor = (label: string): OrchestrationCommand => ({ + type: "thread.create", + commandId: CommandId.make("cmd-thread-extra-create"), + threadId: ThreadId.make("thread-extra"), + projectId: ProjectId.make("project-shared"), + title: `Extra thread of ${label}`, + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: null, + createdAt, +}); + +type UserEngine = { + readonly run: ( + effect: Effect.Effect, + ) => Effect.Effect; + readonly scope: Scope.Closeable; +}; + +// Builds one user's engine (bootstrap + command read model) over the shared +// pool and platform services, in its own scope so it can be evicted. +const activateUser = (userId: string) => + Effect.gen(function* () { + const shared = yield* Effect.context(); + const scope = yield* Scope.make(); + const context = yield* Layer.buildWithScope(makeHubUserEngineLayer({ userId, shared }), scope); + return { + scope, + run: (effect) => Effect.provideContext(effect, context), + } satisfies UserEngine; + }); + +const readEvents = (user: UserEngine, fromSequenceExclusive = 0) => + user.run( + Effect.gen(function* () { + const engine = yield* OrchestrationEngineService; + const chunk = yield* Stream.runCollect(engine.readEvents(fromSequenceExclusive)); + return Array.from(chunk) as OrchestrationEvent[]; + }), + ); + +const dispatch = (user: UserEngine, command: OrchestrationCommand) => + user.run( + Effect.flatMap(Effect.service(OrchestrationEngineService), (engine) => + engine.dispatch(command), + ), + ); + +const withQuery = ( + user: UserEngine, + read: (query: ProjectionSnapshotQuery["Service"]) => Effect.Effect, +) => user.run(Effect.flatMap(Effect.service(ProjectionSnapshotQuery), read)); + +const deleteUsers = (users: ReadonlyArray) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + for (const table of [...HUB_TENANT_TABLES, "hub_users"]) { + yield* sql`DELETE FROM ${sql(table)} WHERE ${sql.in("user_id", users)}`; + } + }); + +if (databaseUrl === undefined) { + it.skip("hub engines on Postgres (set T3_HUB_TEST_DATABASE_URL)", () => {}); +} else { + it.layer(makeHubSharedLayer({ url: databaseUrl, maxConnections: 4 }), { + excludeTestServices: true, + })("hub engines on Postgres", (it) => { + it.effect("isolates projects, threads, events, and sequences per user in one process", () => + Effect.gen(function* () { + yield* deleteUsers([userA, userB]); + const a = yield* activateUser(userA); + const b = yield* activateUser(userB); + + // Interleave the two users' commands through their own engines. + const commandsA = commandsFor("alice"); + const commandsB = commandsFor("bob"); + const sequencesA: number[] = []; + const sequencesB: number[] = []; + for (let index = 0; index < commandsA.length; index += 1) { + sequencesA.push((yield* dispatch(a, commandsA[index]!)).sequence); + sequencesB.push((yield* dispatch(b, commandsB[index]!)).sequence); + } + sequencesA.push((yield* dispatch(a, extraThreadFor("alice"))).sequence); + + const eventsA = yield* readEvents(a); + const eventsB = yield* readEvents(b); + + // Per-user sequences start at 1 and are contiguous, so clients resume per user. + assert.deepStrictEqual( + eventsA.map((event) => event.sequence), + Array.from({ length: eventsA.length }, (_, index) => index + 1), + ); + assert.deepStrictEqual( + eventsB.map((event) => event.sequence), + Array.from({ length: eventsB.length }, (_, index) => index + 1), + ); + assert.strictEqual(sequencesA.at(-1), eventsA.length); + assert.strictEqual(sequencesB.at(-1), eventsB.length); + assert.isAbove(eventsA.length, eventsB.length); + + // Events never cross users, even though ids are identical. + // @effect-diagnostics-next-line preferSchemaOverJson:off + expect(JSON.stringify(eventsA)).not.toContain("bob"); + // @effect-diagnostics-next-line preferSchemaOverJson:off + expect(JSON.stringify(eventsB)).not.toContain("alice"); + + // Resume-by-sequence reads only the caller's later events. + const resumeFrom = 3; + const resumedA = yield* readEvents(a, resumeFrom); + assert.deepStrictEqual( + resumedA.map((event) => event.sequence), + eventsA.filter((event) => event.sequence > resumeFrom).map((event) => event.sequence), + ); + + // Shell snapshots. + const shellA = yield* withQuery(a, (query) => query.getShellSnapshot()); + const shellB = yield* withQuery(b, (query) => query.getShellSnapshot()); + assert.deepStrictEqual( + shellA.projects.map((project) => project.title), + ["Project of alice"], + ); + assert.deepStrictEqual( + shellB.projects.map((project) => project.title), + ["Project of bob"], + ); + assert.deepStrictEqual(shellA.threads.map((thread) => thread.title).toSorted(), [ + "Extra thread of alice", + "Thread of alice", + ]); + assert.deepStrictEqual( + shellB.threads.map((thread) => thread.title), + ["Thread of bob"], + ); + assert.strictEqual(shellA.snapshotSequence, eventsA.length); + assert.strictEqual(shellB.snapshotSequence, eventsB.length); + + // Thread detail snapshots: full and windowed. + for (const window of [undefined, { turnLimit: 5 }]) { + const threadA = Option.getOrThrow( + yield* withQuery(a, (query) => + query.getThreadDetailSnapshot(ThreadId.make("thread-shared"), window), + ), + ).thread; + const threadB = Option.getOrThrow( + yield* withQuery(b, (query) => + query.getThreadDetailSnapshot(ThreadId.make("thread-shared"), window), + ), + ).thread; + assert.deepStrictEqual( + threadA.messages.map((message) => message.text), + ["hello from alice", "reply to alice"], + ); + assert.deepStrictEqual( + threadB.messages.map((message) => message.text), + ["hello from bob", "reply to bob"], + ); + assert.deepStrictEqual( + threadA.activities.map((activity) => activity.summary), + ["tool ran for alice"], + ); + assert.deepStrictEqual( + threadB.activities.map((activity) => activity.summary), + ["tool ran for bob"], + ); + } + + // Counts and command receipts are tenant-scoped too: replaying user A's + // command id against user B's engine is B's own replay, not a conflict. + assert.deepStrictEqual(yield* withQuery(a, (query) => query.getCounts()), { + projectCount: 1, + threadCount: 2, + }); + assert.deepStrictEqual(yield* withQuery(b, (query) => query.getCounts()), { + projectCount: 1, + threadCount: 1, + }); + assert.strictEqual((yield* dispatch(b, commandsB[0]!)).replayed, true); + + // Evicting and reactivating A rebuilds the same command read model. + const readModelBefore = yield* withQuery(a, (query) => query.getCommandReadModel()); + yield* Scope.close(a.scope, Exit.void); + const a2 = yield* activateUser(userA); + const readModelAfter = yield* withQuery(a2, (query) => query.getCommandReadModel()); + assert.deepStrictEqual(readModelAfter, readModelBefore); + assert.strictEqual(readModelAfter.snapshotSequence, eventsA.length); + assert.strictEqual(readModelAfter.threads.length, 2); + + yield* Scope.close(a2.scope, Exit.void); + yield* Scope.close(b.scope, Exit.void); + yield* deleteUsers([userA, userB]); + }), + ); + + it.effect("row-level security backstop fails closed without a tenant context", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* deleteUsers([userA, userB]); + yield* sql` + INSERT INTO projection_projects (user_id, project_id, title, workspace_root, scripts_json, created_at, updated_at) + VALUES + (${userA}, 'rls-project', 'rls a', '/a', '[]', ${createdAt}, ${createdAt}), + (${userB}, 'rls-project', 'rls b', '/b', '[]', ${createdAt}, ${createdAt}) + `; + yield* sql` + DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'hub_rls_probe') THEN + CREATE ROLE hub_rls_probe NOLOGIN; + END IF; + END $$ + `; + yield* sql`GRANT SELECT, INSERT ON projection_projects TO hub_rls_probe`; + for (const statement of hubRowLevelSecurityStatements(["projection_projects"])) { + yield* sql.unsafe(statement); + } + + // A non-owner role inside a transaction, optionally with a tenant context. + const asProbe = (tenant: string | null, effect: Effect.Effect) => + sql.withTransaction( + Effect.gen(function* () { + yield* sql`SET LOCAL ROLE hub_rls_probe`; + if (tenant !== null) { + yield* sql`SELECT set_config('hub.user_id', ${tenant}, true)`; + } + return yield* effect; + }), + ); + const titles = (tenant: string | null) => + asProbe( + tenant, + sql<{ title: string }>` + SELECT title FROM projection_projects WHERE project_id = 'rls-project' ORDER BY title + `.pipe(Effect.map((rows) => rows.map((row) => row.title))), + ); + + yield* Effect.gen(function* () { + // No tenant context: nothing is visible, even without a user_id predicate. + assert.deepStrictEqual(yield* titles(null), []); + assert.deepStrictEqual(yield* titles(userA), ["rls a"]); + assert.deepStrictEqual(yield* titles(userB), ["rls b"]); + // Writes for another tenant are rejected by WITH CHECK. + const forged = yield* asProbe( + userA, + sql` + INSERT INTO projection_projects (user_id, project_id, title, workspace_root, scripts_json, created_at, updated_at) + VALUES (${userB}, 'rls-forged', 'forged', '/b', '[]', ${createdAt}, ${createdAt}) + `, + ).pipe(Effect.exit); + assert.isTrue(Exit.isFailure(forged)); + // The setting is transaction-local, so the next transaction starts clean. + assert.deepStrictEqual(yield* titles(null), []); + }).pipe( + Effect.ensuring( + sql`ALTER TABLE projection_projects DISABLE ROW LEVEL SECURITY`.pipe(Effect.orDie), + ), + ); + yield* deleteUsers([userA, userB]); + }), + ); + + it.effect( + "shows why schema-per-user cannot rely on session search_path over a shared connection", + () => + Effect.gen(function* () { + // One pooled connection stands in for a PgBouncer server connection that + // transaction pooling hands to whichever client runs the next transaction. + const single = yield* Layer.build( + Layer.fresh( + PgClient.layer({ + url: Redacted.make(databaseUrl), + maxConnections: 1, + types: hubPgTypes, + }), + ), + ); + const db = Context.get(single, SqlClient.SqlClient); + for (const schema of ["hub_probe_a", "hub_probe_b"]) { + yield* db`CREATE SCHEMA IF NOT EXISTS ${db(schema)}`; + yield* db`CREATE TABLE IF NOT EXISTS ${db(schema)}.probe_threads (title text)`; + yield* db`TRUNCATE ${db(schema)}.probe_threads`; + yield* db`INSERT INTO ${db(schema)}.probe_threads VALUES (${schema})`; + } + // Tenant A sets its schema for the session... + yield* db`SET search_path TO hub_probe_a`; + // ...and tenant B's next statement on that connection reads A's data. + const leaked = yield* db<{ title: string }>`SELECT title FROM probe_threads`; + yield* db`RESET search_path`; + // SET LOCAL is scoped to one transaction, which is safe behind a pooler, + // but then every statement must run inside such a transaction. + const scoped = yield* db.withTransaction( + Effect.gen(function* () { + yield* db`SET LOCAL search_path TO hub_probe_b`; + return yield* db<{ title: string }>`SELECT title FROM probe_threads`; + }), + ); + const after = yield* db<{ search_path: string }>`SHOW search_path`; + yield* db`DROP SCHEMA hub_probe_a CASCADE`; + yield* db`DROP SCHEMA hub_probe_b CASCADE`; + + assert.deepStrictEqual( + leaked.map((row) => row.title), + ["hub_probe_a"], + ); + assert.deepStrictEqual( + scoped.map((row) => row.title), + ["hub_probe_b"], + ); + assert.strictEqual(after[0]?.search_path, '"$user", public'); + }), + ); + }); +} diff --git a/apps/server/src/persistence/Postgres/HubEngine.ts b/apps/server/src/persistence/Postgres/HubEngine.ts new file mode 100644 index 000000000000..6e185a9c2894 --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubEngine.ts @@ -0,0 +1,92 @@ +/** + * Per-user orchestration engine over the shared hub Postgres pool. + * + * The hub keeps T3's one-writer engine model per user: each active user gets an + * engine (command queue, in-memory command read model, event PubSub) whose + * repositories are scoped by `HubTenant`. All engines in a process share one + * `SqlClient`, so the pool size is independent of the number of active users. + */ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { ServerConfig } from "../../config.ts"; +import { makeHubPostgresPersistenceLive, type HubPostgresConfig } from "../Layers/Postgres.ts"; +import { OrchestrationEngineLive } from "../../orchestration/Layers/OrchestrationEngine.ts"; +import { makeOrchestrationProjectionPipeline } from "../../orchestration/Layers/ProjectionPipeline.ts"; +import { OrchestrationProjectionPipeline } from "../../orchestration/Services/ProjectionPipeline.ts"; +import * as ThreadBackgroundLiveness from "../../orchestration/ThreadBackgroundLiveness.ts"; +import * as ThreadPlanProgress from "../../orchestration/ThreadPlanProgress.ts"; +import { RepositoryIdentityResolver } from "../../project/RepositoryIdentityResolver.ts"; +import { hubTenantLayer } from "./HubTenant.ts"; +import { PgOrchestrationCommandReceiptRepositoryLive } from "./OrchestrationCommandReceipts.ts"; +import { PgOrchestrationEventStoreLive } from "./OrchestrationEventStore.ts"; +import { PgProjectionPendingApprovalRepositoryLive } from "./ProjectionPendingApprovals.ts"; +import { PgProjectionProjectRepositoryLive } from "./ProjectionProjects.ts"; +import { PgProjectionSnapshotQueryLive } from "./ProjectionSnapshotQuery.ts"; +import { PgProjectionStateRepositoryLive } from "./ProjectionState.ts"; +import { PgProjectionThreadActivityRepositoryLive } from "./ProjectionThreadActivities.ts"; +import { PgProjectionThreadMessageRepositoryLive } from "./ProjectionThreadMessages.ts"; +import { PgProjectionThreadProposedPlanRepositoryLive } from "./ProjectionThreadProposedPlans.ts"; +import { PgProjectionThreadSessionRepositoryLive } from "./ProjectionThreadSessions.ts"; +import { PgProjectionThreadRepositoryLive } from "./ProjectionThreads.ts"; +import { PgProjectionTurnRepositoryLive } from "./ProjectionTurns.ts"; + +export const PgOrchestrationProjectionPipelineLive = Layer.effect( + OrchestrationProjectionPipeline, + makeOrchestrationProjectionPipeline(), +).pipe( + Layer.provideMerge(PgProjectionProjectRepositoryLive), + Layer.provideMerge(PgProjectionThreadRepositoryLive), + Layer.provideMerge(PgProjectionThreadMessageRepositoryLive), + Layer.provideMerge(PgProjectionThreadProposedPlanRepositoryLive), + Layer.provideMerge(PgProjectionThreadActivityRepositoryLive), + Layer.provideMerge(PgProjectionThreadSessionRepositoryLive), + Layer.provideMerge(PgProjectionTurnRepositoryLive), + Layer.provideMerge(PgProjectionPendingApprovalRepositoryLive), + Layer.provideMerge(PgProjectionStateRepositoryLive), +); + +// The hub has no repositories on disk. Repository identity moves onto the +// project record (see the design doc); until then projects report none. +const NoRepositoryIdentityLive = Layer.succeed(RepositoryIdentityResolver, { + resolve: () => Effect.succeed(null), +}); + +/** Process-wide services every user engine shares: the pool and the platform. */ +export type HubSharedServices = SqlClient.SqlClient | NodeServices.NodeServices; + +export const makeHubSharedLayer = (config: HubPostgresConfig) => + Layer.merge(makeHubPostgresPersistenceLive(config), NodeServices.layer); + +/** + * Engine + snapshot query for one user over the process-wide services. + * ServerConfig is only used for attachment side effects, which move to object + * storage in the hub; a per-user temp config keeps the prototype self-contained. + */ +export const makeHubUserEngineLayer = (input: { + readonly userId: string; + readonly shared: Context.Context; +}) => { + const serverConfigLayer = ServerConfig.layerTest(process.cwd(), { + prefix: `t3-hub-${input.userId.replace(/[^a-zA-Z0-9_-]/g, "_")}-`, + }); + return Layer.mergeAll( + OrchestrationEngineLive.pipe( + Layer.provide(PgProjectionSnapshotQueryLive), + Layer.provide(PgOrchestrationProjectionPipelineLive), + ), + PgProjectionSnapshotQueryLive, + ).pipe( + Layer.provideMerge(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide(PgOrchestrationEventStoreLive), + Layer.provideMerge(PgOrchestrationCommandReceiptRepositoryLive), + Layer.provide(NoRepositoryIdentityLive), + Layer.provide(hubTenantLayer(input.userId)), + Layer.provideMerge(serverConfigLayer), + Layer.provideMerge(Layer.succeedContext(input.shared)), + ); +}; diff --git a/apps/server/src/persistence/Postgres/HubTenant.ts b/apps/server/src/persistence/Postgres/HubTenant.ts new file mode 100644 index 000000000000..d7351010cbbd --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubTenant.ts @@ -0,0 +1,18 @@ +import * as Context from "effect/Context"; +import * as Layer from "effect/Layer"; + +/** + * The hub user whose rows a Postgres repository may read or write. + * + * Every hub repository resolves this once at construction and adds + * `user_id = ${userId}` to each statement, so one process can run many users' + * engines over a single shared connection pool. + */ +export class HubTenant extends Context.Service< + HubTenant, + { + readonly userId: string; + } +>()("t3/persistence/Postgres/HubTenant") {} + +export const hubTenantLayer = (userId: string) => Layer.succeed(HubTenant, { userId }); diff --git a/apps/server/src/persistence/Postgres/OrchestrationCommandReceipts.ts b/apps/server/src/persistence/Postgres/OrchestrationCommandReceipts.ts new file mode 100644 index 000000000000..144e1be4607f --- /dev/null +++ b/apps/server/src/persistence/Postgres/OrchestrationCommandReceipts.ts @@ -0,0 +1,95 @@ +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import { toPersistenceSqlError } from "../Errors.ts"; +import { HubTenant } from "./HubTenant.ts"; + +import { + GetByCommandIdInput, + OrchestrationCommandReceipt, + OrchestrationCommandReceiptRepository, + type OrchestrationCommandReceiptRepositoryShape, +} from "../Services/OrchestrationCommandReceipts.ts"; + +const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const upsertReceiptRow = SqlSchema.void({ + Request: OrchestrationCommandReceipt, + execute: (receipt) => + sql` + INSERT INTO orchestration_command_receipts ( + user_id, + command_id, + aggregate_kind, + aggregate_id, + accepted_at, + result_sequence, + status, + error + ) + VALUES ( + ${userId}, + ${receipt.commandId}, + ${receipt.aggregateKind}, + ${receipt.aggregateId}, + ${receipt.acceptedAt}, + ${receipt.resultSequence}, + ${receipt.status}, + ${receipt.error} + ) + ON CONFLICT (user_id, command_id) + DO UPDATE SET + aggregate_kind = excluded.aggregate_kind, + aggregate_id = excluded.aggregate_id, + accepted_at = excluded.accepted_at, + result_sequence = excluded.result_sequence, + status = excluded.status, + error = excluded.error + `, + }); + + const findReceiptByCommandId = SqlSchema.findOneOption({ + Request: GetByCommandIdInput, + Result: OrchestrationCommandReceipt, + execute: ({ commandId }) => + sql` + SELECT + command_id AS "commandId", + aggregate_kind AS "aggregateKind", + aggregate_id AS "aggregateId", + accepted_at AS "acceptedAt", + result_sequence AS "resultSequence", + status, + error + FROM orchestration_command_receipts + WHERE user_id = ${userId} + AND command_id = ${commandId} + `, + }); + + const upsert: OrchestrationCommandReceiptRepositoryShape["upsert"] = (receipt) => + upsertReceiptRow(receipt).pipe( + Effect.mapError(toPersistenceSqlError("OrchestrationCommandReceiptRepository.upsert:query")), + ); + + const getByCommandId: OrchestrationCommandReceiptRepositoryShape["getByCommandId"] = (input) => + findReceiptByCommandId(input).pipe( + Effect.mapError( + toPersistenceSqlError("OrchestrationCommandReceiptRepository.getByCommandId:query"), + ), + ); + + return { + upsert, + getByCommandId, + } satisfies OrchestrationCommandReceiptRepositoryShape; +}); + +export const PgOrchestrationCommandReceiptRepositoryLive = Layer.effect( + OrchestrationCommandReceiptRepository, + makeOrchestrationCommandReceiptRepository, +); diff --git a/apps/server/src/persistence/Postgres/OrchestrationEventStore.ts b/apps/server/src/persistence/Postgres/OrchestrationEventStore.ts new file mode 100644 index 000000000000..e4d237069868 --- /dev/null +++ b/apps/server/src/persistence/Postgres/OrchestrationEventStore.ts @@ -0,0 +1,427 @@ +import { + CommandId, + EventId, + IsoDateTime, + NonNegativeInt, + OrchestrationActorKind, + OrchestrationAggregateKind, + OrchestrationEvent, + OrchestrationEventMetadata, + OrchestrationEventType, + ProjectId, + ThreadId, +} from "@t3tools/contracts"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +import { + toPersistenceDecodeError, + toPersistenceSqlError, + type OrchestrationEventStoreError, +} from "../Errors.ts"; +import { + OrchestrationEventStore, + type OrchestrationEventStoreShape, +} from "../Services/OrchestrationEventStore.ts"; +import { HubTenant } from "./HubTenant.ts"; + +const decodeEvent = Schema.decodeUnknownEffect(OrchestrationEvent); +const UnknownFromJsonString = Schema.fromJsonString(Schema.Unknown); +const EventMetadataFromJsonString = Schema.fromJsonString(OrchestrationEventMetadata); + +const AppendEventRequestSchema = Schema.Struct({ + eventId: EventId, + aggregateKind: OrchestrationAggregateKind, + streamId: Schema.Union([ProjectId, ThreadId]), + type: OrchestrationEventType, + causationEventId: Schema.NullOr(EventId), + correlationId: Schema.NullOr(CommandId), + actorKind: OrchestrationActorKind, + occurredAt: IsoDateTime, + commandId: Schema.NullOr(CommandId), + payloadJson: UnknownFromJsonString, + metadataJson: EventMetadataFromJsonString, +}); + +const OrchestrationEventPersistedRowSchema = Schema.Struct({ + sequence: NonNegativeInt, + eventId: EventId, + type: OrchestrationEventType, + aggregateKind: OrchestrationAggregateKind, + aggregateId: Schema.Union([ProjectId, ThreadId]), + occurredAt: IsoDateTime, + commandId: Schema.NullOr(CommandId), + causationEventId: Schema.NullOr(EventId), + correlationId: Schema.NullOr(CommandId), + payload: UnknownFromJsonString, + metadata: EventMetadataFromJsonString, +}); + +const HasEventAfterRequestSchema = Schema.Struct({ + aggregateKind: Schema.String, + aggregateId: Schema.String, + type: Schema.optional(Schema.String), + sequenceExclusive: NonNegativeInt, +}); + +const ReadFromSequenceRequestSchema = Schema.Struct({ + sequenceExclusive: NonNegativeInt, + limit: Schema.Number, +}); +const ReadCreatedEventRequestSchema = Schema.Struct({ + aggregateKind: OrchestrationAggregateKind, + aggregateId: Schema.Union([ProjectId, ThreadId]), +}); +const DEFAULT_READ_FROM_SEQUENCE_LIMIT = 1_000; +const READ_PAGE_SIZE = 500; + +function inferActorKind( + event: Omit, +): Schema.Schema.Type { + if (event.commandId !== null && event.commandId.startsWith("provider:")) { + return "provider"; + } + if (event.commandId !== null && event.commandId.startsWith("server:")) { + return "server"; + } + if ( + event.metadata.providerTurnId !== undefined || + event.metadata.providerItemId !== undefined || + event.metadata.adapterKey !== undefined + ) { + return "provider"; + } + if (event.commandId === null) { + return "server"; + } + return "client"; +} + +function toPersistenceSqlOrDecodeError(sqlOperation: string, decodeOperation: string) { + return (cause: unknown): OrchestrationEventStoreError => + Schema.isSchemaError(cause) + ? toPersistenceDecodeError(decodeOperation)(cause) + : toPersistenceSqlError(sqlOperation)(cause); +} + +const makeEventStore = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + // SQLite's global AUTOINCREMENT becomes a per-user counter. The counter row is + // locked by the append transaction, so appends for one user serialize and the + // sequence stays gap-free (a rolled-back append rolls the counter back too). + const appendEventRow = SqlSchema.findOne({ + Request: AppendEventRequestSchema, + Result: OrchestrationEventPersistedRowSchema, + execute: (request) => + sql` + WITH next_sequence AS ( + INSERT INTO hub_users (user_id, last_event_sequence) + VALUES (${userId}, 1) + ON CONFLICT (user_id) + DO UPDATE SET last_event_sequence = hub_users.last_event_sequence + 1 + RETURNING last_event_sequence + ) + INSERT INTO orchestration_events ( + user_id, + sequence, + event_id, + aggregate_kind, + stream_id, + stream_version, + event_type, + occurred_at, + command_id, + causation_event_id, + correlation_id, + actor_kind, + payload_json, + metadata_json + ) + VALUES ( + ${userId}, + (SELECT last_event_sequence FROM next_sequence), + ${request.eventId}, + ${request.aggregateKind}, + ${request.streamId}, + COALESCE( + ( + SELECT stream_version + 1 + FROM orchestration_events + WHERE user_id = ${userId} + AND aggregate_kind = ${request.aggregateKind} + AND stream_id = ${request.streamId} + ORDER BY stream_version DESC + LIMIT 1 + ), + 0 + ), + ${request.type}, + ${request.occurredAt}, + ${request.commandId}, + ${request.causationEventId}, + ${request.correlationId}, + ${request.actorKind}, + ${request.payloadJson}, + ${request.metadataJson} + ) + RETURNING + sequence, + event_id AS "eventId", + event_type AS "type", + aggregate_kind AS "aggregateKind", + stream_id AS "aggregateId", + occurred_at AS "occurredAt", + command_id AS "commandId", + causation_event_id AS "causationEventId", + correlation_id AS "correlationId", + payload_json AS "payload", + metadata_json AS "metadata" + `, + }); + + const readEventRowsFromSequence = SqlSchema.findAll({ + Request: ReadFromSequenceRequestSchema, + Result: OrchestrationEventPersistedRowSchema, + execute: (request) => + sql` + SELECT + sequence, + event_id AS "eventId", + event_type AS "type", + aggregate_kind AS "aggregateKind", + stream_id AS "aggregateId", + occurred_at AS "occurredAt", + command_id AS "commandId", + causation_event_id AS "causationEventId", + correlation_id AS "correlationId", + payload_json AS "payload", + metadata_json AS "metadata" + FROM orchestration_events + WHERE user_id = ${userId} + AND sequence > ${request.sequenceExclusive} + ORDER BY sequence ASC + LIMIT ${request.limit} + `, + }); + + const readCreatedEventRows = SqlSchema.findAll({ + Request: ReadCreatedEventRequestSchema, + Result: OrchestrationEventPersistedRowSchema, + execute: (request) => + sql` + SELECT + sequence, + event_id AS "eventId", + event_type AS "type", + aggregate_kind AS "aggregateKind", + stream_id AS "aggregateId", + occurred_at AS "occurredAt", + command_id AS "commandId", + causation_event_id AS "causationEventId", + correlation_id AS "correlationId", + payload_json AS "payload", + metadata_json AS "metadata" + FROM orchestration_events + WHERE user_id = ${userId} + AND aggregate_kind = ${request.aggregateKind} + AND stream_id = ${request.aggregateId} + AND event_type IN ('project.created', 'thread.created') + ORDER BY stream_version DESC + LIMIT 1 + `, + }); + + const readDeletedEventRows = SqlSchema.findAll({ + Request: ReadCreatedEventRequestSchema, + Result: OrchestrationEventPersistedRowSchema, + execute: (request) => + sql` + SELECT + sequence, + event_id AS "eventId", + event_type AS "type", + aggregate_kind AS "aggregateKind", + stream_id AS "aggregateId", + occurred_at AS "occurredAt", + command_id AS "commandId", + causation_event_id AS "causationEventId", + correlation_id AS "correlationId", + payload_json AS "payload", + metadata_json AS "metadata" + FROM orchestration_events + WHERE user_id = ${userId} + AND aggregate_kind = 'thread' + AND stream_id = ${request.aggregateId} + AND event_type = 'thread.deleted' + ORDER BY stream_version DESC + LIMIT 1 + `, + }); + + const append: OrchestrationEventStoreShape["append"] = (event) => + appendEventRow({ + eventId: event.eventId, + aggregateKind: event.aggregateKind, + streamId: event.aggregateId, + type: event.type, + causationEventId: event.causationEventId, + correlationId: event.correlationId, + actorKind: inferActorKind(event), + occurredAt: event.occurredAt, + commandId: event.commandId, + payloadJson: event.payload, + metadataJson: event.metadata, + }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "OrchestrationEventStore.append:insert", + "OrchestrationEventStore.append:decodeRow", + ), + ), + Effect.flatMap((row) => + decodeEvent(row).pipe( + Effect.mapError(toPersistenceDecodeError("OrchestrationEventStore.append:rowToEvent")), + ), + ), + ); + + const readFromSequence: OrchestrationEventStoreShape["readFromSequence"] = ( + sequenceExclusive, + limit = DEFAULT_READ_FROM_SEQUENCE_LIMIT, + ) => { + const normalizedLimit = Math.max(0, Math.floor(limit)); + if (normalizedLimit === 0) { + return Stream.empty; + } + const readPage = ( + cursor: number, + remaining: number, + ): Stream.Stream => + Stream.fromEffect( + readEventRowsFromSequence({ + sequenceExclusive: cursor, + limit: Math.min(remaining, READ_PAGE_SIZE), + }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "OrchestrationEventStore.readFromSequence:query", + "OrchestrationEventStore.readFromSequence:decodeRows", + ), + ), + Effect.flatMap((rows) => + Effect.forEach(rows, (row) => + decodeEvent(row).pipe( + Effect.mapError( + toPersistenceDecodeError("OrchestrationEventStore.readFromSequence:rowToEvent"), + ), + ), + ), + ), + ), + ).pipe( + Stream.flatMap((events) => { + if (events.length === 0) { + return Stream.empty; + } + const nextRemaining = remaining - events.length; + if (nextRemaining <= 0) { + return Stream.fromIterable(events); + } + return Stream.concat( + Stream.fromIterable(events), + readPage(events[events.length - 1]!.sequence, nextRemaining), + ); + }), + ); + + return readPage(sequenceExclusive, normalizedLimit); + }; + + const readCreatedEvent: OrchestrationEventStoreShape["readCreatedEvent"] = ( + aggregateKind, + aggregateId, + ) => + readCreatedEventRows({ aggregateKind, aggregateId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "OrchestrationEventStore.readCreatedEvent:query", + "OrchestrationEventStore.readCreatedEvent:decodeRows", + ), + ), + Effect.flatMap((rows) => { + const row = rows[0]; + return row === undefined + ? Effect.succeed(null) + : decodeEvent(row).pipe( + Effect.mapError( + toPersistenceDecodeError("OrchestrationEventStore.readCreatedEvent:rowToEvent"), + ), + ); + }), + ); + + const readDeletedEvent: OrchestrationEventStoreShape["readDeletedEvent"] = (threadId) => + readDeletedEventRows({ aggregateKind: "thread", aggregateId: threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "OrchestrationEventStore.readDeletedEvent:query", + "OrchestrationEventStore.readDeletedEvent:decodeRows", + ), + ), + Effect.flatMap((rows) => { + const row = rows[0]; + if (row === undefined) return Effect.succeed(null); + return decodeEvent(row).pipe( + Effect.mapError( + toPersistenceDecodeError("OrchestrationEventStore.readDeletedEvent:rowToEvent"), + ), + ); + }), + ); + + const findEventAfter = SqlSchema.findOneOption({ + Request: HasEventAfterRequestSchema, + Result: Schema.Struct({ sequence: Schema.Number }), + execute: (request) => sql` + SELECT sequence + FROM orchestration_events + WHERE user_id = ${userId} + AND aggregate_kind = ${request.aggregateKind} + AND stream_id = ${request.aggregateId} + AND ${sql.and([ + sql`sequence > ${request.sequenceExclusive}`, + ...(request.type === undefined ? [] : [sql`event_type = ${request.type}`]), + ])} + LIMIT 1 + `, + }); + + const hasEventAfter: OrchestrationEventStoreShape["hasEventAfter"] = (input) => + findEventAfter(input).pipe( + Effect.map(Option.isSome), + Effect.mapError( + toPersistenceSqlOrDecodeError( + "OrchestrationEventStore.hasEventAfter:query", + "OrchestrationEventStore.hasEventAfter:decodeRow", + ), + ), + ); + + return { + append, + readFromSequence, + readAll: () => readFromSequence(0, Number.MAX_SAFE_INTEGER), + readCreatedEvent, + readDeletedEvent, + hasEventAfter, + } satisfies OrchestrationEventStoreShape; +}); + +export const PgOrchestrationEventStoreLive = Layer.effect(OrchestrationEventStore, makeEventStore); diff --git a/apps/server/src/persistence/Postgres/ProjectionCheckpoints.ts b/apps/server/src/persistence/Postgres/ProjectionCheckpoints.ts new file mode 100644 index 000000000000..ea0d1d19f654 --- /dev/null +++ b/apps/server/src/persistence/Postgres/ProjectionCheckpoints.ts @@ -0,0 +1,225 @@ +import { OrchestrationCheckpointFile } from "@t3tools/contracts"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as Struct from "effect/Struct"; + +import { toPersistenceDecodeError, toPersistenceSqlError } from "../Errors.ts"; +import { HubTenant } from "./HubTenant.ts"; +import { + DeleteByThreadIdInput, + GetByThreadAndTurnCountInput, + ListByThreadIdInput, + ProjectionCheckpoint, + ProjectionCheckpointRepository, + type ProjectionCheckpointRepositoryShape, +} from "../Services/ProjectionCheckpoints.ts"; + +const ProjectionCheckpointDbRowSchema = ProjectionCheckpoint.mapFields( + Struct.assign({ + files: Schema.fromJsonString(Schema.Array(OrchestrationCheckpointFile)), + }), +); + +function toPersistenceSqlOrDecodeError(sqlOperation: string, decodeOperation: string) { + return (cause: unknown) => + Schema.isSchemaError(cause) + ? toPersistenceDecodeError(decodeOperation)(cause) + : toPersistenceSqlError(sqlOperation)(cause); +} + +const makeProjectionCheckpointRepository = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const clearCheckpointConflict = SqlSchema.void({ + Request: GetByThreadAndTurnCountInput, + execute: ({ threadId, checkpointTurnCount }) => + sql` + UPDATE projection_turns + SET + checkpoint_turn_count = NULL, + checkpoint_ref = NULL, + checkpoint_status = NULL, + checkpoint_files_json = '[]' + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND checkpoint_turn_count = ${checkpointTurnCount} + `, + }); + + const upsertProjectionCheckpointRow = SqlSchema.void({ + Request: ProjectionCheckpointDbRowSchema, + execute: (row) => + sql` + INSERT INTO projection_turns ( + user_id, + thread_id, + turn_id, + pending_message_id, + assistant_message_id, + state, + requested_at, + started_at, + completed_at, + checkpoint_turn_count, + checkpoint_ref, + checkpoint_status, + checkpoint_files_json + ) + VALUES ( + ${userId}, + ${row.threadId}, + ${row.turnId}, + NULL, + ${row.assistantMessageId}, + ${row.status === "error" ? "error" : "completed"}, + ${row.completedAt}, + ${row.completedAt}, + ${row.completedAt}, + ${row.checkpointTurnCount}, + ${row.checkpointRef}, + ${row.status}, + ${row.files} + ) + ON CONFLICT (user_id, thread_id, turn_id) + DO UPDATE SET + assistant_message_id = excluded.assistant_message_id, + state = excluded.state, + completed_at = excluded.completed_at, + checkpoint_turn_count = excluded.checkpoint_turn_count, + checkpoint_ref = excluded.checkpoint_ref, + checkpoint_status = excluded.checkpoint_status, + checkpoint_files_json = excluded.checkpoint_files_json + `, + }); + + const listProjectionCheckpointRows = SqlSchema.findAll({ + Request: ListByThreadIdInput, + Result: ProjectionCheckpointDbRowSchema, + execute: ({ threadId }) => + sql` + SELECT + thread_id AS "threadId", + turn_id AS "turnId", + checkpoint_turn_count AS "checkpointTurnCount", + checkpoint_ref AS "checkpointRef", + checkpoint_status AS "status", + checkpoint_files_json AS "files", + assistant_message_id AS "assistantMessageId", + completed_at AS "completedAt" + FROM projection_turns + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND checkpoint_turn_count IS NOT NULL + ORDER BY checkpoint_turn_count ASC + `, + }); + + const getProjectionCheckpointRow = SqlSchema.findOneOption({ + Request: GetByThreadAndTurnCountInput, + Result: ProjectionCheckpointDbRowSchema, + execute: ({ threadId, checkpointTurnCount }) => + sql` + SELECT + thread_id AS "threadId", + turn_id AS "turnId", + checkpoint_turn_count AS "checkpointTurnCount", + checkpoint_ref AS "checkpointRef", + checkpoint_status AS "status", + checkpoint_files_json AS "files", + assistant_message_id AS "assistantMessageId", + completed_at AS "completedAt" + FROM projection_turns + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND checkpoint_turn_count = ${checkpointTurnCount} + `, + }); + + const deleteProjectionCheckpointRows = SqlSchema.void({ + Request: DeleteByThreadIdInput, + execute: ({ threadId }) => + sql` + UPDATE projection_turns + SET + checkpoint_turn_count = NULL, + checkpoint_ref = NULL, + checkpoint_status = NULL, + checkpoint_files_json = '[]' + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND checkpoint_turn_count IS NOT NULL + `, + }); + + const upsertCheckpointRow = (row: Schema.Schema.Type) => + sql.withTransaction( + clearCheckpointConflict({ + threadId: row.threadId, + checkpointTurnCount: row.checkpointTurnCount, + }).pipe(Effect.flatMap(() => upsertProjectionCheckpointRow(row))), + ); + + const upsert: ProjectionCheckpointRepositoryShape["upsert"] = (row) => + upsertCheckpointRow(row).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionCheckpointRepository.upsert:query", + "ProjectionCheckpointRepository.upsert:encodeRequest", + ), + ), + ); + + const listByThreadId: ProjectionCheckpointRepositoryShape["listByThreadId"] = (input) => + listProjectionCheckpointRows(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionCheckpointRepository.listByThreadId:query", + "ProjectionCheckpointRepository.listByThreadId:decodeRows", + ), + ), + Effect.map((rows) => rows as ReadonlyArray>), + ); + + const getByThreadAndTurnCount: ProjectionCheckpointRepositoryShape["getByThreadAndTurnCount"] = ( + input, + ) => + getProjectionCheckpointRow(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionCheckpointRepository.getByThreadAndTurnCount:query", + "ProjectionCheckpointRepository.getByThreadAndTurnCount:decodeRow", + ), + ), + Effect.flatMap((rowOption) => + Option.match(rowOption, { + onNone: () => Effect.succeed(Option.none()), + onSome: (row) => + Effect.succeed(Option.some(row as Schema.Schema.Type)), + }), + ), + ); + + const deleteByThreadId: ProjectionCheckpointRepositoryShape["deleteByThreadId"] = (input) => + deleteProjectionCheckpointRows(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionCheckpointRepository.deleteByThreadId:query"), + ), + ); + + return { + upsert, + listByThreadId, + getByThreadAndTurnCount, + deleteByThreadId, + } satisfies ProjectionCheckpointRepositoryShape; +}); + +export const PgProjectionCheckpointRepositoryLive = Layer.effect( + ProjectionCheckpointRepository, + makeProjectionCheckpointRepository, +); diff --git a/apps/server/src/persistence/Postgres/ProjectionPendingApprovals.ts b/apps/server/src/persistence/Postgres/ProjectionPendingApprovals.ts new file mode 100644 index 000000000000..2b524afcb278 --- /dev/null +++ b/apps/server/src/persistence/Postgres/ProjectionPendingApprovals.ts @@ -0,0 +1,162 @@ +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import { toPersistenceSqlError } from "../Errors.ts"; +import { HubTenant } from "./HubTenant.ts"; +import { + GetProjectionPendingApprovalInput, + DeleteProjectionPendingApprovalInput, + ListProjectionPendingApprovalsInput, + ProjectionPendingApproval, + ProjectionPendingApprovalRepository, + type ProjectionPendingApprovalRepositoryShape, +} from "../Services/ProjectionPendingApprovals.ts"; + +const makeProjectionPendingApprovalRepository = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const upsertProjectionPendingApprovalRow = SqlSchema.void({ + Request: ProjectionPendingApproval, + execute: (row) => + sql` + INSERT INTO projection_pending_approvals ( + user_id, + request_id, + thread_id, + turn_id, + status, + decision, + created_at, + resolved_at + ) + VALUES ( + ${userId}, + ${row.requestId}, + ${row.threadId}, + ${row.turnId}, + ${row.status}, + ${row.decision}, + ${row.createdAt}, + ${row.resolvedAt} + ) + ON CONFLICT (user_id, request_id) + DO UPDATE SET + thread_id = excluded.thread_id, + turn_id = excluded.turn_id, + status = excluded.status, + decision = excluded.decision, + created_at = excluded.created_at, + resolved_at = excluded.resolved_at + `, + }); + + const listProjectionPendingApprovalRows = SqlSchema.findAll({ + Request: ListProjectionPendingApprovalsInput, + Result: ProjectionPendingApproval, + execute: ({ threadId }) => + sql` + SELECT + request_id AS "requestId", + thread_id AS "threadId", + turn_id AS "turnId", + status, + decision, + created_at AS "createdAt", + resolved_at AS "resolvedAt" + FROM projection_pending_approvals + WHERE user_id = ${userId} + AND thread_id = ${threadId} + ORDER BY created_at ASC, request_id ASC + `, + }); + + const getProjectionPendingApprovalRow = SqlSchema.findOneOption({ + Request: GetProjectionPendingApprovalInput, + Result: ProjectionPendingApproval, + execute: ({ requestId }) => + sql` + SELECT + request_id AS "requestId", + thread_id AS "threadId", + turn_id AS "turnId", + status, + decision, + created_at AS "createdAt", + resolved_at AS "resolvedAt" + FROM projection_pending_approvals + WHERE user_id = ${userId} + AND request_id = ${requestId} + `, + }); + + const deleteProjectionPendingApprovalRow = SqlSchema.void({ + Request: DeleteProjectionPendingApprovalInput, + execute: ({ requestId }) => + sql` + DELETE FROM projection_pending_approvals + WHERE user_id = ${userId} + AND request_id = ${requestId} + `, + }); + + const deleteProjectionPendingApprovalRowsByThread = SqlSchema.void({ + Request: ListProjectionPendingApprovalsInput, + execute: ({ threadId }) => + sql` + DELETE FROM projection_pending_approvals + WHERE user_id = ${userId} + AND thread_id = ${threadId} + `, + }); + + const upsert: ProjectionPendingApprovalRepositoryShape["upsert"] = (row) => + upsertProjectionPendingApprovalRow(row).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionPendingApprovalRepository.upsert:query")), + ); + + const listByThreadId: ProjectionPendingApprovalRepositoryShape["listByThreadId"] = (input) => + listProjectionPendingApprovalRows(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionPendingApprovalRepository.listByThreadId:query"), + ), + ); + + const getByRequestId: ProjectionPendingApprovalRepositoryShape["getByRequestId"] = (input) => + getProjectionPendingApprovalRow(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionPendingApprovalRepository.getByRequestId:query"), + ), + ); + + const deleteByRequestId: ProjectionPendingApprovalRepositoryShape["deleteByRequestId"] = ( + input, + ) => + deleteProjectionPendingApprovalRow(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionPendingApprovalRepository.deleteByRequestId:query"), + ), + ); + + const deleteByThreadId: ProjectionPendingApprovalRepositoryShape["deleteByThreadId"] = (input) => + deleteProjectionPendingApprovalRowsByThread(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionPendingApprovalRepository.deleteByThreadId:query"), + ), + ); + + return { + upsert, + listByThreadId, + getByRequestId, + deleteByRequestId, + deleteByThreadId, + } satisfies ProjectionPendingApprovalRepositoryShape; +}); + +export const PgProjectionPendingApprovalRepositoryLive = Layer.effect( + ProjectionPendingApprovalRepository, + makeProjectionPendingApprovalRepository, +); diff --git a/apps/server/src/persistence/Postgres/ProjectionProjects.ts b/apps/server/src/persistence/Postgres/ProjectionProjects.ts new file mode 100644 index 000000000000..75c7073d7165 --- /dev/null +++ b/apps/server/src/persistence/Postgres/ProjectionProjects.ts @@ -0,0 +1,160 @@ +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Struct from "effect/Struct"; + +import { ModelSelection, ProjectScript } from "@t3tools/contracts"; +import { toPersistenceSqlError } from "../Errors.ts"; +import { HubTenant } from "./HubTenant.ts"; +import { + DeleteProjectionProjectInput, + GetProjectionProjectInput, + ProjectionProject, + ProjectionProjectRepository, + type ProjectionProjectRepositoryShape, +} from "../Services/ProjectionProjects.ts"; + +const ProjectionProjectDbRow = ProjectionProject.mapFields( + Struct.assign({ + defaultModelSelection: Schema.NullOr(Schema.fromJsonString(ModelSelection)), + scripts: Schema.fromJsonString(Schema.Array(ProjectScript)), + }), +); +type ProjectionProjectDbRow = typeof ProjectionProjectDbRow.Type; + +const makeProjectionProjectRepository = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const upsertProjectionProjectRow = SqlSchema.void({ + Request: ProjectionProject, + execute: (row) => + sql` + INSERT INTO projection_projects ( + user_id, + project_id, + title, + workspace_root, + default_model_selection_json, + default_thread_env_mode, + favicon_path, + scripts_json, + created_at, + updated_at, + deleted_at + ) + VALUES ( + ${userId}, + ${row.projectId}, + ${row.title}, + ${row.workspaceRoot}, + ${row.defaultModelSelection !== null ? JSON.stringify(row.defaultModelSelection) : null}, + ${row.defaultThreadEnvMode}, + ${row.faviconPath ?? null}, + ${JSON.stringify(row.scripts)}, + ${row.createdAt}, + ${row.updatedAt}, + ${row.deletedAt} + ) + ON CONFLICT (user_id, project_id) + DO UPDATE SET + title = excluded.title, + workspace_root = excluded.workspace_root, + default_model_selection_json = excluded.default_model_selection_json, + default_thread_env_mode = excluded.default_thread_env_mode, + favicon_path = excluded.favicon_path, + scripts_json = excluded.scripts_json, + created_at = excluded.created_at, + updated_at = excluded.updated_at, + deleted_at = excluded.deleted_at + `, + }); + + const getProjectionProjectRow = SqlSchema.findOneOption({ + Request: GetProjectionProjectInput, + Result: ProjectionProjectDbRow, + execute: ({ projectId }) => + sql` + SELECT + project_id AS "projectId", + title, + workspace_root AS "workspaceRoot", + default_model_selection_json AS "defaultModelSelection", + default_thread_env_mode AS "defaultThreadEnvMode", + favicon_path AS "faviconPath", + scripts_json AS "scripts", + created_at AS "createdAt", + updated_at AS "updatedAt", + deleted_at AS "deletedAt" + FROM projection_projects + WHERE user_id = ${userId} + AND project_id = ${projectId} + `, + }); + + const listProjectionProjectRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionProjectDbRow, + execute: () => + sql` + SELECT + project_id AS "projectId", + title, + workspace_root AS "workspaceRoot", + default_model_selection_json AS "defaultModelSelection", + default_thread_env_mode AS "defaultThreadEnvMode", + favicon_path AS "faviconPath", + scripts_json AS "scripts", + created_at AS "createdAt", + updated_at AS "updatedAt", + deleted_at AS "deletedAt" + FROM projection_projects + WHERE user_id = ${userId} + ORDER BY created_at ASC, project_id ASC + `, + }); + + const deleteProjectionProjectRow = SqlSchema.void({ + Request: DeleteProjectionProjectInput, + execute: ({ projectId }) => + sql` + DELETE FROM projection_projects + WHERE user_id = ${userId} + AND project_id = ${projectId} + `, + }); + + const upsert: ProjectionProjectRepositoryShape["upsert"] = (row) => + upsertProjectionProjectRow(row).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionProjectRepository.upsert:query")), + ); + + const getById: ProjectionProjectRepositoryShape["getById"] = (input) => + getProjectionProjectRow(input).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionProjectRepository.getById:query")), + ); + + const listAll: ProjectionProjectRepositoryShape["listAll"] = () => + listProjectionProjectRows().pipe( + Effect.mapError(toPersistenceSqlError("ProjectionProjectRepository.listAll:query")), + ); + + const deleteById: ProjectionProjectRepositoryShape["deleteById"] = (input) => + deleteProjectionProjectRow(input).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionProjectRepository.deleteById:query")), + ); + + return { + upsert, + getById, + listAll, + deleteById, + } satisfies ProjectionProjectRepositoryShape; +}); + +export const PgProjectionProjectRepositoryLive = Layer.effect( + ProjectionProjectRepository, + makeProjectionProjectRepository, +); diff --git a/apps/server/src/persistence/Postgres/ProjectionSnapshotQuery.ts b/apps/server/src/persistence/Postgres/ProjectionSnapshotQuery.ts new file mode 100644 index 000000000000..25230e6464c0 --- /dev/null +++ b/apps/server/src/persistence/Postgres/ProjectionSnapshotQuery.ts @@ -0,0 +1,3193 @@ +import { + ChatAttachment, + CheckpointRef, + IsoDateTime, + MessageId, + NonNegativeInt, + OrchestrationCheckpointFile, + OrchestrationProposedPlanId, + OrchestrationReadModel, + OrchestrationThreadSearchSource, + OrchestrationShellSnapshot, + OrchestrationThread, + OrchestrationThreadDetailSnapshot, + ProjectScript, + TurnId, + type OrchestrationCheckpointSummary, + type OrchestrationLatestTurn, + type OrchestrationMessage, + type OrchestrationProjectShell, + type OrchestrationProposedPlan, + type OrchestrationProject, + type OrchestrationSession, + type OrchestrationThreadActivity, + type OrchestrationThreadShell, + ModelSelection, + ProjectId, + ThreadLinkedPullRequest, + ThreadId, +} from "@t3tools/contracts"; +import * as Arr from "effect/Array"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Result from "effect/Result"; +import * as Schema from "effect/Schema"; +import * as Struct from "effect/Struct"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; + +import { + isPersistenceError, + toPersistenceDecodeError, + toPersistenceSqlError, + type ProjectionRepositoryError, +} from "../Errors.ts"; +import { ProjectionCheckpoint } from "../Services/ProjectionCheckpoints.ts"; +import { ThreadBackgroundLivenessService } from "../../orchestration/ThreadBackgroundLiveness.ts"; +import { ThreadPlanProgressService } from "../../orchestration/ThreadPlanProgress.ts"; +import { ProjectionProject } from "../Services/ProjectionProjects.ts"; +import { ProjectionState } from "../Services/ProjectionState.ts"; +import { ProjectionThreadActivity } from "../Services/ProjectionThreadActivities.ts"; +import { ProjectionThreadMessage } from "../Services/ProjectionThreadMessages.ts"; +import { ProjectionThreadProposedPlan } from "../Services/ProjectionThreadProposedPlans.ts"; +import { ProjectionThreadSession } from "../Services/ProjectionThreadSessions.ts"; +import { ProjectionThread } from "../Services/ProjectionThreads.ts"; +import { + decodeThreadDetailPageCursor, + encodeThreadDetailPageCursor, +} from "../../orchestration/threadDetailCursor.ts"; +import { projectActivityPayload } from "../../orchestration/ActivityPayloadProjection.ts"; +import * as RepositoryIdentityResolver from "../../project/RepositoryIdentityResolver.ts"; +import { HubTenant } from "./HubTenant.ts"; +import { ORCHESTRATION_PROJECTOR_NAMES } from "../../orchestration/Layers/ProjectionPipeline.ts"; +import { + ProjectionSnapshotQuery, + type ProjectionFullThreadDiffContext, + type ProjectionSnapshotCounts, + type ProjectionThreadCheckpointContext, + type ProjectionThreadDetailQuery, + type ProjectionSnapshotQueryShape, +} from "../../orchestration/Services/ProjectionSnapshotQuery.ts"; + +const decodeReadModel = Schema.decodeUnknownEffect(OrchestrationReadModel); +const decodeShellSnapshot = Schema.decodeUnknownEffect(OrchestrationShellSnapshot); +const decodeThread = Schema.decodeUnknownEffect(OrchestrationThread); +// Keep detail reads consistent with the in-memory projector's retained +// activity window. Applying the limit in SQL avoids decoding an unbounded +// payload_json set before the projector can enforce that invariant. +const THREAD_DETAIL_ACTIVITY_LIMIT = 500; +// Snapshot payloads are decoded and projected in small sequential batches so +// one client read does not retain the raw payloads for the full activity window. +const THREAD_DETAIL_ACTIVITY_PAYLOAD_BATCH_SIZE = 25; +const ProjectionProjectDbRowSchema = ProjectionProject.mapFields( + Struct.assign({ + defaultModelSelection: Schema.NullOr(Schema.fromJsonString(ModelSelection)), + scripts: Schema.fromJsonString(Schema.Array(ProjectScript)), + }), +); +const ProjectionThreadMessageDbRowSchema = ProjectionThreadMessage.mapFields( + Struct.assign({ + isStreaming: Schema.Number, + attachments: Schema.NullOr(Schema.fromJsonString(Schema.Array(ChatAttachment))), + }), +); +const ProjectionThreadProposedPlanDbRowSchema = ProjectionThreadProposedPlan; +const ProjectionThreadDbRowSchema = ProjectionThread.mapFields( + Struct.assign({ + modelSelection: Schema.fromJsonString(ModelSelection), + linkedPullRequest: Schema.NullOr(Schema.fromJsonString(ThreadLinkedPullRequest)), + }), +); +const ProjectionThreadActivityDbRowSchema = ProjectionThreadActivity.mapFields( + Struct.assign({ + payload: Schema.fromJsonString(Schema.Unknown), + sequence: Schema.NullOr(NonNegativeInt), + }), +); +const ProjectionThreadActivityIdRowSchema = Schema.Struct({ + activityId: ProjectionThreadActivity.fields.activityId, +}); +const ProjectionThreadSessionDbRowSchema = ProjectionThreadSession; +const ProjectionCheckpointDbRowSchema = ProjectionCheckpoint.mapFields( + Struct.assign({ + files: Schema.fromJsonString(Schema.Array(OrchestrationCheckpointFile)), + }), +); +const ProjectionLatestTurnDbRowSchema = Schema.Struct({ + threadId: ProjectionThread.fields.threadId, + turnId: TurnId, + state: Schema.String, + requestedAt: IsoDateTime, + startedAt: Schema.NullOr(IsoDateTime), + completedAt: Schema.NullOr(IsoDateTime), + assistantMessageId: Schema.NullOr(MessageId), + sourceProposedPlanThreadId: Schema.NullOr(ThreadId), + sourceProposedPlanId: Schema.NullOr(OrchestrationProposedPlanId), +}); +const ProjectionStateDbRowSchema = ProjectionState; +const ProjectionCountsRowSchema = Schema.Struct({ + projectCount: Schema.Number, + threadCount: Schema.Number, +}); +const ProjectionThreadSearchRequest = Schema.Struct({ + pattern: Schema.String, + limit: Schema.Int, +}); +const ProjectionThreadSearchRow = Schema.Struct({ + threadId: ThreadId, + projectId: ProjectId, + source: OrchestrationThreadSearchSource, + matchText: Schema.String, + messageCreatedAt: Schema.NullOr(IsoDateTime), +}); +const WorkspaceRootLookupInput = Schema.Struct({ + workspaceRoot: Schema.String, +}); +const ProjectIdLookupInput = Schema.Struct({ + projectId: ProjectId, +}); +const ThreadIdLookupInput = Schema.Struct({ + threadId: ThreadId, +}); +const ThreadActivityKindsLookupInput = Schema.Struct({ + threadId: ThreadId, + activityKinds: Schema.Array(Schema.String), +}); +const ThreadActivityIdsLookupInput = Schema.Struct({ + activityIds: Schema.Array(ProjectionThreadActivity.fields.activityId), +}); +// Windowed reads order turns by the stable keyset (anchor, turn key), where +// anchor is requested_at and turn key is +// COALESCE(turn_id, ''). Both are event-derived, so cursors survive the +// revert projector's row-id rewrite and full projection rebuilds. +const ThreadTurnWindowLookupInput = Schema.Struct({ + threadId: ThreadId, + // Exclusive keyset upper bound. Sentinels "~"/"" mean unbounded ("~" sorts + // after every ISO timestamp). + beforeAnchorAt: Schema.String, + beforeTurnKey: Schema.String, + userTurnLimit: Schema.Number, + maxRawTurns: Schema.Number, +}); +const ProjectionTurnWindowRowSchema = Schema.Struct({ + // The turn's timeline anchor, used to bound rows that have no turn linkage + // (user messages and turnless activities) to the same page window. + anchorAt: Schema.String, + turnKey: Schema.String, +}); +const ThreadTurnRangeLookupInput = Schema.Struct({ + threadId: ThreadId, + // Turn-linked rows are bounded by the keyset range [min, before) over + // (anchor, turn key); turnless rows by the matching [minAnchorAt, + // beforeAnchorAt) time range. Unbounded ends use sentinels: "" for the + // lower bound, "~" (sorts after ISO dates) for the upper bound. + minAnchorAt: Schema.String, + minTurnKey: Schema.String, + beforeAnchorAt: Schema.String, + beforeTurnKey: Schema.String, +}); +const ProjectionProjectLookupRowSchema = ProjectionProjectDbRowSchema; +const ProjectionThreadIdLookupRowSchema = Schema.Struct({ + threadId: ThreadId, +}); +const ProjectionThreadCheckpointContextThreadRowSchema = Schema.Struct({ + threadId: ThreadId, + projectId: ProjectId, + workspaceRoot: Schema.String, + worktreePath: Schema.NullOr(Schema.String), +}); +const FullThreadDiffContextLookupInput = Schema.Struct({ + threadId: ThreadId, + checkpointTurnCount: NonNegativeInt, +}); +const ProjectionFullThreadDiffContextRowSchema = Schema.Struct({ + threadId: ThreadId, + projectId: ProjectId, + workspaceRoot: Schema.String, + worktreePath: Schema.NullOr(Schema.String), + latestCheckpointTurnCount: Schema.NullOr(NonNegativeInt), + toCheckpointRef: Schema.NullOr(CheckpointRef), +}); + +const REQUIRED_SNAPSHOT_PROJECTORS = [ + ORCHESTRATION_PROJECTOR_NAMES.projects, + ORCHESTRATION_PROJECTOR_NAMES.threads, + ORCHESTRATION_PROJECTOR_NAMES.threadMessages, + ORCHESTRATION_PROJECTOR_NAMES.threadProposedPlans, + ORCHESTRATION_PROJECTOR_NAMES.threadActivities, + ORCHESTRATION_PROJECTOR_NAMES.threadSessions, + ORCHESTRATION_PROJECTOR_NAMES.checkpoints, +] as const; + +function maxIso(left: string | null, right: string): string { + if (left === null) { + return right; + } + return left > right ? left : right; +} + +function escapeLikePattern(value: string): string { + return value.replaceAll("!", "!!").replaceAll("%", "!%").replaceAll("_", "!_"); +} + +function foldAsciiCase(value: string): string { + return value.replace(/[A-Z]/g, (character) => character.toLowerCase()); +} + +function buildSearchSnippet(text: string, query: string): string { + const normalizedText = text.replace(/\s+/g, " ").trim(); + if (normalizedText.length <= 240) { + return normalizedText; + } + + const normalizedQuery = foldAsciiCase(query.replace(/\s+/g, " ").trim()); + const matchIndex = foldAsciiCase(normalizedText).indexOf(normalizedQuery); + const bodyLength = 236; + const idealStart = Math.max(0, matchIndex - 72); + const start = Math.min(idealStart, normalizedText.length - bodyLength); + const end = Math.min(normalizedText.length, start + bodyLength); + return `${start > 0 ? "…" : ""}${normalizedText.slice(start, end)}${ + end < normalizedText.length ? "…" : "" + }`; +} + +function computeSnapshotSequence( + stateRows: ReadonlyArray>, +): number { + if (stateRows.length === 0) { + return 0; + } + const sequenceByProjector = new Map( + stateRows.map((row) => [row.projector, row.lastAppliedSequence] as const), + ); + + let minSequence = Number.POSITIVE_INFINITY; + for (const projector of REQUIRED_SNAPSHOT_PROJECTORS) { + const sequence = sequenceByProjector.get(projector); + if (sequence === undefined) { + return 0; + } + if (sequence < minSequence) { + minSequence = sequence; + } + } + + return Number.isFinite(minSequence) ? minSequence : 0; +} + +function mapLatestTurn( + row: Schema.Schema.Type, +): OrchestrationLatestTurn { + return { + turnId: row.turnId, + state: + row.state === "error" + ? "error" + : row.state === "interrupted" + ? "interrupted" + : row.state === "completed" + ? "completed" + : "running", + requestedAt: row.requestedAt, + startedAt: row.startedAt, + completedAt: row.completedAt, + assistantMessageId: row.assistantMessageId, + ...(row.sourceProposedPlanThreadId !== null && row.sourceProposedPlanId !== null + ? { + sourceProposedPlan: { + threadId: row.sourceProposedPlanThreadId, + planId: row.sourceProposedPlanId, + }, + } + : {}), + }; +} + +function mapTitleRegeneration(row: Schema.Schema.Type) { + return row.titleRegenerationRequestId != null && row.titleRegenerationStartedAt != null + ? { + requestId: row.titleRegenerationRequestId, + startedAt: row.titleRegenerationStartedAt, + } + : null; +} + +function mapSessionRow( + row: Schema.Schema.Type, +): OrchestrationSession { + return { + threadId: row.threadId, + status: row.status, + providerName: row.providerName, + ...(row.providerInstanceId !== null ? { providerInstanceId: row.providerInstanceId } : {}), + runtimeMode: row.runtimeMode, + activeTurnId: row.activeTurnId, + lastError: row.lastError, + updatedAt: row.updatedAt, + }; +} + +function mapProjectShellRow( + row: Schema.Schema.Type, + repositoryIdentity: OrchestrationProject["repositoryIdentity"], +): OrchestrationProjectShell { + return { + id: row.projectId, + title: row.title, + workspaceRoot: row.workspaceRoot, + repositoryIdentity, + defaultModelSelection: row.defaultModelSelection, + defaultThreadEnvMode: row.defaultThreadEnvMode, + faviconPath: row.faviconPath ?? null, + scripts: row.scripts, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} + +function mapProposedPlanRow( + row: Schema.Schema.Type, +): OrchestrationProposedPlan { + return { + id: row.planId, + turnId: row.turnId, + planMarkdown: row.planMarkdown, + implementedAt: row.implementedAt, + implementationThreadId: row.implementationThreadId, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} + +function mapThreadActivityRow( + row: Schema.Schema.Type, +): OrchestrationThreadActivity { + return { + id: row.activityId, + tone: row.tone, + kind: row.kind, + summary: row.summary, + payload: row.payload, + turnId: row.turnId, + createdAt: row.createdAt, + ...(row.sequence !== null ? { sequence: row.sequence } : {}), + }; +} + +function toPersistenceSqlOrDecodeError(sqlOperation: string, decodeOperation: string) { + return (cause: unknown): ProjectionRepositoryError => + Schema.isSchemaError(cause) + ? toPersistenceDecodeError(decodeOperation)(cause) + : toPersistenceSqlError(sqlOperation)(cause); +} + +const makeProjectionSnapshotQuery = Effect.gen(function* () { + const threadBackgroundLiveness = yield* ThreadBackgroundLivenessService; + const threadPlanProgress = yield* ThreadPlanProgressService; + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + const repositoryIdentityResolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; + const repositoryIdentityResolutionConcurrency = 4; + const resolveRepositoryIdentitiesForProjects = Effect.fn( + "ProjectionSnapshotQuery.resolveRepositoryIdentitiesForProjects", + )(function* ( + projectRows: ReadonlyArray>, + options?: { + readonly includeDeleted?: boolean; + }, + ) { + const filteredProjectRows = + options?.includeDeleted === true + ? projectRows + : projectRows.filter((row) => row.deletedAt === null); + const uniqueWorkspaceRoots = [...new Set(filteredProjectRows.map((row) => row.workspaceRoot))]; + const repositoryIdentityByWorkspaceRoot = new Map( + yield* Effect.forEach( + uniqueWorkspaceRoots, + (workspaceRoot) => + repositoryIdentityResolver + .resolve(workspaceRoot) + .pipe(Effect.map((identity) => [workspaceRoot, identity] as const)), + { concurrency: repositoryIdentityResolutionConcurrency }, + ), + ); + + return new Map( + filteredProjectRows.map((row) => [ + row.projectId, + repositoryIdentityByWorkspaceRoot.get(row.workspaceRoot) ?? null, + ]), + ); + }); + + // A SQLite read transaction reads one snapshot. Under Postgres READ COMMITTED + // each statement would read its own, so a snapshot could pair rows with a newer + // snapshotSequence and a resuming client would skip events. REPEATABLE READ + // restores the SQLite guarantee; it must be the transaction's first statement. + const withSnapshotTransaction = (effect: Effect.Effect) => + sql.withTransaction( + sql`SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY`.pipe(Effect.andThen(effect)), + ); + + const listProjectRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionProjectDbRowSchema, + execute: () => + sql` + SELECT + project_id AS "projectId", + title, + workspace_root AS "workspaceRoot", + default_model_selection_json AS "defaultModelSelection", + default_thread_env_mode AS "defaultThreadEnvMode", + favicon_path AS "faviconPath", + scripts_json AS "scripts", + created_at AS "createdAt", + updated_at AS "updatedAt", + deleted_at AS "deletedAt" + FROM projection_projects + WHERE user_id = ${userId} + ORDER BY created_at ASC, project_id ASC + `, + }); + + const listThreadRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionThreadDbRowSchema, + execute: () => + sql` + SELECT + thread_id AS "threadId", + project_id AS "projectId", + title, + model_selection_json AS "modelSelection", + runtime_mode AS "runtimeMode", + interaction_mode AS "interactionMode", + branch, + worktree_path AS "worktreePath", + linked_pull_request_json AS "linkedPullRequest", + latest_turn_id AS "latestTurnId", + created_at AS "createdAt", + updated_at AS "updatedAt", + archived_at AS "archivedAt", + settled_override AS "settledOverride", + settled_at AS "settledAt", + unsettled_at AS "unsettledAt", + snoozed_until AS "snoozedUntil", + snoozed_at AS "snoozedAt", + pinned_at AS "pinnedAt", + pin_order_key AS "pinOrderKey", + title_regeneration_request_id AS "titleRegenerationRequestId", + title_regeneration_started_at AS "titleRegenerationStartedAt", + latest_user_message_at AS "latestUserMessageAt", + pending_approval_count AS "pendingApprovalCount", + pending_user_input_count AS "pendingUserInputCount", + has_actionable_proposed_plan AS "hasActionableProposedPlan", + deleted_at AS "deletedAt" + FROM projection_threads + WHERE user_id = ${userId} + ORDER BY created_at ASC, thread_id ASC + `, + }); + + const listActiveThreadRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionThreadDbRowSchema, + execute: () => + sql` + SELECT + thread_id AS "threadId", + project_id AS "projectId", + title, + model_selection_json AS "modelSelection", + runtime_mode AS "runtimeMode", + interaction_mode AS "interactionMode", + branch, + worktree_path AS "worktreePath", + linked_pull_request_json AS "linkedPullRequest", + latest_turn_id AS "latestTurnId", + created_at AS "createdAt", + updated_at AS "updatedAt", + archived_at AS "archivedAt", + settled_override AS "settledOverride", + settled_at AS "settledAt", + unsettled_at AS "unsettledAt", + snoozed_until AS "snoozedUntil", + snoozed_at AS "snoozedAt", + pinned_at AS "pinnedAt", + pin_order_key AS "pinOrderKey", + title_regeneration_request_id AS "titleRegenerationRequestId", + title_regeneration_started_at AS "titleRegenerationStartedAt", + latest_user_message_at AS "latestUserMessageAt", + pending_approval_count AS "pendingApprovalCount", + pending_user_input_count AS "pendingUserInputCount", + has_actionable_proposed_plan AS "hasActionableProposedPlan", + deleted_at AS "deletedAt" + FROM projection_threads + WHERE user_id = ${userId} + AND deleted_at IS NULL + AND archived_at IS NULL + ORDER BY project_id ASC, created_at ASC, thread_id ASC + `, + }); + + const listArchivedThreadRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionThreadDbRowSchema, + execute: () => + sql` + SELECT + thread_id AS "threadId", + project_id AS "projectId", + title, + model_selection_json AS "modelSelection", + runtime_mode AS "runtimeMode", + interaction_mode AS "interactionMode", + branch, + worktree_path AS "worktreePath", + linked_pull_request_json AS "linkedPullRequest", + latest_turn_id AS "latestTurnId", + created_at AS "createdAt", + updated_at AS "updatedAt", + archived_at AS "archivedAt", + settled_override AS "settledOverride", + settled_at AS "settledAt", + unsettled_at AS "unsettledAt", + snoozed_until AS "snoozedUntil", + snoozed_at AS "snoozedAt", + pinned_at AS "pinnedAt", + pin_order_key AS "pinOrderKey", + title_regeneration_request_id AS "titleRegenerationRequestId", + title_regeneration_started_at AS "titleRegenerationStartedAt", + latest_user_message_at AS "latestUserMessageAt", + pending_approval_count AS "pendingApprovalCount", + pending_user_input_count AS "pendingUserInputCount", + has_actionable_proposed_plan AS "hasActionableProposedPlan", + deleted_at AS "deletedAt" + FROM projection_threads + WHERE user_id = ${userId} + AND deleted_at IS NULL + AND archived_at IS NOT NULL + ORDER BY project_id ASC, archived_at DESC, thread_id DESC + `, + }); + + const listThreadMessageRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionThreadMessageDbRowSchema, + execute: () => + sql` + SELECT + message_id AS "messageId", + thread_id AS "threadId", + turn_id AS "turnId", + role, + text, + attachments_json AS "attachments", + is_streaming AS "isStreaming", + created_at AS "createdAt", + updated_at AS "updatedAt" + FROM projection_thread_messages + WHERE user_id = ${userId} + ORDER BY thread_id ASC, created_at ASC, message_id ASC + `, + }); + + const listThreadProposedPlanRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionThreadProposedPlanDbRowSchema, + execute: () => + sql` + SELECT + plan_id AS "planId", + thread_id AS "threadId", + turn_id AS "turnId", + plan_markdown AS "planMarkdown", + implemented_at AS "implementedAt", + implementation_thread_id AS "implementationThreadId", + created_at AS "createdAt", + updated_at AS "updatedAt" + FROM projection_thread_proposed_plans + WHERE user_id = ${userId} + ORDER BY thread_id ASC, created_at ASC, plan_id ASC + `, + }); + + const listThreadActivityRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionThreadActivityDbRowSchema, + execute: () => + sql` + SELECT + activity_id AS "activityId", + thread_id AS "threadId", + turn_id AS "turnId", + tone, + kind, + summary, + payload_json AS "payload", + sequence, + created_at AS "createdAt" + FROM projection_thread_activities + WHERE user_id = ${userId} + ORDER BY + thread_id ASC, + sequence ASC NULLS FIRST, + created_at ASC, + activity_id ASC + `, + }); + + const listThreadSessionRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionThreadSessionDbRowSchema, + execute: () => + sql` + SELECT + thread_id AS "threadId", + status, + provider_name AS "providerName", + provider_instance_id AS "providerInstanceId", + provider_session_id AS "providerSessionId", + provider_thread_id AS "providerThreadId", + runtime_mode AS "runtimeMode", + active_turn_id AS "activeTurnId", + last_error AS "lastError", + updated_at AS "updatedAt" + FROM projection_thread_sessions + WHERE user_id = ${userId} + ORDER BY thread_id ASC + `, + }); + + const listActiveThreadSessionRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionThreadSessionDbRowSchema, + execute: () => + sql` + SELECT + sessions.thread_id AS "threadId", + sessions.status, + sessions.provider_name AS "providerName", + sessions.provider_instance_id AS "providerInstanceId", + sessions.provider_session_id AS "providerSessionId", + sessions.provider_thread_id AS "providerThreadId", + sessions.runtime_mode AS "runtimeMode", + sessions.active_turn_id AS "activeTurnId", + sessions.last_error AS "lastError", + sessions.updated_at AS "updatedAt" + FROM projection_thread_sessions sessions + INNER JOIN projection_threads threads + ON threads.user_id = sessions.user_id + AND threads.thread_id = sessions.thread_id + WHERE sessions.user_id = ${userId} + AND threads.deleted_at IS NULL + AND threads.archived_at IS NULL + ORDER BY sessions.thread_id ASC + `, + }); + + const listArchivedThreadSessionRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionThreadSessionDbRowSchema, + execute: () => + sql` + SELECT + sessions.thread_id AS "threadId", + sessions.status, + sessions.provider_name AS "providerName", + sessions.provider_instance_id AS "providerInstanceId", + sessions.provider_session_id AS "providerSessionId", + sessions.provider_thread_id AS "providerThreadId", + sessions.runtime_mode AS "runtimeMode", + sessions.active_turn_id AS "activeTurnId", + sessions.last_error AS "lastError", + sessions.updated_at AS "updatedAt" + FROM projection_thread_sessions sessions + INNER JOIN projection_threads threads + ON threads.user_id = sessions.user_id + AND threads.thread_id = sessions.thread_id + WHERE sessions.user_id = ${userId} + AND threads.deleted_at IS NULL + AND threads.archived_at IS NOT NULL + ORDER BY sessions.thread_id ASC + `, + }); + + const listCheckpointRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionCheckpointDbRowSchema, + execute: () => + sql` + SELECT + thread_id AS "threadId", + turn_id AS "turnId", + checkpoint_turn_count AS "checkpointTurnCount", + checkpoint_ref AS "checkpointRef", + checkpoint_status AS "status", + checkpoint_files_json AS "files", + assistant_message_id AS "assistantMessageId", + completed_at AS "completedAt" + FROM projection_turns + WHERE user_id = ${userId} + AND checkpoint_turn_count IS NOT NULL + ORDER BY thread_id ASC, checkpoint_turn_count ASC + `, + }); + + const listLatestTurnRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionLatestTurnDbRowSchema, + execute: () => + sql` + SELECT + turns.thread_id AS "threadId", + turns.turn_id AS "turnId", + turns.state, + turns.requested_at AS "requestedAt", + turns.started_at AS "startedAt", + turns.completed_at AS "completedAt", + turns.assistant_message_id AS "assistantMessageId", + turns.source_proposed_plan_thread_id AS "sourceProposedPlanThreadId", + turns.source_proposed_plan_id AS "sourceProposedPlanId" + FROM projection_threads threads + JOIN projection_turns turns + ON turns.user_id = threads.user_id + AND turns.thread_id = threads.thread_id + AND turns.turn_id = threads.latest_turn_id + WHERE threads.user_id = ${userId} + AND threads.latest_turn_id IS NOT NULL + ORDER BY turns.thread_id ASC + `, + }); + + const listActiveLatestTurnRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionLatestTurnDbRowSchema, + execute: () => + sql` + SELECT + turns.thread_id AS "threadId", + turns.turn_id AS "turnId", + turns.state, + turns.requested_at AS "requestedAt", + turns.started_at AS "startedAt", + turns.completed_at AS "completedAt", + turns.assistant_message_id AS "assistantMessageId", + turns.source_proposed_plan_thread_id AS "sourceProposedPlanThreadId", + turns.source_proposed_plan_id AS "sourceProposedPlanId" + FROM projection_threads threads + JOIN projection_turns turns + ON turns.user_id = threads.user_id + AND turns.thread_id = threads.thread_id + AND turns.turn_id = threads.latest_turn_id + WHERE threads.user_id = ${userId} + AND threads.deleted_at IS NULL + AND threads.archived_at IS NULL + AND threads.latest_turn_id IS NOT NULL + ORDER BY turns.thread_id ASC + `, + }); + + const listArchivedLatestTurnRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionLatestTurnDbRowSchema, + execute: () => + sql` + SELECT + turns.thread_id AS "threadId", + turns.turn_id AS "turnId", + turns.state, + turns.requested_at AS "requestedAt", + turns.started_at AS "startedAt", + turns.completed_at AS "completedAt", + turns.assistant_message_id AS "assistantMessageId", + turns.source_proposed_plan_thread_id AS "sourceProposedPlanThreadId", + turns.source_proposed_plan_id AS "sourceProposedPlanId" + FROM projection_threads threads + JOIN projection_turns turns + ON turns.user_id = threads.user_id + AND turns.thread_id = threads.thread_id + AND turns.turn_id = threads.latest_turn_id + WHERE threads.user_id = ${userId} + AND threads.deleted_at IS NULL + AND threads.archived_at IS NOT NULL + AND threads.latest_turn_id IS NOT NULL + ORDER BY turns.thread_id ASC + `, + }); + + const listProjectionStateRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionStateDbRowSchema, + execute: () => + sql` + SELECT + projector, + last_applied_sequence AS "lastAppliedSequence", + updated_at AS "updatedAt" + FROM projection_state + WHERE user_id = ${userId} + `, + }); + + const readProjectionCounts = SqlSchema.findOne({ + Request: Schema.Void, + Result: ProjectionCountsRowSchema, + execute: () => + sql` + SELECT + (SELECT COUNT(*) FROM projection_projects WHERE user_id = ${userId}) AS "projectCount", + (SELECT COUNT(*) FROM projection_threads WHERE user_id = ${userId}) AS "threadCount" + `, + }); + + const searchActiveThreadRows = SqlSchema.findAll({ + Request: ProjectionThreadSearchRequest, + Result: ProjectionThreadSearchRow, + execute: ({ pattern, limit }) => + sql` + WITH ranked AS ( + SELECT + threads.thread_id AS thread_id, + threads.project_id AS project_id, + CASE messages.role + WHEN 'user' THEN 'user' + ELSE 'assistant' + END AS source, + messages.text AS match_text, + messages.created_at AS message_created_at, + CASE messages.role + WHEN 'user' THEN 0 + ELSE 1 + END AS match_rank, + threads.updated_at AS thread_updated_at, + ROW_NUMBER() OVER ( + PARTITION BY threads.thread_id + ORDER BY + CASE messages.role + WHEN 'user' THEN 0 + ELSE 1 + END ASC, + messages.created_at DESC, + messages.message_id ASC + ) AS thread_match_rank + FROM projection_thread_messages AS messages + INNER JOIN projection_threads AS threads + ON threads.user_id = messages.user_id + AND threads.thread_id = messages.thread_id + INNER JOIN projection_projects AS projects + ON projects.user_id = threads.user_id + AND projects.project_id = threads.project_id + WHERE messages.user_id = ${userId} + AND threads.deleted_at IS NULL + AND threads.archived_at IS NULL + AND projects.deleted_at IS NULL + AND messages.is_streaming = 0 + AND ( + messages.role = 'user' + OR ( + messages.role = 'assistant' + AND messages.message_id IN ( + SELECT turns.assistant_message_id + FROM projection_turns AS turns + WHERE turns.user_id = ${userId} + AND turns.assistant_message_id IS NOT NULL + ) + ) + ) + AND messages.text ILIKE ${pattern} ESCAPE '!' + ) + SELECT + thread_id AS "threadId", + project_id AS "projectId", + source, + match_text AS "matchText", + message_created_at AS "messageCreatedAt" + FROM ranked + WHERE thread_match_rank = 1 + ORDER BY + match_rank ASC, + thread_updated_at DESC, + thread_id ASC + LIMIT ${limit} + `, + }); + + const getActiveProjectRowByWorkspaceRoot = SqlSchema.findOneOption({ + Request: WorkspaceRootLookupInput, + Result: ProjectionProjectLookupRowSchema, + execute: ({ workspaceRoot }) => + sql` + SELECT + project_id AS "projectId", + title, + workspace_root AS "workspaceRoot", + default_model_selection_json AS "defaultModelSelection", + default_thread_env_mode AS "defaultThreadEnvMode", + favicon_path AS "faviconPath", + scripts_json AS "scripts", + created_at AS "createdAt", + updated_at AS "updatedAt", + deleted_at AS "deletedAt" + FROM projection_projects + WHERE user_id = ${userId} + AND workspace_root = ${workspaceRoot} + AND deleted_at IS NULL + ORDER BY created_at ASC, project_id ASC + LIMIT 1 + `, + }); + + const getActiveProjectRowById = SqlSchema.findOneOption({ + Request: ProjectIdLookupInput, + Result: ProjectionProjectLookupRowSchema, + execute: ({ projectId }) => + sql` + SELECT + project_id AS "projectId", + title, + workspace_root AS "workspaceRoot", + default_model_selection_json AS "defaultModelSelection", + default_thread_env_mode AS "defaultThreadEnvMode", + favicon_path AS "faviconPath", + scripts_json AS "scripts", + created_at AS "createdAt", + updated_at AS "updatedAt", + deleted_at AS "deletedAt" + FROM projection_projects + WHERE user_id = ${userId} + AND project_id = ${projectId} + AND deleted_at IS NULL + LIMIT 1 + `, + }); + + const getFirstActiveThreadIdByProject = SqlSchema.findOneOption({ + Request: ProjectIdLookupInput, + Result: ProjectionThreadIdLookupRowSchema, + execute: ({ projectId }) => + sql` + SELECT + thread_id AS "threadId" + FROM projection_threads + WHERE user_id = ${userId} + AND project_id = ${projectId} + AND deleted_at IS NULL + AND archived_at IS NULL + ORDER BY created_at ASC, thread_id ASC + LIMIT 1 + `, + }); + + const getThreadCheckpointContextThreadRow = SqlSchema.findOneOption({ + Request: ThreadIdLookupInput, + Result: ProjectionThreadCheckpointContextThreadRowSchema, + execute: ({ threadId }) => + sql` + SELECT + threads.thread_id AS "threadId", + threads.project_id AS "projectId", + projects.workspace_root AS "workspaceRoot", + threads.worktree_path AS "worktreePath" + FROM projection_threads AS threads + INNER JOIN projection_projects AS projects + ON projects.user_id = threads.user_id + AND projects.project_id = threads.project_id + WHERE threads.user_id = ${userId} + AND threads.thread_id = ${threadId} + AND threads.deleted_at IS NULL + LIMIT 1 + `, + }); + + const getActiveThreadRowById = SqlSchema.findOneOption({ + Request: ThreadIdLookupInput, + Result: ProjectionThreadDbRowSchema, + execute: ({ threadId }) => + sql` + SELECT + thread_id AS "threadId", + project_id AS "projectId", + title, + model_selection_json AS "modelSelection", + runtime_mode AS "runtimeMode", + interaction_mode AS "interactionMode", + branch, + worktree_path AS "worktreePath", + linked_pull_request_json AS "linkedPullRequest", + latest_turn_id AS "latestTurnId", + created_at AS "createdAt", + updated_at AS "updatedAt", + archived_at AS "archivedAt", + settled_override AS "settledOverride", + settled_at AS "settledAt", + unsettled_at AS "unsettledAt", + snoozed_until AS "snoozedUntil", + snoozed_at AS "snoozedAt", + pinned_at AS "pinnedAt", + pin_order_key AS "pinOrderKey", + title_regeneration_request_id AS "titleRegenerationRequestId", + title_regeneration_started_at AS "titleRegenerationStartedAt", + latest_user_message_at AS "latestUserMessageAt", + pending_approval_count AS "pendingApprovalCount", + pending_user_input_count AS "pendingUserInputCount", + has_actionable_proposed_plan AS "hasActionableProposedPlan", + deleted_at AS "deletedAt" + FROM projection_threads + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND deleted_at IS NULL + AND archived_at IS NULL + LIMIT 1 + `, + }); + + const listThreadMessageRowsByThread = SqlSchema.findAll({ + Request: ThreadIdLookupInput, + Result: ProjectionThreadMessageDbRowSchema, + execute: ({ threadId }) => + sql` + SELECT + message_id AS "messageId", + thread_id AS "threadId", + turn_id AS "turnId", + role, + text, + attachments_json AS "attachments", + is_streaming AS "isStreaming", + created_at AS "createdAt", + updated_at AS "updatedAt" + FROM projection_thread_messages + WHERE user_id = ${userId} + AND thread_id = ${threadId} + ORDER BY created_at ASC, message_id ASC + `, + }); + + const listThreadProposedPlanRowsByThread = SqlSchema.findAll({ + Request: ThreadIdLookupInput, + Result: ProjectionThreadProposedPlanDbRowSchema, + execute: ({ threadId }) => + sql` + SELECT + plan_id AS "planId", + thread_id AS "threadId", + turn_id AS "turnId", + plan_markdown AS "planMarkdown", + implemented_at AS "implementedAt", + implementation_thread_id AS "implementationThreadId", + created_at AS "createdAt", + updated_at AS "updatedAt" + FROM projection_thread_proposed_plans + WHERE user_id = ${userId} + AND thread_id = ${threadId} + ORDER BY created_at ASC, plan_id ASC + `, + }); + + const listThreadActivityRowsByThread = SqlSchema.findAll({ + Request: ThreadIdLookupInput, + Result: ProjectionThreadActivityDbRowSchema, + execute: ({ threadId }) => + sql` + SELECT + activity_id AS "activityId", + thread_id AS "threadId", + turn_id AS "turnId", + tone, + kind, + summary, + payload_json AS "payload", + sequence, + created_at AS "createdAt" + FROM ( + SELECT + activity_id, + thread_id, + turn_id, + tone, + kind, + summary, + payload_json, + sequence, + created_at + FROM projection_thread_activities + WHERE user_id = ${userId} + AND thread_id = ${threadId} + ORDER BY + sequence DESC NULLS LAST, + created_at DESC, + activity_id DESC + LIMIT ${THREAD_DETAIL_ACTIVITY_LIMIT} + ) AS recent_activities + ORDER BY + sequence ASC NULLS FIRST, + created_at ASC, + activity_id ASC + `, + }); + + const listThreadActivityIdsByThread = SqlSchema.findAll({ + Request: ThreadIdLookupInput, + Result: ProjectionThreadActivityIdRowSchema, + execute: ({ threadId }) => + sql` + SELECT activity_id AS "activityId" + FROM projection_thread_activities + WHERE user_id = ${userId} + AND thread_id = ${threadId} + ORDER BY + sequence DESC NULLS LAST, + created_at DESC, + activity_id DESC + LIMIT ${THREAD_DETAIL_ACTIVITY_LIMIT} + `, + }); + + const listThreadActivityRowsByIds = SqlSchema.findAll({ + Request: ThreadActivityIdsLookupInput, + Result: ProjectionThreadActivityDbRowSchema, + execute: ({ activityIds }) => + sql` + SELECT + activity_id AS "activityId", + thread_id AS "threadId", + turn_id AS "turnId", + tone, + kind, + summary, + payload_json AS "payload", + sequence, + created_at AS "createdAt" + FROM projection_thread_activities + -- The selectors already scoped these globally unique ids to the + -- thread inside this transaction. Keep this as a primary-key lookup. + WHERE user_id = ${userId} + AND ${sql.in("activity_id", activityIds)} + `, + }); + + const listThreadActivityRowsByThreadAndKinds = SqlSchema.findAll({ + Request: ThreadActivityKindsLookupInput, + Result: ProjectionThreadActivityDbRowSchema, + execute: ({ threadId, activityKinds }) => + sql` + SELECT + activity_id AS "activityId", + thread_id AS "threadId", + turn_id AS "turnId", + tone, + kind, + summary, + payload_json AS "payload", + sequence, + created_at AS "createdAt" + FROM ( + SELECT + activity_id, + thread_id, + turn_id, + tone, + kind, + summary, + payload_json, + sequence, + created_at + FROM projection_thread_activities + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND ${sql.in("kind", activityKinds)} + ORDER BY + sequence DESC NULLS LAST, + created_at DESC, + activity_id DESC + LIMIT ${THREAD_DETAIL_ACTIVITY_LIMIT} + ) AS recent_activities + ORDER BY + sequence ASC NULLS FIRST, + created_at ASC, + activity_id ASC + `, + }); + + const getThreadSessionRowByThread = SqlSchema.findOneOption({ + Request: ThreadIdLookupInput, + Result: ProjectionThreadSessionDbRowSchema, + execute: ({ threadId }) => + sql` + SELECT + thread_id AS "threadId", + status, + provider_name AS "providerName", + provider_instance_id AS "providerInstanceId", + runtime_mode AS "runtimeMode", + active_turn_id AS "activeTurnId", + last_error AS "lastError", + updated_at AS "updatedAt" + FROM projection_thread_sessions + WHERE user_id = ${userId} + AND thread_id = ${threadId} + LIMIT 1 + `, + }); + + const getLatestTurnRowByThread = SqlSchema.findOneOption({ + Request: ThreadIdLookupInput, + Result: ProjectionLatestTurnDbRowSchema, + execute: ({ threadId }) => + sql` + SELECT + turns.thread_id AS "threadId", + turns.turn_id AS "turnId", + turns.state, + turns.requested_at AS "requestedAt", + turns.started_at AS "startedAt", + turns.completed_at AS "completedAt", + turns.assistant_message_id AS "assistantMessageId", + turns.source_proposed_plan_thread_id AS "sourceProposedPlanThreadId", + turns.source_proposed_plan_id AS "sourceProposedPlanId" + FROM projection_threads threads + JOIN projection_turns turns + ON turns.user_id = threads.user_id + AND turns.thread_id = threads.thread_id + AND turns.turn_id = threads.latest_turn_id + WHERE threads.user_id = ${userId} + AND threads.thread_id = ${threadId} + AND threads.deleted_at IS NULL + AND threads.archived_at IS NULL + LIMIT 1 + `, + }); + + const listCheckpointRowsByThread = SqlSchema.findAll({ + Request: ThreadIdLookupInput, + Result: ProjectionCheckpointDbRowSchema, + execute: ({ threadId }) => + sql` + SELECT + thread_id AS "threadId", + turn_id AS "turnId", + checkpoint_turn_count AS "checkpointTurnCount", + checkpoint_ref AS "checkpointRef", + checkpoint_status AS "status", + checkpoint_files_json AS "files", + assistant_message_id AS "assistantMessageId", + completed_at AS "completedAt" + FROM projection_turns + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND checkpoint_turn_count IS NOT NULL + ORDER BY checkpoint_turn_count ASC + `, + }); + + // Resolves a page of recent turns for a windowed thread detail read. Walks + // back from the exclusive (beforeAnchorAt, beforeTurnKey) keyset boundary + // (sentinels "~"/"" mean unbounded, i.e. the first page) until it has seen + // `userTurnLimit` user-anchored turns — turns whose pending message is a + // user message; subagent/fan-out turns between them ride along — or hits the + // `maxRawTurns` ceiling that bounds pathological fan-out. The `candidates` + // CTE applies the keyset bound and LIMIT before the window functions run; + // its ORDER BY uses raw columns so the migration-037 + // (thread_id, requested_at, turn_id) index serves both range and order with + // no temp B-tree — the scan is genuinely bounded by the LIMIT. (Raw + // turn_id DESC places NULLs exactly where COALESCE-to-'' would, below every + // real id.) The caller derives the continuation cursor from the oldest + // returned row. + // Highest thread-DETAIL event sequence for this thread that the projection + // has applied (bounded by the global snapshot sequence read in the same + // transaction). This is the thread-scoped watermark a windowed page carries + // so clients can defer merging until their live subscription has caught up; + // the global sequence is not waitable per-thread. The event_type filter + // must match ws.ts's isThreadDetailEvent exactly: the subscription only + // delivers these types, so a watermark counting any other event could + // never be reached by the client and would park the page forever. Served + // by the event store's (aggregate_kind, stream_id, sequence) index. + const getThreadEventWatermarkRow = SqlSchema.findOneOption({ + Request: Schema.Struct({ threadId: ThreadId, maxSequence: Schema.Number }), + Result: Schema.Struct({ threadSequence: Schema.NullOr(Schema.Number) }), + execute: ({ threadId, maxSequence }) => + sql` + SELECT MAX(sequence) AS "threadSequence" + FROM orchestration_events + WHERE user_id = ${userId} + AND aggregate_kind = 'thread' + AND stream_id = ${threadId} + AND sequence <= ${maxSequence} + AND event_type IN ( + 'thread.message-sent', + 'thread.proposed-plan-upserted', + 'thread.activity-appended', + 'thread.turn-diff-completed', + 'thread.reverted', + 'thread.session-set' + ) + `, + }); + + const listTurnWindowRows = SqlSchema.findAll({ + Request: ThreadTurnWindowLookupInput, + Result: ProjectionTurnWindowRowSchema, + execute: ({ threadId, beforeAnchorAt, beforeTurnKey, userTurnLimit, maxRawTurns }) => + sql` + WITH candidates AS ( + SELECT + turns.requested_at AS anchor_at, + COALESCE(turns.turn_id, '') AS turn_key, + turns.pending_message_id + FROM projection_turns AS turns + WHERE turns.user_id = ${userId} + AND turns.thread_id = ${threadId} + AND ( + turns.requested_at < ${beforeAnchorAt} + OR ( + turns.requested_at = ${beforeAnchorAt} + AND COALESCE(turns.turn_id, '') < ${beforeTurnKey} + ) + ) + ORDER BY turns.requested_at DESC, turns.turn_id DESC NULLS LAST + LIMIT ${maxRawTurns} + ), + walked AS ( + SELECT + candidates.anchor_at, + candidates.turn_key, + CASE WHEN messages.role = 'user' THEN 1 ELSE 0 END AS is_user_turn, + SUM(CASE WHEN messages.role = 'user' THEN 1 ELSE 0 END) OVER ( + ORDER BY candidates.anchor_at DESC, candidates.turn_key DESC + ) AS user_turns_seen + FROM candidates + LEFT JOIN projection_thread_messages AS messages + ON messages.user_id = ${userId} + AND messages.message_id = candidates.pending_message_id + ) + SELECT + anchor_at AS "anchorAt", + turn_key AS "turnKey" + FROM walked + WHERE user_turns_seen < ${userTurnLimit} + OR (user_turns_seen = ${userTurnLimit} AND is_user_turn = 1) + ORDER BY anchor_at ASC, turn_key ASC + `, + }); + + // Windowed variants of the two heavy collections. Turn-linked rows are + // bounded by the page's (anchor, turn key) keyset range over + // projection_turns; rows with no turn linkage (user messages always, and + // turnless activities like pre-turn context-window updates) are bounded by + // the matching turn-anchor time range so they land on the same page as the + // turns around them. Proposed plans and checkpoints stay unwindowed: they + // are metadata-scale. + const listThreadMessageRowsByThreadWindow = SqlSchema.findAll({ + Request: ThreadTurnRangeLookupInput, + Result: ProjectionThreadMessageDbRowSchema, + execute: ({ threadId, minAnchorAt, minTurnKey, beforeAnchorAt, beforeTurnKey }) => + sql` + SELECT + message_id AS "messageId", + thread_id AS "threadId", + turn_id AS "turnId", + role, + text, + attachments_json AS "attachments", + is_streaming AS "isStreaming", + created_at AS "createdAt", + updated_at AS "updatedAt" + FROM projection_thread_messages + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND ( + turn_id IN ( + SELECT turn_id FROM projection_turns + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND turn_id IS NOT NULL + AND ( + requested_at > ${minAnchorAt} + OR ( + requested_at = ${minAnchorAt} + AND turn_id >= ${minTurnKey} + ) + ) + AND ( + requested_at < ${beforeAnchorAt} + OR ( + requested_at = ${beforeAnchorAt} + AND turn_id < ${beforeTurnKey} + ) + ) + ) + OR ( + turn_id IS NULL + AND created_at >= ${minAnchorAt} + AND created_at < ${beforeAnchorAt} + ) + ) + ORDER BY created_at ASC, message_id ASC + `, + }); + + // SQLite json_extract accepts the "\u0000" escapes JSON.stringify emits for NUL + // characters (13 real activity payloads in the owner's history have them); + // Postgres json/jsonb parsing rejects them. Neutralize the escape before + // parsing: only requestId/detail are extracted and neither contains it. + const activityPayloadJson = sql.literal( + String.raw`replace(activity.payload_json, '\u0000', '\ufffd')::json`, + ); + + const pinnedThreadActivityIdsCte = (threadId: string) => sql` +pending_approval_requests AS ( + SELECT request_id, thread_id + FROM projection_pending_approvals + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND status = 'pending' + ), + pending_approval_activities AS ( + SELECT + activity.activity_id, + ROW_NUMBER() OVER ( + PARTITION BY pending.request_id + ORDER BY activity.created_at DESC, activity.activity_id DESC + ) AS request_order + FROM pending_approval_requests AS pending + CROSS JOIN projection_thread_activities AS activity + WHERE activity.user_id = ${userId} + AND activity.thread_id = pending.thread_id + AND activity.kind = 'approval.requested' + AND (${activityPayloadJson} ->> 'requestId') = pending.request_id + ), + pending_user_input_thread AS ( + SELECT thread_id + FROM projection_threads + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND pending_user_input_count > 0 + ), + user_input_lifecycle AS ( + SELECT + activity.activity_id, + activity.kind, + ROW_NUMBER() OVER ( + PARTITION BY (${activityPayloadJson} ->> 'requestId') + ORDER BY activity.created_at DESC, activity.activity_id DESC + ) AS request_order + FROM pending_user_input_thread AS pending + CROSS JOIN projection_thread_activities AS activity + WHERE activity.user_id = ${userId} + AND activity.thread_id = pending.thread_id + AND ( + activity.kind IN ('user-input.requested', 'user-input.resolved') + OR ( + activity.kind = 'provider.user-input.respond.failed' + AND ( + lower(COALESCE(${activityPayloadJson} ->> 'detail', '')) + LIKE '%stale pending user-input request%' + OR lower(COALESCE(${activityPayloadJson} ->> 'detail', '')) + LIKE '%unknown pending user-input request%' + OR lower(COALESCE(${activityPayloadJson} ->> 'detail', '')) + LIKE '%unknown pending user input request%' + OR lower(COALESCE(${activityPayloadJson} ->> 'detail', '')) + LIKE '%unknown pending codex user input request%' + ) + ) + ) + AND (${activityPayloadJson} ->> 'requestId') IS NOT NULL + ), + pinned_activity_ids AS ( + SELECT activity_id + FROM pending_approval_activities + WHERE request_order = 1 + UNION ALL + SELECT activity_id + FROM user_input_lifecycle + WHERE request_order = 1 + AND kind = 'user-input.requested' + ) + `; + + // Blocking request payloads must remain available even if they predate the + // recent activity window. Each CTE returns at most one unresolved row per + // request, so the merge below stays bounded by actionable work. + const listPinnedThreadActivityRowsByThread = SqlSchema.findAll({ + Request: ThreadIdLookupInput, + Result: ProjectionThreadActivityDbRowSchema, + execute: ({ threadId }) => + sql` + WITH ${pinnedThreadActivityIdsCte(threadId)} + SELECT + activity.activity_id AS "activityId", + activity.thread_id AS "threadId", + activity.turn_id AS "turnId", + activity.tone, + activity.kind, + activity.summary, + activity.payload_json AS "payload", + activity.sequence, + activity.created_at AS "createdAt" + FROM pinned_activity_ids AS pinned + INNER JOIN projection_thread_activities AS activity + ON activity.user_id = ${userId} + AND activity.activity_id = pinned.activity_id + ORDER BY activity.created_at ASC, activity.activity_id ASC + `, + }); + + const listPinnedThreadActivityIdsByThread = SqlSchema.findAll({ + Request: ThreadIdLookupInput, + Result: ProjectionThreadActivityIdRowSchema, + execute: ({ threadId }) => + sql` + WITH ${pinnedThreadActivityIdsCte(threadId)} + SELECT activity_id AS "activityId" + FROM pinned_activity_ids + `, + }); + + const listThreadActivityRowsByThreadWindow = SqlSchema.findAll({ + Request: ThreadTurnRangeLookupInput, + Result: ProjectionThreadActivityDbRowSchema, + execute: ({ threadId, minAnchorAt, minTurnKey, beforeAnchorAt, beforeTurnKey }) => + sql` + SELECT + activity_id AS "activityId", + thread_id AS "threadId", + turn_id AS "turnId", + tone, + kind, + summary, + payload_json AS "payload", + sequence, + created_at AS "createdAt" + FROM ( + SELECT + activity_id, + thread_id, + turn_id, + tone, + kind, + summary, + payload_json, + sequence, + created_at + FROM projection_thread_activities + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND ( + turn_id IN ( + SELECT turn_id FROM projection_turns + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND turn_id IS NOT NULL + AND ( + requested_at > ${minAnchorAt} + OR ( + requested_at = ${minAnchorAt} + AND turn_id >= ${minTurnKey} + ) + ) + AND ( + requested_at < ${beforeAnchorAt} + OR ( + requested_at = ${beforeAnchorAt} + AND turn_id < ${beforeTurnKey} + ) + ) + ) + OR ( + turn_id IS NULL + AND created_at >= ${minAnchorAt} + AND created_at < ${beforeAnchorAt} + ) + ) + ORDER BY + sequence DESC NULLS LAST, + created_at DESC, + activity_id DESC + LIMIT ${THREAD_DETAIL_ACTIVITY_LIMIT} + ) AS recent_activities + ORDER BY + sequence ASC NULLS FIRST, + created_at ASC, + activity_id ASC + `, + }); + + const listThreadActivityIdsByThreadWindow = SqlSchema.findAll({ + Request: ThreadTurnRangeLookupInput, + Result: ProjectionThreadActivityIdRowSchema, + execute: ({ threadId, minAnchorAt, minTurnKey, beforeAnchorAt, beforeTurnKey }) => + sql` + SELECT activity_id AS "activityId" + FROM projection_thread_activities + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND ( + turn_id IN ( + SELECT turn_id FROM projection_turns + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND turn_id IS NOT NULL + AND ( + requested_at > ${minAnchorAt} + OR ( + requested_at = ${minAnchorAt} + AND turn_id >= ${minTurnKey} + ) + ) + AND ( + requested_at < ${beforeAnchorAt} + OR ( + requested_at = ${beforeAnchorAt} + AND turn_id < ${beforeTurnKey} + ) + ) + ) + OR ( + turn_id IS NULL + AND created_at >= ${minAnchorAt} + AND created_at < ${beforeAnchorAt} + ) + ) + ORDER BY + sequence DESC NULLS LAST, + created_at DESC, + activity_id DESC + LIMIT ${THREAD_DETAIL_ACTIVITY_LIMIT} + `, + }); + + const getFullThreadDiffContextRow = SqlSchema.findOneOption({ + Request: FullThreadDiffContextLookupInput, + Result: ProjectionFullThreadDiffContextRowSchema, + execute: ({ threadId, checkpointTurnCount }) => + sql` + SELECT + threads.thread_id AS "threadId", + threads.project_id AS "projectId", + projects.workspace_root AS "workspaceRoot", + threads.worktree_path AS "worktreePath", + ( + SELECT MAX(turns.checkpoint_turn_count) + FROM projection_turns AS turns + WHERE turns.user_id = threads.user_id + AND turns.thread_id = threads.thread_id + AND turns.checkpoint_turn_count IS NOT NULL + ) AS "latestCheckpointTurnCount", + ( + SELECT turns.checkpoint_ref + FROM projection_turns AS turns + WHERE turns.user_id = threads.user_id + AND turns.thread_id = threads.thread_id + AND turns.checkpoint_turn_count = ${checkpointTurnCount} + LIMIT 1 + ) AS "toCheckpointRef" + FROM projection_threads AS threads + INNER JOIN projection_projects AS projects + ON projects.user_id = threads.user_id + AND projects.project_id = threads.project_id + WHERE threads.user_id = ${userId} + AND threads.thread_id = ${threadId} + AND threads.deleted_at IS NULL + LIMIT 1 + `, + }); + + const getSnapshot: ProjectionSnapshotQueryShape["getSnapshot"] = () => + withSnapshotTransaction( + Effect.all([ + listProjectRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getSnapshot:listProjects:query", + "ProjectionSnapshotQuery.getSnapshot:listProjects:decodeRows", + ), + ), + ), + listThreadRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getSnapshot:listThreads:query", + "ProjectionSnapshotQuery.getSnapshot:listThreads:decodeRows", + ), + ), + ), + listThreadMessageRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getSnapshot:listThreadMessages:query", + "ProjectionSnapshotQuery.getSnapshot:listThreadMessages:decodeRows", + ), + ), + ), + listThreadProposedPlanRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getSnapshot:listThreadProposedPlans:query", + "ProjectionSnapshotQuery.getSnapshot:listThreadProposedPlans:decodeRows", + ), + ), + ), + listThreadActivityRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getSnapshot:listThreadActivities:query", + "ProjectionSnapshotQuery.getSnapshot:listThreadActivities:decodeRows", + ), + ), + ), + listThreadSessionRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getSnapshot:listThreadSessions:query", + "ProjectionSnapshotQuery.getSnapshot:listThreadSessions:decodeRows", + ), + ), + ), + listCheckpointRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getSnapshot:listCheckpoints:query", + "ProjectionSnapshotQuery.getSnapshot:listCheckpoints:decodeRows", + ), + ), + ), + listLatestTurnRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getSnapshot:listLatestTurns:query", + "ProjectionSnapshotQuery.getSnapshot:listLatestTurns:decodeRows", + ), + ), + ), + listProjectionStateRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getSnapshot:listProjectionState:query", + "ProjectionSnapshotQuery.getSnapshot:listProjectionState:decodeRows", + ), + ), + ), + ]), + ).pipe( + Effect.flatMap( + ([ + projectRows, + threadRows, + messageRows, + proposedPlanRows, + activityRows, + sessionRows, + checkpointRows, + latestTurnRows, + stateRows, + ]) => + Effect.gen(function* () { + const messagesByThread = new Map>(); + const proposedPlansByThread = new Map>(); + const activitiesByThread = new Map>(); + const checkpointsByThread = new Map>(); + const sessionsByThread = new Map(); + const latestTurnByThread = new Map(); + + let updatedAt: string | null = null; + + for (const row of projectRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + } + for (const row of threadRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + } + for (const row of stateRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + } + + for (const row of messageRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + const threadMessages = messagesByThread.get(row.threadId) ?? []; + threadMessages.push({ + id: row.messageId, + role: row.role, + text: row.text, + ...(row.attachments !== null ? { attachments: row.attachments } : {}), + turnId: row.turnId, + streaming: row.isStreaming === 1, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }); + messagesByThread.set(row.threadId, threadMessages); + } + + for (const row of proposedPlanRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + const threadProposedPlans = proposedPlansByThread.get(row.threadId) ?? []; + threadProposedPlans.push({ + id: row.planId, + turnId: row.turnId, + planMarkdown: row.planMarkdown, + implementedAt: row.implementedAt, + implementationThreadId: row.implementationThreadId, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }); + proposedPlansByThread.set(row.threadId, threadProposedPlans); + } + + for (const row of activityRows) { + updatedAt = maxIso(updatedAt, row.createdAt); + const threadActivities = activitiesByThread.get(row.threadId) ?? []; + threadActivities.push({ + id: row.activityId, + tone: row.tone, + kind: row.kind, + summary: row.summary, + payload: row.payload, + turnId: row.turnId, + ...(row.sequence !== null ? { sequence: row.sequence } : {}), + createdAt: row.createdAt, + }); + activitiesByThread.set(row.threadId, threadActivities); + } + + for (const row of checkpointRows) { + updatedAt = maxIso(updatedAt, row.completedAt); + const threadCheckpoints = checkpointsByThread.get(row.threadId) ?? []; + threadCheckpoints.push({ + turnId: row.turnId, + checkpointTurnCount: row.checkpointTurnCount, + checkpointRef: row.checkpointRef, + status: row.status, + files: row.files, + assistantMessageId: row.assistantMessageId, + completedAt: row.completedAt, + }); + checkpointsByThread.set(row.threadId, threadCheckpoints); + } + + for (const row of latestTurnRows) { + updatedAt = maxIso(updatedAt, row.requestedAt); + if (row.startedAt !== null) { + updatedAt = maxIso(updatedAt, row.startedAt); + } + if (row.completedAt !== null) { + updatedAt = maxIso(updatedAt, row.completedAt); + } + if (latestTurnByThread.has(row.threadId)) { + continue; + } + latestTurnByThread.set(row.threadId, { + turnId: row.turnId, + state: + row.state === "error" + ? "error" + : row.state === "interrupted" + ? "interrupted" + : row.state === "completed" + ? "completed" + : "running", + requestedAt: row.requestedAt, + startedAt: row.startedAt, + completedAt: row.completedAt, + assistantMessageId: row.assistantMessageId, + ...(row.sourceProposedPlanThreadId !== null && row.sourceProposedPlanId !== null + ? { + sourceProposedPlan: { + threadId: row.sourceProposedPlanThreadId, + planId: row.sourceProposedPlanId, + }, + } + : {}), + }); + } + + for (const row of sessionRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + sessionsByThread.set(row.threadId, { + threadId: row.threadId, + status: row.status, + providerName: row.providerName, + ...(row.providerInstanceId !== null + ? { providerInstanceId: row.providerInstanceId } + : {}), + runtimeMode: row.runtimeMode, + activeTurnId: row.activeTurnId, + lastError: row.lastError, + updatedAt: row.updatedAt, + }); + } + + const repositoryIdentities = yield* resolveRepositoryIdentitiesForProjects( + projectRows, + { includeDeleted: true }, + ); + + const projects: ReadonlyArray = projectRows.map((row) => ({ + id: row.projectId, + title: row.title, + workspaceRoot: row.workspaceRoot, + repositoryIdentity: repositoryIdentities.get(row.projectId) ?? null, + defaultModelSelection: row.defaultModelSelection, + defaultThreadEnvMode: row.defaultThreadEnvMode, + faviconPath: row.faviconPath ?? null, + scripts: row.scripts, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + deletedAt: row.deletedAt, + })); + + const threads: ReadonlyArray = threadRows.map((row) => ({ + id: row.threadId, + projectId: row.projectId, + title: row.title, + modelSelection: row.modelSelection, + runtimeMode: row.runtimeMode, + interactionMode: row.interactionMode, + branch: row.branch, + worktreePath: row.worktreePath, + ...(row.linkedPullRequest === null + ? {} + : { linkedPullRequest: row.linkedPullRequest }), + latestTurn: latestTurnByThread.get(row.threadId) ?? null, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + archivedAt: row.archivedAt, + settledOverride: row.settledOverride, + settledAt: row.settledAt, + unsettledAt: row.unsettledAt, + snoozedUntil: row.snoozedUntil, + snoozedAt: row.snoozedAt, + pinnedAt: row.pinnedAt, + pinOrderKey: row.pinOrderKey ?? null, + titleRegeneration: mapTitleRegeneration(row), + deletedAt: row.deletedAt, + messages: messagesByThread.get(row.threadId) ?? [], + proposedPlans: proposedPlansByThread.get(row.threadId) ?? [], + activities: activitiesByThread.get(row.threadId) ?? [], + checkpoints: checkpointsByThread.get(row.threadId) ?? [], + session: sessionsByThread.get(row.threadId) ?? null, + })); + + const snapshot = { + snapshotSequence: computeSnapshotSequence(stateRows), + projects, + threads, + updatedAt: updatedAt ?? "1970-01-01T00:00:00.000Z", + }; + + return yield* decodeReadModel(snapshot).pipe( + Effect.mapError( + toPersistenceDecodeError("ProjectionSnapshotQuery.getSnapshot:decodeReadModel"), + ), + ); + }), + ), + Effect.mapError((error) => { + if (isPersistenceError(error)) { + return error; + } + return toPersistenceSqlError("ProjectionSnapshotQuery.getSnapshot:query")(error); + }), + ); + + const getCommandReadModel: ProjectionSnapshotQueryShape["getCommandReadModel"] = () => + withSnapshotTransaction( + Effect.all([ + listProjectRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getCommandReadModel:listProjects:query", + "ProjectionSnapshotQuery.getCommandReadModel:listProjects:decodeRows", + ), + ), + ), + listThreadRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getCommandReadModel:listThreads:query", + "ProjectionSnapshotQuery.getCommandReadModel:listThreads:decodeRows", + ), + ), + ), + listThreadProposedPlanRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getCommandReadModel:listThreadProposedPlans:query", + "ProjectionSnapshotQuery.getCommandReadModel:listThreadProposedPlans:decodeRows", + ), + ), + ), + listThreadSessionRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getCommandReadModel:listThreadSessions:query", + "ProjectionSnapshotQuery.getCommandReadModel:listThreadSessions:decodeRows", + ), + ), + ), + listLatestTurnRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getCommandReadModel:listLatestTurns:query", + "ProjectionSnapshotQuery.getCommandReadModel:listLatestTurns:decodeRows", + ), + ), + ), + listProjectionStateRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getCommandReadModel:listProjectionState:query", + "ProjectionSnapshotQuery.getCommandReadModel:listProjectionState:decodeRows", + ), + ), + ), + ]), + ).pipe( + Effect.flatMap( + ([projectRows, threadRows, proposedPlanRows, sessionRows, latestTurnRows, stateRows]) => + Effect.sync(() => { + let updatedAt: string | null = null; + const projects: OrchestrationProject[] = []; + const threads: OrchestrationThread[] = []; + + for (let index = 0; index < projectRows.length; index += 1) { + const row = projectRows[index]; + if (!row) { + continue; + } + updatedAt = maxIso(updatedAt, row.updatedAt); + projects.push({ + id: row.projectId, + title: row.title, + workspaceRoot: row.workspaceRoot, + defaultModelSelection: row.defaultModelSelection, + defaultThreadEnvMode: row.defaultThreadEnvMode, + faviconPath: row.faviconPath ?? null, + scripts: row.scripts, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + deletedAt: row.deletedAt, + }); + } + for (let index = 0; index < threadRows.length; index += 1) { + const row = threadRows[index]; + if (!row) { + continue; + } + updatedAt = maxIso(updatedAt, row.updatedAt); + } + for (let index = 0; index < proposedPlanRows.length; index += 1) { + const row = proposedPlanRows[index]; + if (!row) { + continue; + } + updatedAt = maxIso(updatedAt, row.updatedAt); + } + for (let index = 0; index < sessionRows.length; index += 1) { + const row = sessionRows[index]; + if (!row) { + continue; + } + updatedAt = maxIso(updatedAt, row.updatedAt); + } + for (let index = 0; index < latestTurnRows.length; index += 1) { + const row = latestTurnRows[index]; + if (!row) { + continue; + } + updatedAt = maxIso(updatedAt, row.requestedAt); + if (row.startedAt !== null) { + updatedAt = maxIso(updatedAt, row.startedAt); + } + if (row.completedAt !== null) { + updatedAt = maxIso(updatedAt, row.completedAt); + } + } + for (let index = 0; index < stateRows.length; index += 1) { + const row = stateRows[index]; + if (!row) { + continue; + } + updatedAt = maxIso(updatedAt, row.updatedAt); + } + + const latestTurnByThread = new Map(); + for (let index = 0; index < latestTurnRows.length; index += 1) { + const row = latestTurnRows[index]; + if (!row) { + continue; + } + latestTurnByThread.set(row.threadId, mapLatestTurn(row)); + } + const proposedPlansByThread = new Map>(); + const sessionByThread = new Map(); + + for (let index = 0; index < sessionRows.length; index += 1) { + const row = sessionRows[index]; + if (!row) { + continue; + } + sessionByThread.set(row.threadId, mapSessionRow(row)); + } + + for (let index = 0; index < proposedPlanRows.length; index += 1) { + const row = proposedPlanRows[index]; + if (!row) { + continue; + } + const threadProposedPlans = proposedPlansByThread.get(row.threadId) ?? []; + threadProposedPlans.push(mapProposedPlanRow(row)); + proposedPlansByThread.set(row.threadId, threadProposedPlans); + } + + for (let index = 0; index < threadRows.length; index += 1) { + const row = threadRows[index]; + if (!row) { + continue; + } + threads.push({ + id: row.threadId, + projectId: row.projectId, + title: row.title, + modelSelection: row.modelSelection, + runtimeMode: row.runtimeMode, + interactionMode: row.interactionMode, + branch: row.branch, + worktreePath: row.worktreePath, + ...(row.linkedPullRequest === null + ? {} + : { linkedPullRequest: row.linkedPullRequest }), + latestTurn: latestTurnByThread.get(row.threadId) ?? null, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + archivedAt: row.archivedAt, + settledOverride: row.settledOverride, + settledAt: row.settledAt, + unsettledAt: row.unsettledAt, + snoozedUntil: row.snoozedUntil, + snoozedAt: row.snoozedAt, + pinnedAt: row.pinnedAt, + pinOrderKey: row.pinOrderKey ?? null, + titleRegeneration: mapTitleRegeneration(row), + deletedAt: row.deletedAt, + messages: [], + proposedPlans: proposedPlansByThread.get(row.threadId) ?? [], + activities: [], + checkpoints: [], + session: sessionByThread.get(row.threadId) ?? null, + }); + } + + return { + snapshotSequence: computeSnapshotSequence(stateRows), + projects, + threads, + updatedAt: updatedAt ?? "1970-01-01T00:00:00.000Z", + } satisfies OrchestrationReadModel; + }), + ), + Effect.mapError((error) => { + if (isPersistenceError(error)) { + return error; + } + return toPersistenceSqlError("ProjectionSnapshotQuery.getCommandReadModel:query")(error); + }), + ); + + const getShellSnapshot: ProjectionSnapshotQueryShape["getShellSnapshot"] = () => + withSnapshotTransaction( + Effect.all([ + listProjectRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getShellSnapshot:listProjects:query", + "ProjectionSnapshotQuery.getShellSnapshot:listProjects:decodeRows", + ), + ), + ), + listActiveThreadRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getShellSnapshot:listThreads:query", + "ProjectionSnapshotQuery.getShellSnapshot:listThreads:decodeRows", + ), + ), + ), + listActiveThreadSessionRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getShellSnapshot:listThreadSessions:query", + "ProjectionSnapshotQuery.getShellSnapshot:listThreadSessions:decodeRows", + ), + ), + ), + listActiveLatestTurnRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getShellSnapshot:listLatestTurns:query", + "ProjectionSnapshotQuery.getShellSnapshot:listLatestTurns:decodeRows", + ), + ), + ), + listProjectionStateRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getShellSnapshot:listProjectionState:query", + "ProjectionSnapshotQuery.getShellSnapshot:listProjectionState:decodeRows", + ), + ), + ), + ]), + ).pipe( + Effect.flatMap(([projectRows, threadRows, sessionRows, latestTurnRows, stateRows]) => + Effect.gen(function* () { + let updatedAt: string | null = null; + for (const row of projectRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + } + for (const row of threadRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + } + for (const row of sessionRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + } + for (const row of latestTurnRows) { + updatedAt = maxIso(updatedAt, row.requestedAt); + if (row.startedAt !== null) { + updatedAt = maxIso(updatedAt, row.startedAt); + } + if (row.completedAt !== null) { + updatedAt = maxIso(updatedAt, row.completedAt); + } + } + for (const row of stateRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + } + + const repositoryIdentities = yield* resolveRepositoryIdentitiesForProjects(projectRows); + const latestTurnByThread = new Map( + latestTurnRows.map((row) => [row.threadId, mapLatestTurn(row)] as const), + ); + const sessionByThread = new Map( + sessionRows.map((row) => [row.threadId, mapSessionRow(row)] as const), + ); + + const snapshot = { + snapshotSequence: computeSnapshotSequence(stateRows), + projects: Arr.filterMap(projectRows, (row) => + row.deletedAt === null + ? Result.succeed( + mapProjectShellRow(row, repositoryIdentities.get(row.projectId) ?? null), + ) + : Result.failVoid, + ), + threads: Arr.filterMap(threadRows, (row) => + row.deletedAt === null + ? Result.succeed({ + id: row.threadId, + projectId: row.projectId, + title: row.title, + modelSelection: row.modelSelection, + runtimeMode: row.runtimeMode, + interactionMode: row.interactionMode, + branch: row.branch, + worktreePath: row.worktreePath, + ...(row.linkedPullRequest === null + ? {} + : { linkedPullRequest: row.linkedPullRequest }), + latestTurn: latestTurnByThread.get(row.threadId) ?? null, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + archivedAt: row.archivedAt, + settledOverride: row.settledOverride, + settledAt: row.settledAt, + unsettledAt: row.unsettledAt, + snoozedUntil: row.snoozedUntil, + snoozedAt: row.snoozedAt, + pinnedAt: row.pinnedAt, + pinOrderKey: row.pinOrderKey ?? null, + titleRegeneration: mapTitleRegeneration(row), + session: sessionByThread.get(row.threadId) ?? null, + latestUserMessageAt: row.latestUserMessageAt, + hasPendingApprovals: row.pendingApprovalCount > 0, + hasPendingUserInput: row.pendingUserInputCount > 0, + hasActionableProposedPlan: row.hasActionableProposedPlan > 0, + backgroundLiveness: threadBackgroundLiveness.getThreadBackgroundLiveness( + row.threadId, + ), + planProgress: threadPlanProgress.getThreadPlanProgress(row.threadId), + } satisfies OrchestrationThreadShell) + : Result.failVoid, + ), + updatedAt: updatedAt ?? "1970-01-01T00:00:00.000Z", + }; + + return yield* decodeShellSnapshot(snapshot).pipe( + Effect.mapError( + toPersistenceDecodeError( + "ProjectionSnapshotQuery.getShellSnapshot:decodeShellSnapshot", + ), + ), + ); + }), + ), + Effect.mapError((error) => { + if (isPersistenceError(error)) { + return error; + } + return toPersistenceSqlError("ProjectionSnapshotQuery.getShellSnapshot:query")(error); + }), + ); + + const getArchivedShellSnapshot: ProjectionSnapshotQueryShape["getArchivedShellSnapshot"] = () => + withSnapshotTransaction( + Effect.all([ + listProjectRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getArchivedShellSnapshot:listProjects:query", + "ProjectionSnapshotQuery.getArchivedShellSnapshot:listProjects:decodeRows", + ), + ), + ), + listArchivedThreadRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getArchivedShellSnapshot:listThreads:query", + "ProjectionSnapshotQuery.getArchivedShellSnapshot:listThreads:decodeRows", + ), + ), + ), + listArchivedThreadSessionRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getArchivedShellSnapshot:listThreadSessions:query", + "ProjectionSnapshotQuery.getArchivedShellSnapshot:listThreadSessions:decodeRows", + ), + ), + ), + listArchivedLatestTurnRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getArchivedShellSnapshot:listLatestTurns:query", + "ProjectionSnapshotQuery.getArchivedShellSnapshot:listLatestTurns:decodeRows", + ), + ), + ), + listProjectionStateRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getArchivedShellSnapshot:listProjectionState:query", + "ProjectionSnapshotQuery.getArchivedShellSnapshot:listProjectionState:decodeRows", + ), + ), + ), + ]), + ).pipe( + Effect.flatMap(([projectRows, threadRows, sessionRows, latestTurnRows, stateRows]) => + Effect.gen(function* () { + let updatedAt: string | null = null; + for (const row of projectRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + } + for (const row of threadRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + } + for (const row of sessionRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + } + for (const row of latestTurnRows) { + updatedAt = maxIso(updatedAt, row.requestedAt); + if (row.startedAt !== null) { + updatedAt = maxIso(updatedAt, row.startedAt); + } + if (row.completedAt !== null) { + updatedAt = maxIso(updatedAt, row.completedAt); + } + } + for (const row of stateRows) { + updatedAt = maxIso(updatedAt, row.updatedAt); + } + + const activeProjectIds = new Set(threadRows.map((row) => row.projectId)); + const repositoryIdentities = yield* resolveRepositoryIdentitiesForProjects( + projectRows.filter((row) => activeProjectIds.has(row.projectId)), + ); + const latestTurnByThread = new Map( + latestTurnRows.map((row) => [row.threadId, mapLatestTurn(row)] as const), + ); + const sessionByThread = new Map( + sessionRows.map((row) => [row.threadId, mapSessionRow(row)] as const), + ); + + const snapshot = { + snapshotSequence: computeSnapshotSequence(stateRows), + projects: Arr.filterMap(projectRows, (row) => + row.deletedAt === null && activeProjectIds.has(row.projectId) + ? Result.succeed( + mapProjectShellRow(row, repositoryIdentities.get(row.projectId) ?? null), + ) + : Result.failVoid, + ), + threads: threadRows.map( + (row): OrchestrationThreadShell => ({ + id: row.threadId, + projectId: row.projectId, + title: row.title, + modelSelection: row.modelSelection, + runtimeMode: row.runtimeMode, + interactionMode: row.interactionMode, + branch: row.branch, + worktreePath: row.worktreePath, + ...(row.linkedPullRequest === null + ? {} + : { linkedPullRequest: row.linkedPullRequest }), + latestTurn: latestTurnByThread.get(row.threadId) ?? null, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + archivedAt: row.archivedAt, + settledOverride: row.settledOverride, + settledAt: row.settledAt, + unsettledAt: row.unsettledAt, + snoozedUntil: row.snoozedUntil, + snoozedAt: row.snoozedAt, + pinnedAt: row.pinnedAt, + pinOrderKey: row.pinOrderKey ?? null, + titleRegeneration: mapTitleRegeneration(row), + session: sessionByThread.get(row.threadId) ?? null, + latestUserMessageAt: row.latestUserMessageAt, + hasPendingApprovals: row.pendingApprovalCount > 0, + hasPendingUserInput: row.pendingUserInputCount > 0, + hasActionableProposedPlan: row.hasActionableProposedPlan > 0, + backgroundLiveness: threadBackgroundLiveness.getThreadBackgroundLiveness( + row.threadId, + ), + planProgress: threadPlanProgress.getThreadPlanProgress(row.threadId), + }), + ), + updatedAt: updatedAt ?? "1970-01-01T00:00:00.000Z", + }; + + return yield* decodeShellSnapshot(snapshot).pipe( + Effect.mapError( + toPersistenceDecodeError( + "ProjectionSnapshotQuery.getArchivedShellSnapshot:decodeShellSnapshot", + ), + ), + ); + }), + ), + Effect.mapError((error) => { + if (isPersistenceError(error)) { + return error; + } + return toPersistenceSqlError("ProjectionSnapshotQuery.getArchivedShellSnapshot:query")( + error, + ); + }), + ); + + const getSnapshotSequence: ProjectionSnapshotQueryShape["getSnapshotSequence"] = () => + listProjectionStateRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getSnapshotSequence:query", + "ProjectionSnapshotQuery.getSnapshotSequence:decodeRows", + ), + ), + Effect.map((stateRows) => ({ + snapshotSequence: computeSnapshotSequence(stateRows), + })), + ); + + const getCounts: ProjectionSnapshotQueryShape["getCounts"] = () => + readProjectionCounts(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getCounts:query", + "ProjectionSnapshotQuery.getCounts:decodeRow", + ), + ), + Effect.map( + (row): ProjectionSnapshotCounts => ({ + projectCount: row.projectCount, + threadCount: row.threadCount, + }), + ), + ); + + const searchThreads: ProjectionSnapshotQueryShape["searchThreads"] = Effect.fn( + "ProjectionSnapshotQuery.searchThreads", + )(function* (input) { + const escapedQuery = escapeLikePattern(input.query); + const rows = yield* searchActiveThreadRows({ + pattern: `%${escapedQuery}%`, + limit: input.limit ?? 50, + }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.searchThreads:query", + "ProjectionSnapshotQuery.searchThreads:decodeRows", + ), + ), + ); + return { + matches: rows.map((row) => ({ + threadId: row.threadId, + projectId: row.projectId, + source: row.source, + snippet: buildSearchSnippet(row.matchText, input.query), + messageCreatedAt: row.messageCreatedAt, + })), + }; + }); + + const getActiveProjectByWorkspaceRoot: ProjectionSnapshotQueryShape["getActiveProjectByWorkspaceRoot"] = + (workspaceRoot) => + getActiveProjectRowByWorkspaceRoot({ workspaceRoot }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getActiveProjectByWorkspaceRoot:query", + "ProjectionSnapshotQuery.getActiveProjectByWorkspaceRoot:decodeRow", + ), + ), + Effect.flatMap((option) => + Option.isNone(option) + ? Effect.succeed(Option.none()) + : repositoryIdentityResolver.resolve(option.value.workspaceRoot).pipe( + Effect.map((repositoryIdentity) => + Option.some({ + id: option.value.projectId, + title: option.value.title, + workspaceRoot: option.value.workspaceRoot, + repositoryIdentity, + defaultModelSelection: option.value.defaultModelSelection, + defaultThreadEnvMode: option.value.defaultThreadEnvMode, + faviconPath: option.value.faviconPath ?? null, + scripts: option.value.scripts, + createdAt: option.value.createdAt, + updatedAt: option.value.updatedAt, + deletedAt: option.value.deletedAt, + } satisfies OrchestrationProject), + ), + ), + ), + ); + + const getProjectShellById: ProjectionSnapshotQueryShape["getProjectShellById"] = (projectId) => + getActiveProjectRowById({ projectId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getProjectShellById:query", + "ProjectionSnapshotQuery.getProjectShellById:decodeRow", + ), + ), + Effect.flatMap((option) => + Option.isNone(option) + ? Effect.succeed(Option.none()) + : repositoryIdentityResolver + .resolve(option.value.workspaceRoot) + .pipe( + Effect.map((repositoryIdentity) => + Option.some(mapProjectShellRow(option.value, repositoryIdentity)), + ), + ), + ), + ); + + const getFirstActiveThreadIdByProjectId: ProjectionSnapshotQueryShape["getFirstActiveThreadIdByProjectId"] = + (projectId) => + getFirstActiveThreadIdByProject({ projectId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getFirstActiveThreadIdByProjectId:query", + "ProjectionSnapshotQuery.getFirstActiveThreadIdByProjectId:decodeRow", + ), + ), + Effect.map(Option.map((row) => row.threadId)), + ); + + const getThreadCheckpointContext: ProjectionSnapshotQueryShape["getThreadCheckpointContext"] = ( + threadId, + ) => + Effect.gen(function* () { + const threadRow = yield* getThreadCheckpointContextThreadRow({ threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadCheckpointContext:getThread:query", + "ProjectionSnapshotQuery.getThreadCheckpointContext:getThread:decodeRow", + ), + ), + ); + if (Option.isNone(threadRow)) { + return Option.none(); + } + + const checkpointRows = yield* listCheckpointRowsByThread({ threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadCheckpointContext:listCheckpoints:query", + "ProjectionSnapshotQuery.getThreadCheckpointContext:listCheckpoints:decodeRows", + ), + ), + ); + + return Option.some({ + threadId: threadRow.value.threadId, + projectId: threadRow.value.projectId, + workspaceRoot: threadRow.value.workspaceRoot, + worktreePath: threadRow.value.worktreePath, + checkpoints: checkpointRows.map( + (row): OrchestrationCheckpointSummary => ({ + turnId: row.turnId, + checkpointTurnCount: row.checkpointTurnCount, + checkpointRef: row.checkpointRef, + status: row.status, + files: row.files, + assistantMessageId: row.assistantMessageId, + completedAt: row.completedAt, + }), + ), + }); + }); + + const getFullThreadDiffContext: NonNullable< + ProjectionSnapshotQueryShape["getFullThreadDiffContext"] + > = (threadId, toTurnCount) => + Effect.gen(function* () { + const row = yield* getFullThreadDiffContextRow({ + threadId, + checkpointTurnCount: toTurnCount, + }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getFullThreadDiffContext:query", + "ProjectionSnapshotQuery.getFullThreadDiffContext:decodeRow", + ), + ), + ); + if (Option.isNone(row)) { + return Option.none(); + } + + return Option.some({ + threadId: row.value.threadId, + projectId: row.value.projectId, + workspaceRoot: row.value.workspaceRoot, + worktreePath: row.value.worktreePath, + latestCheckpointTurnCount: row.value.latestCheckpointTurnCount ?? 0, + toCheckpointRef: row.value.toCheckpointRef, + }); + }); + + const getThreadShellById: ProjectionSnapshotQueryShape["getThreadShellById"] = (threadId) => + Effect.gen(function* () { + const [threadRow, latestTurnRow, sessionRow] = yield* Effect.all([ + getActiveThreadRowById({ threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadShellById:getThread:query", + "ProjectionSnapshotQuery.getThreadShellById:getThread:decodeRow", + ), + ), + ), + getLatestTurnRowByThread({ threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadShellById:getLatestTurn:query", + "ProjectionSnapshotQuery.getThreadShellById:getLatestTurn:decodeRow", + ), + ), + ), + getThreadSessionRowByThread({ threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadShellById:getSession:query", + "ProjectionSnapshotQuery.getThreadShellById:getSession:decodeRow", + ), + ), + ), + ]); + + if (Option.isNone(threadRow)) { + return Option.none(); + } + + return Option.some({ + id: threadRow.value.threadId, + projectId: threadRow.value.projectId, + title: threadRow.value.title, + modelSelection: threadRow.value.modelSelection, + runtimeMode: threadRow.value.runtimeMode, + interactionMode: threadRow.value.interactionMode, + branch: threadRow.value.branch, + worktreePath: threadRow.value.worktreePath, + ...(threadRow.value.linkedPullRequest === null + ? {} + : { linkedPullRequest: threadRow.value.linkedPullRequest }), + latestTurn: Option.isSome(latestTurnRow) ? mapLatestTurn(latestTurnRow.value) : null, + createdAt: threadRow.value.createdAt, + updatedAt: threadRow.value.updatedAt, + archivedAt: threadRow.value.archivedAt, + settledOverride: threadRow.value.settledOverride, + settledAt: threadRow.value.settledAt, + unsettledAt: threadRow.value.unsettledAt, + snoozedUntil: threadRow.value.snoozedUntil, + snoozedAt: threadRow.value.snoozedAt, + pinnedAt: threadRow.value.pinnedAt, + pinOrderKey: threadRow.value.pinOrderKey ?? null, + titleRegeneration: mapTitleRegeneration(threadRow.value), + session: Option.isSome(sessionRow) ? mapSessionRow(sessionRow.value) : null, + latestUserMessageAt: threadRow.value.latestUserMessageAt, + hasPendingApprovals: threadRow.value.pendingApprovalCount > 0, + hasPendingUserInput: threadRow.value.pendingUserInputCount > 0, + hasActionableProposedPlan: threadRow.value.hasActionableProposedPlan > 0, + backgroundLiveness: threadBackgroundLiveness.getThreadBackgroundLiveness( + threadRow.value.threadId, + ), + planProgress: threadPlanProgress.getThreadPlanProgress(threadRow.value.threadId), + } satisfies OrchestrationThreadShell); + }); + + // Contiguous turn range bounding a windowed detail read; undefined loads the + // full thread. Resolved from a window request inside the snapshot + // transaction (see getThreadDetailSnapshot). + interface ThreadDetailBounds { + readonly minAnchorAt: string; + readonly minTurnKey: string; + readonly beforeAnchorAt: string; + readonly beforeTurnKey: string; + } + + type ThreadDetailActivityRead = + | { + readonly mode: "raw"; + readonly query?: ProjectionThreadDetailQuery; + } + | { + readonly mode: "client"; + }; + + const listProjectedThreadActivities = Effect.fn( + "ProjectionSnapshotQuery.listProjectedThreadActivities", + )(function* (threadId: ThreadId, bounds: ThreadDetailBounds | undefined) { + const [activityIdRows, pinnedActivityIdRows] = yield* Effect.all([ + (bounds === undefined + ? listThreadActivityIdsByThread({ threadId }) + : listThreadActivityIdsByThreadWindow({ threadId, ...bounds }) + ).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailById:listActivityIds:query", + "ProjectionSnapshotQuery.getThreadDetailById:listActivityIds:decodeRows", + ), + ), + ), + listPinnedThreadActivityIdsByThread({ threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailById:listPinnedActivityIds:query", + "ProjectionSnapshotQuery.getThreadDetailById:listPinnedActivityIds:decodeRows", + ), + ), + ), + ]); + const activityIds = [ + ...new Set([...activityIdRows, ...pinnedActivityIdRows].map(({ activityId }) => activityId)), + ]; + const activities: OrchestrationThreadActivity[] = []; + + for ( + let offset = 0; + offset < activityIds.length; + offset += THREAD_DETAIL_ACTIVITY_PAYLOAD_BATCH_SIZE + ) { + const batchIds = activityIds.slice( + offset, + offset + THREAD_DETAIL_ACTIVITY_PAYLOAD_BATCH_SIZE, + ); + const batchRows = yield* listThreadActivityRowsByIds({ activityIds: batchIds }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailById:listActivityPayloadBatch:query", + "ProjectionSnapshotQuery.getThreadDetailById:listActivityPayloadBatch:decodeRows", + ), + ), + ); + for (const row of batchRows) { + activities.push(projectActivityPayload(mapThreadActivityRow(row))); + } + } + + return activities.toSorted( + (left, right) => + (left.sequence ?? -1) - (right.sequence ?? -1) || + left.createdAt.localeCompare(right.createdAt) || + left.id.localeCompare(right.id), + ); + }); + + const getThreadDetailByIdBounded = ( + threadId: ThreadId, + bounds: ThreadDetailBounds | undefined, + activityRead: ThreadDetailActivityRead = { mode: "raw" }, + ) => + Effect.gen(function* () { + const activitiesEffect = + activityRead.mode === "client" + ? listProjectedThreadActivities(threadId, bounds) + : Effect.all([ + (activityRead.query?.activityKinds === undefined + ? bounds === undefined + ? listThreadActivityRowsByThread({ threadId }) + : listThreadActivityRowsByThreadWindow({ threadId, ...bounds }) + : activityRead.query.activityKinds.length === 0 + ? Effect.succeed([]) + : listThreadActivityRowsByThreadAndKinds({ + threadId, + activityKinds: activityRead.query.activityKinds, + }) + ).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailById:listActivities:query", + "ProjectionSnapshotQuery.getThreadDetailById:listActivities:decodeRows", + ), + ), + ), + activityRead.query?.activityKinds === undefined + ? listPinnedThreadActivityRowsByThread({ threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailById:listPinnedActivities:query", + "ProjectionSnapshotQuery.getThreadDetailById:listPinnedActivities:decodeRows", + ), + ), + ) + : Effect.succeed([]), + ]).pipe( + Effect.map(([activityRows, pinnedActivityRows]) => + [ + ...new Map( + [...activityRows, ...pinnedActivityRows].map( + (row) => [row.activityId, row] as const, + ), + ).values(), + ] + .toSorted( + (left, right) => + (left.sequence ?? -1) - (right.sequence ?? -1) || + left.createdAt.localeCompare(right.createdAt) || + left.activityId.localeCompare(right.activityId), + ) + .map(mapThreadActivityRow), + ), + ); + + const [ + threadRow, + messageRows, + proposedPlanRows, + activities, + checkpointRows, + latestTurnRow, + sessionRow, + ] = yield* Effect.all([ + getActiveThreadRowById({ threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailById:getThread:query", + "ProjectionSnapshotQuery.getThreadDetailById:getThread:decodeRow", + ), + ), + ), + (bounds === undefined + ? listThreadMessageRowsByThread({ threadId }) + : listThreadMessageRowsByThreadWindow({ threadId, ...bounds }) + ).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailById:listMessages:query", + "ProjectionSnapshotQuery.getThreadDetailById:listMessages:decodeRows", + ), + ), + ), + listThreadProposedPlanRowsByThread({ threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailById:listPlans:query", + "ProjectionSnapshotQuery.getThreadDetailById:listPlans:decodeRows", + ), + ), + ), + activitiesEffect, + listCheckpointRowsByThread({ threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailById:listCheckpoints:query", + "ProjectionSnapshotQuery.getThreadDetailById:listCheckpoints:decodeRows", + ), + ), + ), + getLatestTurnRowByThread({ threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailById:getLatestTurn:query", + "ProjectionSnapshotQuery.getThreadDetailById:getLatestTurn:decodeRow", + ), + ), + ), + getThreadSessionRowByThread({ threadId }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailById:getSession:query", + "ProjectionSnapshotQuery.getThreadDetailById:getSession:decodeRow", + ), + ), + ), + ]); + + if (Option.isNone(threadRow)) { + return Option.none(); + } + + const thread = { + id: threadRow.value.threadId, + projectId: threadRow.value.projectId, + title: threadRow.value.title, + modelSelection: threadRow.value.modelSelection, + runtimeMode: threadRow.value.runtimeMode, + interactionMode: threadRow.value.interactionMode, + branch: threadRow.value.branch, + worktreePath: threadRow.value.worktreePath, + ...(threadRow.value.linkedPullRequest === null + ? {} + : { linkedPullRequest: threadRow.value.linkedPullRequest }), + latestTurn: Option.isSome(latestTurnRow) ? mapLatestTurn(latestTurnRow.value) : null, + createdAt: threadRow.value.createdAt, + updatedAt: threadRow.value.updatedAt, + archivedAt: threadRow.value.archivedAt, + settledOverride: threadRow.value.settledOverride, + settledAt: threadRow.value.settledAt, + unsettledAt: threadRow.value.unsettledAt, + snoozedUntil: threadRow.value.snoozedUntil, + snoozedAt: threadRow.value.snoozedAt, + pinnedAt: threadRow.value.pinnedAt, + pinOrderKey: threadRow.value.pinOrderKey ?? null, + titleRegeneration: mapTitleRegeneration(threadRow.value), + deletedAt: null, + messages: messageRows.map((row) => { + const message = { + id: row.messageId, + role: row.role, + text: row.text, + turnId: row.turnId, + streaming: row.isStreaming === 1, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; + if (row.attachments !== null) { + return Object.assign(message, { attachments: row.attachments }); + } + return message; + }), + proposedPlans: proposedPlanRows.map(mapProposedPlanRow), + activities, + checkpoints: checkpointRows.map((row) => ({ + turnId: row.turnId, + checkpointTurnCount: row.checkpointTurnCount, + checkpointRef: row.checkpointRef, + status: row.status, + files: row.files, + assistantMessageId: row.assistantMessageId, + completedAt: row.completedAt, + })), + session: Option.isSome(sessionRow) ? mapSessionRow(sessionRow.value) : null, + }; + + return Option.some( + yield* decodeThread(thread).pipe( + Effect.mapError( + toPersistenceDecodeError("ProjectionSnapshotQuery.getThreadDetailById:decodeThread"), + ), + ), + ); + }); + + const getThreadDetailById: ProjectionSnapshotQueryShape["getThreadDetailById"] = ( + threadId, + query, + ) => + getThreadDetailByIdBounded(threadId, undefined, { + mode: "raw", + ...(query === undefined ? {} : { query }), + }); + + // Bounds pathological fan-out: one user turn that spawned hundreds of + // subagent turns still pages in bounded chunks, at the cost of splitting the + // fan-out group across pages (the cursor continues the same group). Also + // structurally bounds the window scan via the candidates CTE's LIMIT. + const THREAD_DETAIL_MAX_RAW_TURNS_PER_PAGE = 150; + // Sentinels for unbounded keyset ends; "~" sorts after any ISO timestamp. + const ANCHOR_UNBOUNDED = "~"; + + const getThreadDetailSnapshot: ProjectionSnapshotQueryShape["getThreadDetailSnapshot"] = ( + threadId, + window, + ) => + // Read the thread detail and the snapshot sequence within a single + // transaction so the sequence is consistent with the returned state; a + // projector update landing between two separate reads could otherwise return + // a sequence ahead of the thread detail, causing the client to resume from + // too far and drop events. Window resolution runs inside the same + // transaction so the page boundary is consistent with the returned rows. + withSnapshotTransaction( + Effect.gen(function* () { + if (window?.turnLimit === undefined) { + const thread = yield* getThreadDetailByIdBounded(threadId, undefined, { + mode: "client", + }); + if (Option.isNone(thread)) { + return Option.none(); + } + const { snapshotSequence } = yield* getSnapshotSequence(); + return Option.some({ snapshotSequence, thread: thread.value }); + } + + // A malformed or foreign-thread cursor falls back to the first page + // rather than failing: the client's stale cursor after a revert or + // reconnect should degrade to "reload recent history", not error. + const decodedCursor = + window.beforeCursor === undefined + ? null + : decodeThreadDetailPageCursor(window.beforeCursor); + const cursor = decodedCursor?.threadId === threadId ? decodedCursor : null; + + const windowRows = yield* listTurnWindowRows({ + threadId, + beforeAnchorAt: cursor?.beforeAnchorAt ?? ANCHOR_UNBOUNDED, + beforeTurnKey: cursor?.beforeTurnId ?? "", + userTurnLimit: window.turnLimit, + maxRawTurns: THREAD_DETAIL_MAX_RAW_TURNS_PER_PAGE, + }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailSnapshot:listTurnWindow:query", + "ProjectionSnapshotQuery.getThreadDetailSnapshot:listTurnWindow:decodeRows", + ), + ), + ); + + const oldest = windowRows[0]; + // An empty window (no turns before the cursor, or a thread with no + // turns at all) still returns thread metadata with empty collections + // for turn-linked rows; turnless rows are bounded to the same empty + // range. The first page of a turnless thread stays unwindowed so + // pre-turn content (e.g. a just-created thread) is not hidden. + const bounds: ThreadDetailBounds | undefined = + oldest === undefined && cursor === null + ? undefined + : { + minAnchorAt: oldest?.anchorAt ?? "", + minTurnKey: oldest?.turnKey ?? "", + beforeAnchorAt: cursor?.beforeAnchorAt ?? ANCHOR_UNBOUNDED, + beforeTurnKey: cursor?.beforeTurnId ?? "", + }; + // Empty window behind a cursor: nothing older remains. + const emptyBounds = + oldest === undefined && cursor !== null + ? { minAnchorAt: "", minTurnKey: "", beforeAnchorAt: "", beforeTurnKey: "" } + : undefined; + + const thread = yield* getThreadDetailByIdBounded(threadId, emptyBounds ?? bounds, { + mode: "client", + }); + if (Option.isNone(thread)) { + return Option.none(); + } + + const hasMore = + oldest !== undefined && + (yield* listTurnWindowRows({ + threadId, + beforeAnchorAt: oldest.anchorAt, + beforeTurnKey: oldest.turnKey, + userTurnLimit: 1, + maxRawTurns: 1, + }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailSnapshot:probeOlder:query", + "ProjectionSnapshotQuery.getThreadDetailSnapshot:probeOlder:decodeRows", + ), + ), + )).length > 0; + + const { snapshotSequence } = yield* getSnapshotSequence(); + const watermarkRow = yield* getThreadEventWatermarkRow({ + threadId, + maxSequence: snapshotSequence, + }).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionSnapshotQuery.getThreadDetailSnapshot:threadWatermark:query", + "ProjectionSnapshotQuery.getThreadDetailSnapshot:threadWatermark:decodeRow", + ), + ), + ); + const threadSequence = Option.match(watermarkRow, { + onNone: () => 0, + onSome: (row) => row.threadSequence ?? 0, + }); + return Option.some({ + snapshotSequence, + thread: thread.value, + page: { + beforeCursor: + hasMore && oldest !== undefined + ? encodeThreadDetailPageCursor({ + threadId, + beforeAnchorAt: oldest.anchorAt, + beforeTurnId: oldest.turnKey, + }) + : null, + hasMore, + snapshotSequence, + threadSequence, + }, + }); + }), + ).pipe( + Effect.mapError((error) => + isPersistenceError(error) + ? error + : toPersistenceSqlError("ProjectionSnapshotQuery.getThreadDetailSnapshot:transaction")( + error, + ), + ), + ); + + return { + getCommandReadModel, + getSnapshot, + getShellSnapshot, + getArchivedShellSnapshot, + searchThreads, + getSnapshotSequence, + getCounts, + getActiveProjectByWorkspaceRoot, + getProjectShellById, + getFirstActiveThreadIdByProjectId, + getThreadCheckpointContext, + getFullThreadDiffContext, + getThreadShellById, + getThreadDetailById, + getThreadDetailSnapshot, + } satisfies ProjectionSnapshotQueryShape; +}); + +export const PgProjectionSnapshotQueryLive = Layer.effect( + ProjectionSnapshotQuery, + makeProjectionSnapshotQuery, +); diff --git a/apps/server/src/persistence/Postgres/ProjectionState.ts b/apps/server/src/persistence/Postgres/ProjectionState.ts new file mode 100644 index 000000000000..1811cd9d0271 --- /dev/null +++ b/apps/server/src/persistence/Postgres/ProjectionState.ts @@ -0,0 +1,125 @@ +import { NonNegativeInt } from "@t3tools/contracts"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import { toPersistenceSqlError } from "../Errors.ts"; +import { HubTenant } from "./HubTenant.ts"; + +import { + ProjectionStateRepository, + type ProjectionStateRepositoryShape, + GetProjectionStateInput, + ProjectionState, +} from "../Services/ProjectionState.ts"; + +const MinLastAppliedSequenceRowSchema = Schema.Struct({ + minLastAppliedSequence: Schema.NullOr(NonNegativeInt), +}); + +const makeProjectionStateRepository = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const upsertProjectionStateRow = SqlSchema.void({ + Request: ProjectionState, + execute: (row) => + sql` + INSERT INTO projection_state ( + user_id, + projector, + last_applied_sequence, + updated_at + ) + VALUES ( + ${userId}, + ${row.projector}, + ${row.lastAppliedSequence}, + ${row.updatedAt} + ) + ON CONFLICT (user_id, projector) + DO UPDATE SET + last_applied_sequence = excluded.last_applied_sequence, + updated_at = excluded.updated_at + `, + }); + + const getProjectionStateRow = SqlSchema.findOneOption({ + Request: GetProjectionStateInput, + Result: ProjectionState, + execute: ({ projector }) => + sql` + SELECT + projector, + last_applied_sequence AS "lastAppliedSequence", + updated_at AS "updatedAt" + FROM projection_state + WHERE user_id = ${userId} + AND projector = ${projector} + `, + }); + + const listProjectionStateRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionState, + execute: () => + sql` + SELECT + projector, + last_applied_sequence AS "lastAppliedSequence", + updated_at AS "updatedAt" + FROM projection_state + WHERE user_id = ${userId} + ORDER BY projector ASC + `, + }); + + const readMinLastAppliedSequence = SqlSchema.findOne({ + Request: Schema.Void, + Result: MinLastAppliedSequenceRowSchema, + execute: () => + sql` + SELECT + MIN(last_applied_sequence) AS "minLastAppliedSequence" + FROM projection_state + WHERE user_id = ${userId} + `, + }); + + const upsert: ProjectionStateRepositoryShape["upsert"] = (row) => + upsertProjectionStateRow(row).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionStateRepository.upsert:query")), + ); + + const getByProjector: ProjectionStateRepositoryShape["getByProjector"] = (input) => + getProjectionStateRow(input).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionStateRepository.getByProjector:query")), + ); + + const listAll: ProjectionStateRepositoryShape["listAll"] = () => + listProjectionStateRows(undefined).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionStateRepository.listAll:query")), + ); + + const minLastAppliedSequence: ProjectionStateRepositoryShape["minLastAppliedSequence"] = () => + readMinLastAppliedSequence(undefined).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionStateRepository.minLastAppliedSequence:query"), + ), + Effect.map((row) => row.minLastAppliedSequence), + ); + + return { + upsert, + getByProjector, + listAll, + minLastAppliedSequence, + } satisfies ProjectionStateRepositoryShape; +}); + +export const PgProjectionStateRepositoryLive = Layer.effect( + ProjectionStateRepository, + makeProjectionStateRepository, +); diff --git a/apps/server/src/persistence/Postgres/ProjectionThreadActivities.ts b/apps/server/src/persistence/Postgres/ProjectionThreadActivities.ts new file mode 100644 index 000000000000..4b26a02afb77 --- /dev/null +++ b/apps/server/src/persistence/Postgres/ProjectionThreadActivities.ts @@ -0,0 +1,212 @@ +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; +import { NonNegativeInt } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Struct from "effect/Struct"; + +import { toPersistenceDecodeError, toPersistenceSqlError } from "../Errors.ts"; +import { HubTenant } from "./HubTenant.ts"; + +import { + DeleteProjectionThreadActivitiesInput, + ListProjectionThreadActivitiesInput, + ProjectionThreadActivity, + ProjectionThreadActivityRepository, + type ProjectionThreadActivityRepositoryShape, +} from "../Services/ProjectionThreadActivities.ts"; + +const ProjectionThreadActivityDbRowSchema = ProjectionThreadActivity.mapFields( + Struct.assign({ + payload: Schema.fromJsonString(Schema.Unknown), + sequence: Schema.NullOr(NonNegativeInt), + }), +); + +const mapActivityRows = ( + rows: ReadonlyArray>, +): ReadonlyArray => + rows.map((row) => ({ + activityId: row.activityId, + threadId: row.threadId, + turnId: row.turnId, + tone: row.tone, + kind: row.kind, + summary: row.summary, + payload: row.payload, + ...(row.sequence !== null ? { sequence: row.sequence } : {}), + createdAt: row.createdAt, + })); + +function toPersistenceSqlOrDecodeError(sqlOperation: string, decodeOperation: string) { + return (cause: unknown) => + Schema.isSchemaError(cause) + ? toPersistenceDecodeError(decodeOperation)(cause) + : toPersistenceSqlError(sqlOperation)(cause); +} + +const makeProjectionThreadActivityRepository = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const upsertProjectionThreadActivityRow = SqlSchema.void({ + Request: ProjectionThreadActivity, + execute: (row) => + sql` + INSERT INTO projection_thread_activities ( + user_id, + activity_id, + thread_id, + turn_id, + tone, + kind, + summary, + payload_json, + sequence, + created_at + ) + VALUES ( + ${userId}, + ${row.activityId}, + ${row.threadId}, + ${row.turnId}, + ${row.tone}, + ${row.kind}, + ${row.summary}, + ${JSON.stringify(row.payload)}, + ${row.sequence ?? null}, + ${row.createdAt} + ) + ON CONFLICT (user_id, activity_id) + DO UPDATE SET + thread_id = excluded.thread_id, + turn_id = excluded.turn_id, + tone = excluded.tone, + kind = excluded.kind, + summary = excluded.summary, + payload_json = excluded.payload_json, + sequence = excluded.sequence, + created_at = excluded.created_at + `, + }); + + const listProjectionThreadActivityRows = SqlSchema.findAll({ + Request: ListProjectionThreadActivitiesInput, + Result: ProjectionThreadActivityDbRowSchema, + execute: ({ threadId }) => + sql` + SELECT + activity_id AS "activityId", + thread_id AS "threadId", + turn_id AS "turnId", + tone, + kind, + summary, + payload_json AS "payload", + sequence, + created_at AS "createdAt" + FROM projection_thread_activities + WHERE user_id = ${userId} + AND thread_id = ${threadId} + ORDER BY + CASE WHEN sequence IS NULL THEN 0 ELSE 1 END ASC, + sequence ASC, + created_at ASC, + activity_id ASC + `, + }); + + const listUserInputLifecycleActivityRows = SqlSchema.findAll({ + Request: ListProjectionThreadActivitiesInput, + Result: ProjectionThreadActivityDbRowSchema, + execute: ({ threadId }) => + sql` + SELECT + activity_id AS "activityId", + thread_id AS "threadId", + turn_id AS "turnId", + tone, + kind, + summary, + payload_json AS "payload", + sequence, + created_at AS "createdAt" + FROM projection_thread_activities + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND kind IN ( + 'user-input.requested', + 'user-input.resolved', + 'provider.user-input.respond.failed' + ) + ORDER BY + CASE WHEN sequence IS NULL THEN 0 ELSE 1 END ASC, + sequence ASC, + created_at ASC, + activity_id ASC + `, + }); + + const deleteProjectionThreadActivityRows = SqlSchema.void({ + Request: DeleteProjectionThreadActivitiesInput, + execute: ({ threadId }) => + sql` + DELETE FROM projection_thread_activities + WHERE user_id = ${userId} + AND thread_id = ${threadId} + `, + }); + + const upsert: ProjectionThreadActivityRepositoryShape["upsert"] = (row) => + upsertProjectionThreadActivityRow(row).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionThreadActivityRepository.upsert:query", + "ProjectionThreadActivityRepository.upsert:encodeRequest", + ), + ), + ); + + const listByThreadId: ProjectionThreadActivityRepositoryShape["listByThreadId"] = (input) => + listProjectionThreadActivityRows(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionThreadActivityRepository.listByThreadId:query", + "ProjectionThreadActivityRepository.listByThreadId:decodeRows", + ), + ), + Effect.map(mapActivityRows), + ); + + const listUserInputLifecycleByThreadId: ProjectionThreadActivityRepositoryShape["listUserInputLifecycleByThreadId"] = + (input) => + listUserInputLifecycleActivityRows(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionThreadActivityRepository.listUserInputLifecycleByThreadId:query", + "ProjectionThreadActivityRepository.listUserInputLifecycleByThreadId:decodeRows", + ), + ), + Effect.map(mapActivityRows), + ); + + const deleteByThreadId: ProjectionThreadActivityRepositoryShape["deleteByThreadId"] = (input) => + deleteProjectionThreadActivityRows(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadActivityRepository.deleteByThreadId:query"), + ), + ); + + return { + upsert, + listByThreadId, + listUserInputLifecycleByThreadId, + deleteByThreadId, + } satisfies ProjectionThreadActivityRepositoryShape; +}); + +export const PgProjectionThreadActivityRepositoryLive = Layer.effect( + ProjectionThreadActivityRepository, + makeProjectionThreadActivityRepository, +); diff --git a/apps/server/src/persistence/Postgres/ProjectionThreadMessages.ts b/apps/server/src/persistence/Postgres/ProjectionThreadMessages.ts new file mode 100644 index 000000000000..36478dc8a815 --- /dev/null +++ b/apps/server/src/persistence/Postgres/ProjectionThreadMessages.ts @@ -0,0 +1,251 @@ +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as Struct from "effect/Struct"; +import { ChatAttachment } from "@t3tools/contracts"; + +import { toPersistenceSqlError } from "../Errors.ts"; +import { HubTenant } from "./HubTenant.ts"; +import { + AppendStreamingProjectionThreadMessage, + GetProjectionThreadMessageInput, + ProjectionThreadMessageRepository, + type ProjectionThreadMessageRepositoryShape, + DeleteProjectionThreadMessagesInput, + ListProjectionThreadMessagesInput, + ProjectionThreadMessage, +} from "../Services/ProjectionThreadMessages.ts"; + +const ProjectionThreadMessageDbRowSchema = ProjectionThreadMessage.mapFields( + Struct.assign({ + isStreaming: Schema.Number, + attachments: Schema.NullOr(Schema.fromJsonString(Schema.Array(ChatAttachment))), + }), +); + +function toProjectionThreadMessage( + row: Schema.Schema.Type, +): ProjectionThreadMessage { + return { + messageId: row.messageId, + threadId: row.threadId, + turnId: row.turnId, + role: row.role, + text: row.text, + isStreaming: row.isStreaming === 1, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + ...(row.attachments !== null ? { attachments: row.attachments } : {}), + }; +} + +const makeProjectionThreadMessageRepository = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const upsertProjectionThreadMessageRow = SqlSchema.void({ + Request: ProjectionThreadMessage, + execute: (row) => { + const nextAttachmentsJson = + row.attachments !== undefined ? JSON.stringify(row.attachments) : null; + return sql` + INSERT INTO projection_thread_messages ( + user_id, + message_id, + thread_id, + turn_id, + role, + text, + attachments_json, + is_streaming, + created_at, + updated_at + ) + VALUES ( + ${userId}, + ${row.messageId}, + ${row.threadId}, + ${row.turnId}, + ${row.role}, + ${row.text}, + COALESCE( + ${nextAttachmentsJson}, + ( + SELECT attachments_json + FROM projection_thread_messages + WHERE user_id = ${userId} + AND message_id = ${row.messageId} + ) + ), + ${row.isStreaming ? 1 : 0}, + ${row.createdAt}, + ${row.updatedAt} + ) + ON CONFLICT (user_id, message_id) + DO UPDATE SET + thread_id = excluded.thread_id, + turn_id = excluded.turn_id, + role = excluded.role, + text = excluded.text, + attachments_json = COALESCE( + excluded.attachments_json, + projection_thread_messages.attachments_json + ), + is_streaming = excluded.is_streaming, + created_at = excluded.created_at, + updated_at = excluded.updated_at + `; + }, + }); + + const appendStreamingProjectionThreadMessageRow = SqlSchema.void({ + Request: AppendStreamingProjectionThreadMessage, + execute: (row) => { + const nextAttachmentsJson = + row.attachments !== undefined ? JSON.stringify(row.attachments) : null; + return sql` + INSERT INTO projection_thread_messages ( + user_id, + message_id, + thread_id, + turn_id, + role, + text, + attachments_json, + is_streaming, + created_at, + updated_at + ) + VALUES ( + ${userId}, + ${row.messageId}, + ${row.threadId}, + ${row.turnId}, + ${row.role}, + ${row.text}, + ${nextAttachmentsJson}, + 1, + ${row.createdAt}, + ${row.updatedAt} + ) + ON CONFLICT (user_id, message_id) + DO UPDATE SET + thread_id = excluded.thread_id, + turn_id = excluded.turn_id, + role = excluded.role, + text = projection_thread_messages.text || excluded.text, + attachments_json = COALESCE( + excluded.attachments_json, + projection_thread_messages.attachments_json + ), + is_streaming = 1, + updated_at = excluded.updated_at + `; + }, + }); + + const getProjectionThreadMessageRow = SqlSchema.findOneOption({ + Request: GetProjectionThreadMessageInput, + Result: ProjectionThreadMessageDbRowSchema, + execute: ({ messageId }) => + sql` + SELECT + message_id AS "messageId", + thread_id AS "threadId", + turn_id AS "turnId", + role, + text, + attachments_json AS "attachments", + is_streaming AS "isStreaming", + created_at AS "createdAt", + updated_at AS "updatedAt" + FROM projection_thread_messages + WHERE user_id = ${userId} + AND message_id = ${messageId} + LIMIT 1 + `, + }); + + const listProjectionThreadMessageRows = SqlSchema.findAll({ + Request: ListProjectionThreadMessagesInput, + Result: ProjectionThreadMessageDbRowSchema, + execute: ({ threadId }) => + sql` + SELECT + message_id AS "messageId", + thread_id AS "threadId", + turn_id AS "turnId", + role, + text, + attachments_json AS "attachments", + is_streaming AS "isStreaming", + created_at AS "createdAt", + updated_at AS "updatedAt" + FROM projection_thread_messages + WHERE user_id = ${userId} + AND thread_id = ${threadId} + ORDER BY created_at ASC, message_id ASC + `, + }); + + const deleteProjectionThreadMessageRows = SqlSchema.void({ + Request: DeleteProjectionThreadMessagesInput, + execute: ({ threadId }) => + sql` + DELETE FROM projection_thread_messages + WHERE user_id = ${userId} + AND thread_id = ${threadId} + `, + }); + + const upsert: ProjectionThreadMessageRepositoryShape["upsert"] = (row) => + upsertProjectionThreadMessageRow(row).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionThreadMessageRepository.upsert:query")), + ); + + const appendStreaming: ProjectionThreadMessageRepositoryShape["appendStreaming"] = (row) => + appendStreamingProjectionThreadMessageRow(row).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadMessageRepository.appendStreaming:query"), + ), + ); + + const getByMessageId: ProjectionThreadMessageRepositoryShape["getByMessageId"] = (input) => + getProjectionThreadMessageRow(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadMessageRepository.getByMessageId:query"), + ), + Effect.map(Option.map(toProjectionThreadMessage)), + ); + + const listByThreadId: ProjectionThreadMessageRepositoryShape["listByThreadId"] = (input) => + listProjectionThreadMessageRows(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadMessageRepository.listByThreadId:query"), + ), + Effect.map((rows) => rows.map(toProjectionThreadMessage)), + ); + + const deleteByThreadId: ProjectionThreadMessageRepositoryShape["deleteByThreadId"] = (input) => + deleteProjectionThreadMessageRows(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadMessageRepository.deleteByThreadId:query"), + ), + ); + + return { + upsert, + appendStreaming, + getByMessageId, + listByThreadId, + deleteByThreadId, + } satisfies ProjectionThreadMessageRepositoryShape; +}); + +export const PgProjectionThreadMessageRepositoryLive = Layer.effect( + ProjectionThreadMessageRepository, + makeProjectionThreadMessageRepository, +); diff --git a/apps/server/src/persistence/Postgres/ProjectionThreadProposedPlans.ts b/apps/server/src/persistence/Postgres/ProjectionThreadProposedPlans.ts new file mode 100644 index 000000000000..50573d138800 --- /dev/null +++ b/apps/server/src/persistence/Postgres/ProjectionThreadProposedPlans.ts @@ -0,0 +1,117 @@ +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; + +import { toPersistenceSqlError } from "../Errors.ts"; +import { HubTenant } from "./HubTenant.ts"; +import { + DeleteProjectionThreadProposedPlansInput, + ListProjectionThreadProposedPlansInput, + ProjectionThreadProposedPlan, + ProjectionThreadProposedPlanRepository, + type ProjectionThreadProposedPlanRepositoryShape, +} from "../Services/ProjectionThreadProposedPlans.ts"; + +const makeProjectionThreadProposedPlanRepository = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const upsertProjectionThreadProposedPlanRow = SqlSchema.void({ + Request: ProjectionThreadProposedPlan, + execute: (row) => sql` + INSERT INTO projection_thread_proposed_plans ( + user_id, + plan_id, + thread_id, + turn_id, + plan_markdown, + implemented_at, + implementation_thread_id, + created_at, + updated_at + ) + VALUES ( + ${userId}, + ${row.planId}, + ${row.threadId}, + ${row.turnId}, + ${row.planMarkdown}, + ${row.implementedAt}, + ${row.implementationThreadId}, + ${row.createdAt}, + ${row.updatedAt} + ) + ON CONFLICT (user_id, plan_id) + DO UPDATE SET + thread_id = excluded.thread_id, + turn_id = excluded.turn_id, + plan_markdown = excluded.plan_markdown, + implemented_at = excluded.implemented_at, + implementation_thread_id = excluded.implementation_thread_id, + created_at = excluded.created_at, + updated_at = excluded.updated_at + `, + }); + + const listProjectionThreadProposedPlanRows = SqlSchema.findAll({ + Request: ListProjectionThreadProposedPlansInput, + Result: ProjectionThreadProposedPlan, + execute: ({ threadId }) => sql` + SELECT + plan_id AS "planId", + thread_id AS "threadId", + turn_id AS "turnId", + plan_markdown AS "planMarkdown", + implemented_at AS "implementedAt", + implementation_thread_id AS "implementationThreadId", + created_at AS "createdAt", + updated_at AS "updatedAt" + FROM projection_thread_proposed_plans + WHERE user_id = ${userId} + AND thread_id = ${threadId} + ORDER BY created_at ASC, plan_id ASC + `, + }); + + const deleteProjectionThreadProposedPlanRows = SqlSchema.void({ + Request: DeleteProjectionThreadProposedPlansInput, + execute: ({ threadId }) => sql` + DELETE FROM projection_thread_proposed_plans + WHERE user_id = ${userId} + AND thread_id = ${threadId} + `, + }); + + const upsert: ProjectionThreadProposedPlanRepositoryShape["upsert"] = (row) => + upsertProjectionThreadProposedPlanRow(row).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionThreadProposedPlanRepository.upsert:query")), + ); + + const listByThreadId: ProjectionThreadProposedPlanRepositoryShape["listByThreadId"] = (input) => + listProjectionThreadProposedPlanRows(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadProposedPlanRepository.listByThreadId:query"), + ), + ); + + const deleteByThreadId: ProjectionThreadProposedPlanRepositoryShape["deleteByThreadId"] = ( + input, + ) => + deleteProjectionThreadProposedPlanRows(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadProposedPlanRepository.deleteByThreadId:query"), + ), + ); + + return { + upsert, + listByThreadId, + deleteByThreadId, + } satisfies ProjectionThreadProposedPlanRepositoryShape; +}); + +export const PgProjectionThreadProposedPlanRepositoryLive = Layer.effect( + ProjectionThreadProposedPlanRepository, + makeProjectionThreadProposedPlanRepository, +); diff --git a/apps/server/src/persistence/Postgres/ProjectionThreadSessions.ts b/apps/server/src/persistence/Postgres/ProjectionThreadSessions.ts new file mode 100644 index 000000000000..6964090f4f9b --- /dev/null +++ b/apps/server/src/persistence/Postgres/ProjectionThreadSessions.ts @@ -0,0 +1,118 @@ +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import { toPersistenceSqlError } from "../Errors.ts"; +import { HubTenant } from "./HubTenant.ts"; + +import { + ProjectionThreadSession, + ProjectionThreadSessionRepository, + type ProjectionThreadSessionRepositoryShape, + DeleteProjectionThreadSessionInput, + GetProjectionThreadSessionInput, +} from "../Services/ProjectionThreadSessions.ts"; + +const makeProjectionThreadSessionRepository = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const upsertProjectionThreadSessionRow = SqlSchema.void({ + Request: ProjectionThreadSession, + execute: (row) => + sql` + INSERT INTO projection_thread_sessions ( + user_id, + thread_id, + status, + provider_name, + provider_instance_id, + runtime_mode, + active_turn_id, + last_error, + updated_at + ) + VALUES ( + ${userId}, + ${row.threadId}, + ${row.status}, + ${row.providerName}, + ${row.providerInstanceId}, + ${row.runtimeMode}, + ${row.activeTurnId}, + ${row.lastError}, + ${row.updatedAt} + ) + ON CONFLICT (user_id, thread_id) + DO UPDATE SET + status = excluded.status, + provider_name = excluded.provider_name, + provider_instance_id = excluded.provider_instance_id, + runtime_mode = excluded.runtime_mode, + active_turn_id = excluded.active_turn_id, + last_error = excluded.last_error, + updated_at = excluded.updated_at + `, + }); + + const getProjectionThreadSessionRow = SqlSchema.findOneOption({ + Request: GetProjectionThreadSessionInput, + Result: ProjectionThreadSession, + execute: ({ threadId }) => + sql` + SELECT + thread_id AS "threadId", + status, + provider_name AS "providerName", + provider_instance_id AS "providerInstanceId", + runtime_mode AS "runtimeMode", + active_turn_id AS "activeTurnId", + last_error AS "lastError", + updated_at AS "updatedAt" + FROM projection_thread_sessions + WHERE user_id = ${userId} + AND thread_id = ${threadId} + `, + }); + + const deleteProjectionThreadSessionRow = SqlSchema.void({ + Request: DeleteProjectionThreadSessionInput, + execute: ({ threadId }) => + sql` + DELETE FROM projection_thread_sessions + WHERE user_id = ${userId} + AND thread_id = ${threadId} + `, + }); + + const upsert: ProjectionThreadSessionRepositoryShape["upsert"] = (row) => + upsertProjectionThreadSessionRow(row).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionThreadSessionRepository.upsert:query")), + ); + + const getByThreadId: ProjectionThreadSessionRepositoryShape["getByThreadId"] = (input) => + getProjectionThreadSessionRow(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadSessionRepository.getByThreadId:query"), + ), + ); + + const deleteByThreadId: ProjectionThreadSessionRepositoryShape["deleteByThreadId"] = (input) => + deleteProjectionThreadSessionRow(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadSessionRepository.deleteByThreadId:query"), + ), + ); + + return { + upsert, + getByThreadId, + deleteByThreadId, + } satisfies ProjectionThreadSessionRepositoryShape; +}); + +export const PgProjectionThreadSessionRepositoryLive = Layer.effect( + ProjectionThreadSessionRepository, + makeProjectionThreadSessionRepository, +); diff --git a/apps/server/src/persistence/Postgres/ProjectionThreads.ts b/apps/server/src/persistence/Postgres/ProjectionThreads.ts new file mode 100644 index 000000000000..c3c5432ac57d --- /dev/null +++ b/apps/server/src/persistence/Postgres/ProjectionThreads.ts @@ -0,0 +1,247 @@ +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Struct from "effect/Struct"; + +import { toPersistenceSqlError } from "../Errors.ts"; +import { HubTenant } from "./HubTenant.ts"; +import { + DeleteProjectionThreadInput, + GetProjectionThreadInput, + ListProjectionThreadsByProjectInput, + ProjectionThread, + ProjectionThreadRepository, + type ProjectionThreadRepositoryShape, +} from "../Services/ProjectionThreads.ts"; +import { ModelSelection, ThreadLinkedPullRequest } from "@t3tools/contracts"; + +const ProjectionThreadDbRow = ProjectionThread.mapFields( + Struct.assign({ + modelSelection: Schema.fromJsonString(ModelSelection), + linkedPullRequest: Schema.NullOr(Schema.fromJsonString(ThreadLinkedPullRequest)), + }), +); +type ProjectionThreadDbRow = typeof ProjectionThreadDbRow.Type; + +const makeProjectionThreadRepository = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const upsertProjectionThreadRow = SqlSchema.void({ + Request: ProjectionThread, + execute: (row) => + sql` + INSERT INTO projection_threads ( + user_id, + thread_id, + project_id, + title, + model_selection_json, + runtime_mode, + interaction_mode, + branch, + worktree_path, + linked_pull_request_json, + latest_turn_id, + created_at, + updated_at, + archived_at, + settled_override, + settled_at, + unsettled_at, + snoozed_until, + snoozed_at, + pinned_at, + pin_order_key, + title_regeneration_request_id, + title_regeneration_started_at, + latest_user_message_at, + pending_approval_count, + pending_user_input_count, + has_actionable_proposed_plan, + deleted_at + ) + VALUES ( + ${userId}, + ${row.threadId}, + ${row.projectId}, + ${row.title}, + ${JSON.stringify(row.modelSelection)}, + ${row.runtimeMode}, + ${row.interactionMode}, + ${row.branch}, + ${row.worktreePath}, + ${row.linkedPullRequest === undefined || row.linkedPullRequest === null ? null : JSON.stringify(row.linkedPullRequest)}, + ${row.latestTurnId}, + ${row.createdAt}, + ${row.updatedAt}, + ${row.archivedAt}, + ${row.settledOverride}, + ${row.settledAt}, + ${row.unsettledAt}, + ${row.snoozedUntil}, + ${row.snoozedAt}, + ${row.pinnedAt}, + ${row.pinOrderKey ?? null}, + ${row.titleRegenerationRequestId ?? null}, + ${row.titleRegenerationStartedAt ?? null}, + ${row.latestUserMessageAt}, + ${row.pendingApprovalCount}, + ${row.pendingUserInputCount}, + ${row.hasActionableProposedPlan}, + ${row.deletedAt} + ) + ON CONFLICT (user_id, thread_id) + DO UPDATE SET + project_id = excluded.project_id, + title = excluded.title, + model_selection_json = excluded.model_selection_json, + runtime_mode = excluded.runtime_mode, + interaction_mode = excluded.interaction_mode, + branch = excluded.branch, + worktree_path = excluded.worktree_path, + linked_pull_request_json = excluded.linked_pull_request_json, + latest_turn_id = excluded.latest_turn_id, + created_at = excluded.created_at, + updated_at = excluded.updated_at, + archived_at = excluded.archived_at, + settled_override = excluded.settled_override, + settled_at = excluded.settled_at, + unsettled_at = excluded.unsettled_at, + snoozed_until = excluded.snoozed_until, + snoozed_at = excluded.snoozed_at, + pinned_at = excluded.pinned_at, + pin_order_key = excluded.pin_order_key, + title_regeneration_request_id = excluded.title_regeneration_request_id, + title_regeneration_started_at = excluded.title_regeneration_started_at, + latest_user_message_at = excluded.latest_user_message_at, + pending_approval_count = excluded.pending_approval_count, + pending_user_input_count = excluded.pending_user_input_count, + has_actionable_proposed_plan = excluded.has_actionable_proposed_plan, + deleted_at = excluded.deleted_at + `, + }); + + const getProjectionThreadRow = SqlSchema.findOneOption({ + Request: GetProjectionThreadInput, + Result: ProjectionThreadDbRow, + execute: ({ threadId }) => + sql` + SELECT + thread_id AS "threadId", + project_id AS "projectId", + title, + model_selection_json AS "modelSelection", + runtime_mode AS "runtimeMode", + interaction_mode AS "interactionMode", + branch, + worktree_path AS "worktreePath", + linked_pull_request_json AS "linkedPullRequest", + latest_turn_id AS "latestTurnId", + created_at AS "createdAt", + updated_at AS "updatedAt", + archived_at AS "archivedAt", + settled_override AS "settledOverride", + settled_at AS "settledAt", + unsettled_at AS "unsettledAt", + snoozed_until AS "snoozedUntil", + snoozed_at AS "snoozedAt", + pinned_at AS "pinnedAt", + pin_order_key AS "pinOrderKey", + title_regeneration_request_id AS "titleRegenerationRequestId", + title_regeneration_started_at AS "titleRegenerationStartedAt", + latest_user_message_at AS "latestUserMessageAt", + pending_approval_count AS "pendingApprovalCount", + pending_user_input_count AS "pendingUserInputCount", + has_actionable_proposed_plan AS "hasActionableProposedPlan", + deleted_at AS "deletedAt" + FROM projection_threads + WHERE user_id = ${userId} + AND thread_id = ${threadId} + `, + }); + + const listProjectionThreadRows = SqlSchema.findAll({ + Request: ListProjectionThreadsByProjectInput, + Result: ProjectionThreadDbRow, + execute: ({ projectId }) => + sql` + SELECT + thread_id AS "threadId", + project_id AS "projectId", + title, + model_selection_json AS "modelSelection", + runtime_mode AS "runtimeMode", + interaction_mode AS "interactionMode", + branch, + worktree_path AS "worktreePath", + linked_pull_request_json AS "linkedPullRequest", + latest_turn_id AS "latestTurnId", + created_at AS "createdAt", + updated_at AS "updatedAt", + archived_at AS "archivedAt", + settled_override AS "settledOverride", + settled_at AS "settledAt", + unsettled_at AS "unsettledAt", + snoozed_until AS "snoozedUntil", + snoozed_at AS "snoozedAt", + pinned_at AS "pinnedAt", + pin_order_key AS "pinOrderKey", + title_regeneration_request_id AS "titleRegenerationRequestId", + title_regeneration_started_at AS "titleRegenerationStartedAt", + latest_user_message_at AS "latestUserMessageAt", + pending_approval_count AS "pendingApprovalCount", + pending_user_input_count AS "pendingUserInputCount", + has_actionable_proposed_plan AS "hasActionableProposedPlan", + deleted_at AS "deletedAt" + FROM projection_threads + WHERE user_id = ${userId} + AND project_id = ${projectId} + ORDER BY created_at ASC, thread_id ASC + `, + }); + + const deleteProjectionThreadRow = SqlSchema.void({ + Request: DeleteProjectionThreadInput, + execute: ({ threadId }) => + sql` + DELETE FROM projection_threads + WHERE user_id = ${userId} + AND thread_id = ${threadId} + `, + }); + + const upsert: ProjectionThreadRepositoryShape["upsert"] = (row) => + upsertProjectionThreadRow(row).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionThreadRepository.upsert:query")), + ); + + const getById: ProjectionThreadRepositoryShape["getById"] = (input) => + getProjectionThreadRow(input).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionThreadRepository.getById:query")), + ); + + const listByProjectId: ProjectionThreadRepositoryShape["listByProjectId"] = (input) => + listProjectionThreadRows(input).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionThreadRepository.listByProjectId:query")), + ); + + const deleteById: ProjectionThreadRepositoryShape["deleteById"] = (input) => + deleteProjectionThreadRow(input).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionThreadRepository.deleteById:query")), + ); + + return { + upsert, + getById, + listByProjectId, + deleteById, + } satisfies ProjectionThreadRepositoryShape; +}); + +export const PgProjectionThreadRepositoryLive = Layer.effect( + ProjectionThreadRepository, + makeProjectionThreadRepository, +); diff --git a/apps/server/src/persistence/Postgres/ProjectionTurns.ts b/apps/server/src/persistence/Postgres/ProjectionTurns.ts new file mode 100644 index 000000000000..07a884e914f7 --- /dev/null +++ b/apps/server/src/persistence/Postgres/ProjectionTurns.ts @@ -0,0 +1,400 @@ +import { OrchestrationCheckpointFile } from "@t3tools/contracts"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as Struct from "effect/Struct"; + +import { toPersistenceDecodeError, toPersistenceSqlError } from "../Errors.ts"; +import { HubTenant } from "./HubTenant.ts"; +import { + ClearCheckpointTurnConflictInput, + DeleteProjectionTurnsByThreadInput, + GetProjectionPendingTurnStartInput, + GetProjectionTurnByTurnIdInput, + ListProjectionTurnsByThreadInput, + ProjectionPendingTurnStart, + ProjectionTurn, + ProjectionTurnById, + ProjectionTurnRepository, + type ProjectionTurnRepositoryShape, +} from "../Services/ProjectionTurns.ts"; + +const ProjectionTurnDbRowSchema = ProjectionTurn.mapFields( + Struct.assign({ + checkpointFiles: Schema.fromJsonString(Schema.Array(OrchestrationCheckpointFile)), + }), +); + +const ProjectionTurnByIdDbRowSchema = ProjectionTurnById.mapFields( + Struct.assign({ + checkpointFiles: Schema.fromJsonString(Schema.Array(OrchestrationCheckpointFile)), + }), +); + +function toPersistenceSqlOrDecodeError(sqlOperation: string, decodeOperation: string) { + return (cause: unknown) => + Schema.isSchemaError(cause) + ? toPersistenceDecodeError(decodeOperation)(cause) + : toPersistenceSqlError(sqlOperation)(cause); +} + +const makeProjectionTurnRepository = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const upsertProjectionTurnById = SqlSchema.void({ + Request: ProjectionTurnByIdDbRowSchema, + execute: (row) => + sql` + INSERT INTO projection_turns ( + user_id, + thread_id, + turn_id, + pending_message_id, + source_proposed_plan_thread_id, + source_proposed_plan_id, + assistant_message_id, + state, + requested_at, + started_at, + completed_at, + checkpoint_turn_count, + checkpoint_ref, + checkpoint_status, + checkpoint_files_json + ) + VALUES ( + ${userId}, + ${row.threadId}, + ${row.turnId}, + ${row.pendingMessageId}, + ${row.sourceProposedPlanThreadId}, + ${row.sourceProposedPlanId}, + ${row.assistantMessageId}, + ${row.state}, + ${row.requestedAt}, + ${row.startedAt}, + ${row.completedAt}, + ${row.checkpointTurnCount}, + ${row.checkpointRef}, + ${row.checkpointStatus}, + ${row.checkpointFiles} + ) + ON CONFLICT (user_id, thread_id, turn_id) + DO UPDATE SET + pending_message_id = excluded.pending_message_id, + source_proposed_plan_thread_id = excluded.source_proposed_plan_thread_id, + source_proposed_plan_id = excluded.source_proposed_plan_id, + assistant_message_id = excluded.assistant_message_id, + state = excluded.state, + requested_at = excluded.requested_at, + started_at = excluded.started_at, + completed_at = excluded.completed_at, + checkpoint_turn_count = excluded.checkpoint_turn_count, + checkpoint_ref = excluded.checkpoint_ref, + checkpoint_status = excluded.checkpoint_status, + checkpoint_files_json = excluded.checkpoint_files_json + `, + }); + + const clearPendingProjectionTurnsByThread = SqlSchema.void({ + Request: DeleteProjectionTurnsByThreadInput, + execute: ({ threadId }) => + sql` + DELETE FROM projection_turns + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND turn_id IS NULL + AND state = 'pending' + AND checkpoint_turn_count IS NULL + `, + }); + + const insertPendingProjectionTurn = SqlSchema.void({ + Request: ProjectionPendingTurnStart, + execute: (row) => + sql` + INSERT INTO projection_turns ( + user_id, + thread_id, + turn_id, + pending_message_id, + source_proposed_plan_thread_id, + source_proposed_plan_id, + assistant_message_id, + state, + requested_at, + started_at, + completed_at, + checkpoint_turn_count, + checkpoint_ref, + checkpoint_status, + checkpoint_files_json + ) + VALUES ( + ${userId}, + ${row.threadId}, + NULL, + ${row.messageId}, + ${row.sourceProposedPlanThreadId}, + ${row.sourceProposedPlanId}, + NULL, + 'pending', + ${row.requestedAt}, + NULL, + NULL, + NULL, + NULL, + NULL, + '[]' + ) + `, + }); + + const getPendingProjectionTurn = SqlSchema.findOneOption({ + Request: GetProjectionPendingTurnStartInput, + Result: ProjectionPendingTurnStart, + execute: ({ threadId }) => + sql` + SELECT + thread_id AS "threadId", + pending_message_id AS "messageId", + source_proposed_plan_thread_id AS "sourceProposedPlanThreadId", + source_proposed_plan_id AS "sourceProposedPlanId", + requested_at AS "requestedAt" + FROM projection_turns + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND turn_id IS NULL + AND state = 'pending' + AND pending_message_id IS NOT NULL + AND checkpoint_turn_count IS NULL + ORDER BY requested_at DESC + LIMIT 1 + `, + }); + + const listPendingProjectionTurns = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionPendingTurnStart, + execute: () => + sql` + SELECT + thread_id AS "threadId", + pending_message_id AS "messageId", + source_proposed_plan_thread_id AS "sourceProposedPlanThreadId", + source_proposed_plan_id AS "sourceProposedPlanId", + requested_at AS "requestedAt" + FROM projection_turns + WHERE user_id = ${userId} + AND turn_id IS NULL + AND state = 'pending' + AND pending_message_id IS NOT NULL + AND checkpoint_turn_count IS NULL + ORDER BY requested_at ASC, thread_id ASC + `, + }); + + const listProjectionTurnsByThread = SqlSchema.findAll({ + Request: ListProjectionTurnsByThreadInput, + Result: ProjectionTurnDbRowSchema, + execute: ({ threadId }) => + sql` + SELECT + thread_id AS "threadId", + turn_id AS "turnId", + pending_message_id AS "pendingMessageId", + source_proposed_plan_thread_id AS "sourceProposedPlanThreadId", + source_proposed_plan_id AS "sourceProposedPlanId", + assistant_message_id AS "assistantMessageId", + state, + requested_at AS "requestedAt", + started_at AS "startedAt", + completed_at AS "completedAt", + checkpoint_turn_count AS "checkpointTurnCount", + checkpoint_ref AS "checkpointRef", + checkpoint_status AS "checkpointStatus", + checkpoint_files_json AS "checkpointFiles" + FROM projection_turns + WHERE user_id = ${userId} + AND thread_id = ${threadId} + ORDER BY + CASE + WHEN checkpoint_turn_count IS NULL THEN 1 + ELSE 0 + END ASC, + checkpoint_turn_count ASC, + requested_at ASC, + -- SQLite sorts NULL first; Postgres needs it spelled out. + turn_id ASC NULLS FIRST + `, + }); + + const getProjectionTurnByTurnId = SqlSchema.findOneOption({ + Request: GetProjectionTurnByTurnIdInput, + Result: ProjectionTurnByIdDbRowSchema, + execute: ({ threadId, turnId }) => + sql` + SELECT + thread_id AS "threadId", + turn_id AS "turnId", + pending_message_id AS "pendingMessageId", + source_proposed_plan_thread_id AS "sourceProposedPlanThreadId", + source_proposed_plan_id AS "sourceProposedPlanId", + assistant_message_id AS "assistantMessageId", + state, + requested_at AS "requestedAt", + started_at AS "startedAt", + completed_at AS "completedAt", + checkpoint_turn_count AS "checkpointTurnCount", + checkpoint_ref AS "checkpointRef", + checkpoint_status AS "checkpointStatus", + checkpoint_files_json AS "checkpointFiles" + FROM projection_turns + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND turn_id = ${turnId} + LIMIT 1 + `, + }); + + const clearCheckpointTurnConflictRow = SqlSchema.void({ + Request: ClearCheckpointTurnConflictInput, + execute: ({ threadId, turnId, checkpointTurnCount }) => + sql` + UPDATE projection_turns + SET + checkpoint_turn_count = NULL, + checkpoint_ref = NULL, + checkpoint_status = NULL, + checkpoint_files_json = '[]' + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND checkpoint_turn_count = ${checkpointTurnCount} + AND (turn_id IS NULL OR turn_id <> ${turnId}) + `, + }); + + const deleteProjectionTurnsByThread = SqlSchema.void({ + Request: DeleteProjectionTurnsByThreadInput, + execute: ({ threadId }) => + sql` + DELETE FROM projection_turns + WHERE user_id = ${userId} + AND thread_id = ${threadId} + `, + }); + + const upsertByTurnId: ProjectionTurnRepositoryShape["upsertByTurnId"] = (row) => + upsertProjectionTurnById(row).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionTurnRepository.upsertByTurnId:query", + "ProjectionTurnRepository.upsertByTurnId:encodeRequest", + ), + ), + ); + + const replacePendingTurnStart: ProjectionTurnRepositoryShape["replacePendingTurnStart"] = (row) => + sql + .withTransaction( + clearPendingProjectionTurnsByThread({ threadId: row.threadId }).pipe( + Effect.flatMap(() => insertPendingProjectionTurn(row)), + ), + ) + .pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionTurnRepository.replacePendingTurnStart:query", + "ProjectionTurnRepository.replacePendingTurnStart:encodeRequest", + ), + ), + ); + + const getPendingTurnStartByThreadId: ProjectionTurnRepositoryShape["getPendingTurnStartByThreadId"] = + (input) => + getPendingProjectionTurn(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionTurnRepository.getPendingTurnStartByThreadId:query"), + ), + ); + + const listPendingTurnStarts: ProjectionTurnRepositoryShape["listPendingTurnStarts"] = () => + listPendingProjectionTurns(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionTurnRepository.listPendingTurnStarts:query", + "ProjectionTurnRepository.listPendingTurnStarts:decodeRows", + ), + ), + ); + + const deletePendingTurnStartByThreadId: ProjectionTurnRepositoryShape["deletePendingTurnStartByThreadId"] = + (input) => + clearPendingProjectionTurnsByThread(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionTurnRepository.deletePendingTurnStartByThreadId:query"), + ), + ); + + const listByThreadId: ProjectionTurnRepositoryShape["listByThreadId"] = (input) => + listProjectionTurnsByThread(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionTurnRepository.listByThreadId:query", + "ProjectionTurnRepository.listByThreadId:decodeRows", + ), + ), + Effect.map((rows) => rows as ReadonlyArray>), + ); + + const getByTurnId: ProjectionTurnRepositoryShape["getByTurnId"] = (input) => + getProjectionTurnByTurnId(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProjectionTurnRepository.getByTurnId:query", + "ProjectionTurnRepository.getByTurnId:decodeRow", + ), + ), + Effect.flatMap((rowOption) => + Option.match(rowOption, { + onNone: () => Effect.succeed(Option.none()), + onSome: (row) => + Effect.succeed(Option.some(row as Schema.Schema.Type)), + }), + ), + ); + + const clearCheckpointTurnConflict: ProjectionTurnRepositoryShape["clearCheckpointTurnConflict"] = + (input) => + clearCheckpointTurnConflictRow(input).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionTurnRepository.clearCheckpointTurnConflict:query"), + ), + ); + + const deleteByThreadId: ProjectionTurnRepositoryShape["deleteByThreadId"] = (input) => + deleteProjectionTurnsByThread(input).pipe( + Effect.mapError(toPersistenceSqlError("ProjectionTurnRepository.deleteByThreadId:query")), + ); + + return { + upsertByTurnId, + replacePendingTurnStart, + getPendingTurnStartByThreadId, + listPendingTurnStarts, + deletePendingTurnStartByThreadId, + listByThreadId, + getByTurnId, + clearCheckpointTurnConflict, + deleteByThreadId, + } satisfies ProjectionTurnRepositoryShape; +}); + +export const PgProjectionTurnRepositoryLive = Layer.effect( + ProjectionTurnRepository, + makeProjectionTurnRepository, +); diff --git a/apps/server/src/persistence/Postgres/Schema.ts b/apps/server/src/persistence/Postgres/Schema.ts new file mode 100644 index 000000000000..d133a93404f9 --- /dev/null +++ b/apps/server/src/persistence/Postgres/Schema.ts @@ -0,0 +1,382 @@ +/** + * Hub baseline schema for PostgreSQL. + * + * Derived from the SQLite schema after migration 043 (`sqlite3 .schema`), + * not from a port of the 43 migrations. Tenancy is a `user_id` column that leads + * every primary key, unique constraint, and index, so one database serves every + * hub user and every query stays safe behind a transaction-mode pooler. + * + * Dialect decisions that the queries depend on: + * - Every text column that is compared or ordered uses `COLLATE "C"` so ordering + * matches SQLite's BINARY collation (the thread-detail keyset uses "~" and "" + * sentinels that only work bytewise), whatever the database default is. + * - Nullable sort keys are indexed `NULLS FIRST`, SQLite's NULL ordering, so the + * `DESC NULLS LAST` window reads are served by a backward index scan. + * - JSON stays in `text` columns: the repositories decode with `fromJsonString` + * and the `\u0000` escapes that JSON.stringify can emit are rejected by jsonb. + * - Boolean-like columns stay integers (`is_streaming`, counts) so row decoding is + * identical to SQLite. + * - SQLite's global `orchestration_events.sequence AUTOINCREMENT` becomes a + * per-user counter in `hub_users.last_event_sequence`, allocated inside the + * append transaction, so sequences are gap-free per user like SQLite's. + */ + +export const HUB_SCHEMA_VERSION = 1; + +const c = `COLLATE "C"`; + +export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ + `CREATE TABLE IF NOT EXISTS hub_schema_version ( + version integer PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + )`, + `CREATE TABLE IF NOT EXISTS hub_users ( + user_id text ${c} PRIMARY KEY, + last_event_sequence bigint NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT now() + )`, + + // orchestration_events (001) — sequence is per user. + `CREATE TABLE IF NOT EXISTS orchestration_events ( + user_id text ${c} NOT NULL, + sequence bigint NOT NULL, + event_id text ${c} NOT NULL, + aggregate_kind text ${c} NOT NULL, + stream_id text ${c} NOT NULL, + stream_version integer NOT NULL, + event_type text ${c} NOT NULL, + occurred_at text ${c} NOT NULL, + command_id text ${c}, + causation_event_id text ${c}, + correlation_id text ${c}, + actor_kind text ${c} NOT NULL, + payload_json text NOT NULL, + metadata_json text NOT NULL, + PRIMARY KEY (user_id, sequence), + UNIQUE (user_id, event_id) + )`, + `CREATE UNIQUE INDEX IF NOT EXISTS idx_orch_events_stream_version + ON orchestration_events (user_id, aggregate_kind, stream_id, stream_version)`, + `CREATE INDEX IF NOT EXISTS idx_orch_events_stream_sequence + ON orchestration_events (user_id, aggregate_kind, stream_id, sequence)`, + `CREATE INDEX IF NOT EXISTS idx_orch_events_command_id + ON orchestration_events (user_id, command_id)`, + `CREATE INDEX IF NOT EXISTS idx_orch_events_correlation_id + ON orchestration_events (user_id, correlation_id)`, + + // orchestration_command_receipts (002) + `CREATE TABLE IF NOT EXISTS orchestration_command_receipts ( + user_id text ${c} NOT NULL, + command_id text ${c} NOT NULL, + aggregate_kind text ${c} NOT NULL, + aggregate_id text ${c} NOT NULL, + accepted_at text ${c} NOT NULL, + result_sequence bigint NOT NULL, + status text ${c} NOT NULL, + error text, + PRIMARY KEY (user_id, command_id) + )`, + `CREATE INDEX IF NOT EXISTS idx_orch_command_receipts_aggregate + ON orchestration_command_receipts (user_id, aggregate_kind, aggregate_id)`, + `CREATE INDEX IF NOT EXISTS idx_orch_command_receipts_sequence + ON orchestration_command_receipts (user_id, result_sequence)`, + + // checkpoint_diff_blobs (003) + `CREATE TABLE IF NOT EXISTS checkpoint_diff_blobs ( + user_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + from_turn_count integer NOT NULL, + to_turn_count integer NOT NULL, + diff text NOT NULL, + created_at text ${c} NOT NULL, + UNIQUE (user_id, thread_id, from_turn_count, to_turn_count) + )`, + `CREATE INDEX IF NOT EXISTS idx_checkpoint_diff_blobs_thread_to_turn + ON checkpoint_diff_blobs (user_id, thread_id, to_turn_count)`, + + // provider_session_runtime (004, 009, 027) + `CREATE TABLE IF NOT EXISTS provider_session_runtime ( + user_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + provider_name text ${c} NOT NULL, + adapter_key text ${c} NOT NULL, + runtime_mode text ${c} NOT NULL DEFAULT 'full-access', + status text ${c} NOT NULL, + last_seen_at text ${c} NOT NULL, + resume_cursor_json text, + runtime_payload_json text, + provider_instance_id text ${c}, + PRIMARY KEY (user_id, thread_id) + )`, + `CREATE INDEX IF NOT EXISTS idx_provider_session_runtime_status + ON provider_session_runtime (user_id, status)`, + `CREATE INDEX IF NOT EXISTS idx_provider_session_runtime_provider + ON provider_session_runtime (user_id, provider_name)`, + `CREATE INDEX IF NOT EXISTS idx_provider_session_runtime_instance + ON provider_session_runtime (user_id, provider_instance_id)`, + + // projection_projects (005, 039, 040) + `CREATE TABLE IF NOT EXISTS projection_projects ( + user_id text ${c} NOT NULL, + project_id text ${c} NOT NULL, + title text NOT NULL, + workspace_root text ${c} NOT NULL, + scripts_json text NOT NULL, + created_at text ${c} NOT NULL, + updated_at text ${c} NOT NULL, + deleted_at text ${c}, + default_model_selection_json text, + default_thread_env_mode text ${c}, + favicon_path text, + PRIMARY KEY (user_id, project_id) + )`, + `CREATE INDEX IF NOT EXISTS idx_projection_projects_updated_at + ON projection_projects (user_id, updated_at)`, + `CREATE INDEX IF NOT EXISTS idx_projection_projects_workspace_root_deleted_at + ON projection_projects (user_id, workspace_root, deleted_at)`, + + // projection_threads (005, 010, 012, 016, 017, 023, 033-036, 038, 042, 043) + `CREATE TABLE IF NOT EXISTS projection_threads ( + user_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + project_id text ${c} NOT NULL, + title text NOT NULL, + branch text, + worktree_path text, + latest_turn_id text ${c}, + created_at text ${c} NOT NULL, + updated_at text ${c} NOT NULL, + deleted_at text ${c}, + runtime_mode text ${c} NOT NULL DEFAULT 'full-access', + interaction_mode text ${c} NOT NULL DEFAULT 'default', + model_selection_json text, + archived_at text ${c}, + latest_user_message_at text ${c}, + pending_approval_count integer NOT NULL DEFAULT 0, + pending_user_input_count integer NOT NULL DEFAULT 0, + has_actionable_proposed_plan integer NOT NULL DEFAULT 0, + settled_override text ${c}, + settled_at text ${c}, + snoozed_until text ${c}, + snoozed_at text ${c}, + title_regeneration_request_id text ${c}, + title_regeneration_started_at text ${c}, + pinned_at text ${c}, + pin_order_key text ${c}, + linked_pull_request_json text, + unsettled_at text ${c}, + PRIMARY KEY (user_id, thread_id) + )`, + `CREATE INDEX IF NOT EXISTS idx_projection_threads_project_id + ON projection_threads (user_id, project_id)`, + `CREATE INDEX IF NOT EXISTS idx_projection_threads_project_archived_at + ON projection_threads (user_id, project_id, archived_at)`, + `CREATE INDEX IF NOT EXISTS idx_projection_threads_project_deleted_created + ON projection_threads (user_id, project_id, deleted_at, created_at)`, + `CREATE INDEX IF NOT EXISTS idx_projection_threads_shell_active + ON projection_threads (user_id, deleted_at, archived_at, project_id, created_at, thread_id)`, + `CREATE INDEX IF NOT EXISTS idx_projection_threads_shell_archived + ON projection_threads (user_id, deleted_at, archived_at, project_id, thread_id)`, + + // projection_thread_messages (005, 007, 029). SQLite's (thread_id, created_at) + // and (thread_id, sequence) indexes are prefixes of the 029 indexes and are dropped. + `CREATE TABLE IF NOT EXISTS projection_thread_messages ( + user_id text ${c} NOT NULL, + message_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + turn_id text ${c}, + role text ${c} NOT NULL, + text text NOT NULL, + is_streaming integer NOT NULL, + created_at text ${c} NOT NULL, + updated_at text ${c} NOT NULL, + attachments_json text, + PRIMARY KEY (user_id, message_id) + )`, + `CREATE INDEX IF NOT EXISTS idx_projection_thread_messages_thread_created_id + ON projection_thread_messages (user_id, thread_id, created_at, message_id)`, + + // projection_thread_activities (005, 008, 029) + `CREATE TABLE IF NOT EXISTS projection_thread_activities ( + user_id text ${c} NOT NULL, + activity_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + turn_id text ${c}, + tone text ${c} NOT NULL, + kind text ${c} NOT NULL, + summary text NOT NULL, + payload_json text NOT NULL, + created_at text ${c} NOT NULL, + sequence bigint, + PRIMARY KEY (user_id, activity_id) + )`, + `CREATE INDEX IF NOT EXISTS idx_projection_thread_activities_thread_created + ON projection_thread_activities (user_id, thread_id, created_at)`, + `CREATE INDEX IF NOT EXISTS idx_projection_thread_activities_thread_sequence_created_id + ON projection_thread_activities (user_id, thread_id, sequence NULLS FIRST, created_at, activity_id)`, + + // projection_thread_sessions (005, 006, 028) + `CREATE TABLE IF NOT EXISTS projection_thread_sessions ( + user_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + status text ${c} NOT NULL, + provider_name text ${c}, + provider_session_id text ${c}, + provider_thread_id text ${c}, + active_turn_id text ${c}, + last_error text, + updated_at text ${c} NOT NULL, + runtime_mode text ${c} NOT NULL DEFAULT 'full-access', + provider_instance_id text ${c}, + PRIMARY KEY (user_id, thread_id) + )`, + `CREATE INDEX IF NOT EXISTS idx_projection_thread_sessions_provider_session + ON projection_thread_sessions (user_id, provider_session_id)`, + `CREATE INDEX IF NOT EXISTS idx_projection_thread_sessions_instance + ON projection_thread_sessions (user_id, provider_instance_id)`, + + // projection_turns (005, 015, 037) — row_id keeps SQLite's surrogate key. + `CREATE TABLE IF NOT EXISTS projection_turns ( + row_id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + user_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + turn_id text ${c}, + pending_message_id text ${c}, + assistant_message_id text ${c}, + state text ${c} NOT NULL, + requested_at text ${c} NOT NULL, + started_at text ${c}, + completed_at text ${c}, + checkpoint_turn_count integer, + checkpoint_ref text ${c}, + checkpoint_status text ${c}, + checkpoint_files_json text NOT NULL, + source_proposed_plan_thread_id text ${c}, + source_proposed_plan_id text ${c}, + UNIQUE (user_id, thread_id, turn_id), + UNIQUE (user_id, thread_id, checkpoint_turn_count) + )`, + `CREATE INDEX IF NOT EXISTS idx_projection_turns_thread_requested + ON projection_turns (user_id, thread_id, requested_at)`, + `CREATE INDEX IF NOT EXISTS idx_projection_turns_thread_checkpoint_completed + ON projection_turns (user_id, thread_id, checkpoint_turn_count, completed_at)`, + `CREATE INDEX IF NOT EXISTS idx_projection_turns_thread_keyset + ON projection_turns (user_id, thread_id, requested_at, turn_id NULLS FIRST)`, + + // projection_pending_approvals (005, 025) + `CREATE TABLE IF NOT EXISTS projection_pending_approvals ( + user_id text ${c} NOT NULL, + request_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + turn_id text ${c}, + status text ${c} NOT NULL, + decision text ${c}, + created_at text ${c} NOT NULL, + resolved_at text ${c}, + PRIMARY KEY (user_id, request_id) + )`, + `CREATE INDEX IF NOT EXISTS idx_projection_pending_approvals_thread_status + ON projection_pending_approvals (user_id, thread_id, status)`, + + // projection_state (005) + `CREATE TABLE IF NOT EXISTS projection_state ( + user_id text ${c} NOT NULL, + projector text ${c} NOT NULL, + last_applied_sequence bigint NOT NULL, + updated_at text ${c} NOT NULL, + PRIMARY KEY (user_id, projector) + )`, + + // projection_thread_proposed_plans (013, 014) + `CREATE TABLE IF NOT EXISTS projection_thread_proposed_plans ( + user_id text ${c} NOT NULL, + plan_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + turn_id text ${c}, + plan_markdown text NOT NULL, + created_at text ${c} NOT NULL, + updated_at text ${c} NOT NULL, + implemented_at text ${c}, + implementation_thread_id text ${c}, + PRIMARY KEY (user_id, plan_id) + )`, + `CREATE INDEX IF NOT EXISTS idx_projection_thread_proposed_plans_thread_created + ON projection_thread_proposed_plans (user_id, thread_id, created_at)`, + + // auth_pairing_links / auth_sessions (020-022, 031, 032, 041). The hub replaces + // pairing and bearer sessions with a signed gateway identity; kept for parity. + `CREATE TABLE IF NOT EXISTS auth_pairing_links ( + user_id text ${c} NOT NULL, + id text ${c} NOT NULL, + credential text ${c} NOT NULL, + method text ${c} NOT NULL, + scopes text NOT NULL, + subject text ${c} NOT NULL, + label text, + created_at text ${c} NOT NULL, + expires_at text ${c} NOT NULL, + consumed_at text ${c}, + revoked_at text ${c}, + proof_key_thumbprint text ${c}, + PRIMARY KEY (user_id, id), + UNIQUE (credential) + )`, + `CREATE INDEX IF NOT EXISTS idx_auth_pairing_links_active + ON auth_pairing_links (user_id, revoked_at, consumed_at, expires_at)`, + `CREATE TABLE IF NOT EXISTS auth_sessions ( + user_id text ${c} NOT NULL, + session_id text ${c} NOT NULL, + subject text ${c} NOT NULL, + scopes text NOT NULL, + method text ${c} NOT NULL, + client_label text, + client_ip_address text, + client_user_agent text, + client_device_type text ${c} NOT NULL DEFAULT 'unknown', + client_os text, + client_browser text, + issued_at text ${c} NOT NULL, + expires_at text ${c} NOT NULL, + last_connected_at text ${c}, + revoked_at text ${c}, + client_surface text ${c}, + client_app_version text, + PRIMARY KEY (user_id, session_id) + )`, + `CREATE INDEX IF NOT EXISTS idx_auth_sessions_active + ON auth_sessions (user_id, revoked_at, expires_at, issued_at)`, +]; + +/** Tables whose rows belong to one hub user; used by tests and the RLS backstop. */ +export const HUB_TENANT_TABLES = [ + "orchestration_events", + "orchestration_command_receipts", + "checkpoint_diff_blobs", + "provider_session_runtime", + "projection_projects", + "projection_threads", + "projection_thread_messages", + "projection_thread_activities", + "projection_thread_sessions", + "projection_turns", + "projection_pending_approvals", + "projection_state", + "projection_thread_proposed_plans", + "auth_pairing_links", + "auth_sessions", +] as const; + +/** + * Optional row-level-security backstop. Policies compare `user_id` with the + * transaction-local `hub.user_id` setting, so a statement that runs without a + * tenant context sees no rows and cannot insert (fails closed). Table owners + * bypass RLS unless FORCE is set, so the backstop applies to a non-owner role. + */ +export const hubRowLevelSecurityStatements = (tables: ReadonlyArray = HUB_TENANT_TABLES) => + tables.flatMap((table) => [ + `ALTER TABLE ${table} ENABLE ROW LEVEL SECURITY`, + `DROP POLICY IF EXISTS hub_tenant_isolation ON ${table}`, + `CREATE POLICY hub_tenant_isolation ON ${table} + USING (user_id = current_setting('hub.user_id', true)) + WITH CHECK (user_id = current_setting('hub.user_id', true))`, + ]); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bfb9f008050b..5c487b6d45dd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -482,6 +482,9 @@ importers: '@effect/platform-node-shared': specifier: 4.0.0-beta.103 version: 4.0.0-beta.103(bufferutil@4.1.0)(effect@4.0.0-beta.103(patch_hash=af36b7948b6f9c56623074662b51dade5699880c1a7c71245de73e13c3185fb6))(utf-8-validate@6.0.6) + '@effect/sql-pg': + specifier: 4.0.0-beta.103 + version: 4.0.0-beta.103(effect@4.0.0-beta.103(patch_hash=af36b7948b6f9c56623074662b51dade5699880c1a7c71245de73e13c3185fb6)) '@effect/sql-sqlite-bun': specifier: 4.0.0-beta.103 version: 4.0.0-beta.103(effect@4.0.0-beta.103(patch_hash=af36b7948b6f9c56623074662b51dade5699880c1a7c71245de73e13c3185fb6)) From 031aee45acbb73dc2f924b6a09f5ee69f2a08d6b Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 19:31:18 +0000 Subject: [PATCH 03/58] chore(server): add hub prototype import, benchmark, and parity scripts Scripts that dump the final SQLite schema, stream a T3 SQLite database into the hub schema as one user, benchmark activation, memory, snapshot and dispatch cost, compare SQLite and Postgres read paths, and add artificial network latency. They print only counts and timings. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scripts/hub-proto/bench-activation.ts | 364 ++++++++++++++++++ .../scripts/hub-proto/compare-backends.ts | 159 ++++++++ .../server/scripts/hub-proto/import-sqlite.sh | 41 ++ .../server/scripts/hub-proto/latency-proxy.ts | 31 ++ .../scripts/hub-proto/migrate-temp-sqlite.ts | 18 + 5 files changed, 613 insertions(+) create mode 100644 apps/server/scripts/hub-proto/bench-activation.ts create mode 100644 apps/server/scripts/hub-proto/compare-backends.ts create mode 100755 apps/server/scripts/hub-proto/import-sqlite.sh create mode 100644 apps/server/scripts/hub-proto/latency-proxy.ts create mode 100644 apps/server/scripts/hub-proto/migrate-temp-sqlite.ts diff --git a/apps/server/scripts/hub-proto/bench-activation.ts b/apps/server/scripts/hub-proto/bench-activation.ts new file mode 100644 index 000000000000..c434d07ff8db --- /dev/null +++ b/apps/server/scripts/hub-proto/bench-activation.ts @@ -0,0 +1,364 @@ +// Hub prototype benchmark. Prints only counts, sizes, and timings. +// +// node --expose-gc scripts/hub-proto/bench-activation.ts pg +// node --expose-gc scripts/hub-proto/bench-activation.ts sqlite - +// +// pg mode clones the owner's metadata for synthetic users to measure memory per +// active engine, and counts round trips by wrapping node-postgres' Client.query. +import * as NodeModule from "node:module"; +import * as NodePerfHooks from "node:perf_hooks"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { + CommandId, + EventId, + MessageId, + ThreadId, + type OrchestrationReadModel, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as ManagedRuntime from "effect/ManagedRuntime"; +import * as Option from "effect/Option"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { ServerConfig } from "../../src/config.ts"; +import { OrchestrationEngineLive } from "../../src/orchestration/Layers/OrchestrationEngine.ts"; +import { OrchestrationProjectionPipelineLive } from "../../src/orchestration/Layers/ProjectionPipeline.ts"; +import { OrchestrationProjectionSnapshotQueryLive } from "../../src/orchestration/Layers/ProjectionSnapshotQuery.ts"; +import { OrchestrationEngineService } from "../../src/orchestration/Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../../src/orchestration/Services/ProjectionSnapshotQuery.ts"; +import * as ThreadBackgroundLiveness from "../../src/orchestration/ThreadBackgroundLiveness.ts"; +import * as ThreadPlanProgress from "../../src/orchestration/ThreadPlanProgress.ts"; +import { OrchestrationCommandReceiptRepositoryLive } from "../../src/persistence/Layers/OrchestrationCommandReceipts.ts"; +import { OrchestrationEventStoreLive } from "../../src/persistence/Layers/OrchestrationEventStore.ts"; +import { makeSqlitePersistenceLive } from "../../src/persistence/Layers/Sqlite.ts"; +import { + makeHubSharedLayer, + makeHubUserEngineLayer, + type HubSharedServices, +} from "../../src/persistence/Postgres/HubEngine.ts"; +import { RepositoryIdentityResolver } from "../../src/project/RepositoryIdentityResolver.ts"; + +const [mode, target, ownerArg, threadArg] = process.argv.slice(2); +if ((mode !== "pg" && mode !== "sqlite") || !target || !threadArg) { + throw new Error("usage: bench-activation.ts pg|sqlite "); +} +const owner = ownerArg ?? "owner"; +const threadIds = threadArg.split(",").map((id) => ThreadId.make(id)); +const ACTIVATIONS = Number(process.env.BENCH_ACTIVATIONS ?? 30); +const ENGINES = Number(process.env.BENCH_ENGINES ?? 20); +const DETAIL_RUNS = Number(process.env.BENCH_DETAIL_RUNS ?? 10); +const DISPATCHES = Number(process.env.BENCH_DISPATCHES ?? 50); + +const out = (line: string) => process.stdout.write(`${line}\n`); +const gc = () => (globalThis as { gc?: () => void }).gc?.(); +const mib = (bytes: number) => Math.round((bytes / 1048576) * 10) / 10; +const percentile = (values: ReadonlyArray, p: number) => { + const sorted = values.toSorted((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1)] ?? NaN; +}; +const summary = (values: ReadonlyArray) => + `p50=${percentile(values, 50).toFixed(1)}ms p90=${percentile(values, 90).toFixed(1)}ms min=${Math.min(...values).toFixed(1)}ms max=${Math.max(...values).toFixed(1)}ms n=${values.length}`; +const time = async (run: () => Promise) => { + const start = NodePerfHooks.performance.now(); + const value = await run(); + return { ms: NodePerfHooks.performance.now() - start, value }; +}; + +// Round-trip counter over node-postgres (resolved through @effect/sql-pg). +let pgQueries = 0; +if (mode === "pg") { + const requirePg = NodeModule.createRequire(import.meta.resolve("@effect/sql-pg/PgClient")); + const Pg = requirePg("pg") as { + Client: { prototype: { query: (...args: unknown[]) => unknown } }; + }; + const query = Pg.Client.prototype.query; + Pg.Client.prototype.query = function (this: unknown, ...args: unknown[]) { + pgQueries += 1; + return query.apply(this, args); + }; +} + +const NoRepositoryIdentityLive = Layer.succeed(RepositoryIdentityResolver, { + resolve: () => Effect.succeed(null), +}); + +type EngineRuntime = ManagedRuntime.ManagedRuntime< + OrchestrationEngineService | ProjectionSnapshotQuery, + unknown +>; + +const sqliteEngineLayer = (file: string) => + Layer.mergeAll( + OrchestrationEngineLive.pipe( + Layer.provide(OrchestrationProjectionSnapshotQueryLive), + Layer.provide(OrchestrationProjectionPipelineLive), + ), + OrchestrationProjectionSnapshotQueryLive, + ).pipe( + Layer.provideMerge(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide(OrchestrationEventStoreLive), + Layer.provideMerge(OrchestrationCommandReceiptRepositoryLive), + Layer.provide(NoRepositoryIdentityLive), + Layer.provide(makeSqlitePersistenceLive(file)), + Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-bench-" })), + Layer.provideMerge(NodeServices.layer), + ); + +const shared = + mode === "pg" + ? ManagedRuntime.make(makeHubSharedLayer({ url: target, maxConnections: 10 })) + : null; +const sharedContext = shared ? await shared.runPromise(Effect.context()) : null; +const sql = sharedContext ? Context.get(sharedContext, SqlClient.SqlClient) : null; + +const makeEngine = (userId: string): EngineRuntime => + ManagedRuntime.make( + sharedContext + ? makeHubUserEngineLayer({ userId, shared: sharedContext }) + : sqliteEngineLayer(target), + ) as EngineRuntime; + +const activate = async (userId: string) => { + const runtime = makeEngine(userId); + const queriesBefore = pgQueries; + const { ms } = await time(() => runtime.runPromise(Effect.service(OrchestrationEngineService))); + return { runtime, ms, queries: pgQueries - queriesBefore }; +}; + +const results: Record = { mode }; +// BENCH_SECTIONS=memory measures engine memory in a fresh process (one warm-up +// activation first), so earlier snapshot reads do not pre-grow the heap. +const onlyMemory = process.env.BENCH_SECTIONS === "memory"; +if (onlyMemory) { + const warmup = await activate(owner); + await warmup.runtime.dispose(); +} else { + // Activation (bootstrap + command read model), cold first then repeated. + const cold = await activate(owner); + const readModel = (await cold.runtime.runPromise( + Effect.flatMap(Effect.service(ProjectionSnapshotQuery), (query) => query.getCommandReadModel()), + )) as OrchestrationReadModel; + out( + `read model: projects=${readModel.projects.length} threads=${readModel.threads.length} ` + + `latestTurns=${readModel.threads.filter((thread) => thread.latestTurn !== null).length} ` + + `sessions=${readModel.threads.filter((thread) => thread.session !== null).length} ` + + `plans=${readModel.threads.reduce((n, thread) => n + thread.proposedPlans.length, 0)} ` + + `snapshotSequence=${readModel.snapshotSequence} jsonBytes=${JSON.stringify(readModel).length}`, + ); + out(`cold activation: ${cold.ms.toFixed(1)}ms queries=${cold.queries}`); + await cold.runtime.dispose(); + const activationMs: number[] = []; + let activationQueries = 0; + for (let index = 0; index < ACTIVATIONS; index += 1) { + const run = await activate(owner); + activationMs.push(run.ms); + activationQueries = run.queries; + await run.runtime.dispose(); + } + out(`warm activation: ${summary(activationMs)} queries/activation=${activationQueries}`); + results.activation = { coldMs: cold.ms, warm: activationMs, queries: activationQueries }; + + // getCommandReadModel alone, on a live engine. + const live = await activate(owner); + const query = await live.runtime.runPromise(Effect.service(ProjectionSnapshotQuery)); + const readModelMs: number[] = []; + for (let index = 0; index < ACTIVATIONS; index += 1) { + readModelMs.push((await time(() => live.runtime.runPromise(query.getCommandReadModel()))).ms); + } + out(`getCommandReadModel: ${summary(readModelMs)}`); + const shellMs: number[] = []; + let shellThreads = 0; + for (let index = 0; index < ACTIVATIONS; index += 1) { + const run = await time(() => live.runtime.runPromise(query.getShellSnapshot())); + shellMs.push(run.ms); + shellThreads = run.value.threads.length; + } + out(`getShellSnapshot: ${summary(shellMs)} threads=${shellThreads}`); + results.readModelMs = readModelMs; + results.shellMs = shellMs; + + // Thread detail loads: the client's first page (turnLimit 10) and the full thread. + const details: Record = {}; + for (const [index, threadId] of threadIds.entries()) { + for (const window of [{ turnLimit: 10 }, undefined] as const) { + const samples: number[] = []; + let counts = ""; + for (let run = 0; run < DETAIL_RUNS; run += 1) { + const queriesBefore = pgQueries; + const { ms, value } = await time(() => + live.runtime.runPromise(query.getThreadDetailSnapshot(threadId, window)), + ); + samples.push(ms); + const thread = Option.getOrThrow(value).thread; + counts = + `messages=${thread.messages.length} activities=${thread.activities.length} ` + + `checkpoints=${thread.checkpoints.length} queries=${pgQueries - queriesBefore}`; + } + const label = `thread#${index + 1} ${window ? "page(turnLimit=10)" : "full"}`; + out(`detail ${label}: ${summary(samples)} ${counts}`); + details[label] = { samples, counts }; + } + } + results.details = details; + + // Command dispatch on the owner's largest thread: activity appends and streaming deltas. + const engine = await live.runtime.runPromise(Effect.service(OrchestrationEngineService)); + const dispatchThread = threadIds[0]!; + const appendMs: number[] = []; + const appendQueries: number[] = []; + const createdAt = "2026-09-25T12:00:00.000Z"; + for (let index = 0; index < DISPATCHES; index += 1) { + const queriesBefore = pgQueries; + appendMs.push( + ( + await time(() => + live.runtime.runPromise( + engine.dispatch({ + type: "thread.activity.append", + commandId: CommandId.make(`bench-activity-${process.pid}-${index}`), + threadId: dispatchThread, + activity: { + id: EventId.make(`bench-activity-${process.pid}-${index}`), + tone: "tool", + kind: "bench.synthetic", + summary: "synthetic benchmark activity", + payload: { index }, + turnId: null, + createdAt, + }, + createdAt, + }), + ), + ) + ).ms, + ); + appendQueries.push(pgQueries - queriesBefore); + } + out( + `dispatch thread.activity.append: ${summary(appendMs)} queries/command=${percentile(appendQueries, 50)}`, + ); + const deltaMs: number[] = []; + const deltaQueries: number[] = []; + for (let index = 0; index < DISPATCHES; index += 1) { + const queriesBefore = pgQueries; + deltaMs.push( + ( + await time(() => + live.runtime.runPromise( + engine.dispatch({ + type: "thread.message.assistant.delta", + commandId: CommandId.make(`bench-delta-${process.pid}-${index}`), + threadId: dispatchThread, + messageId: MessageId.make(`bench-message-${process.pid}`), + delta: "x", + createdAt, + }), + ), + ) + ).ms, + ); + deltaQueries.push(pgQueries - queriesBefore); + } + out( + `dispatch thread.message.assistant.delta: ${summary(deltaMs)} queries/command=${percentile(deltaQueries, 50)}`, + ); + results.dispatch = { appendMs, appendQueries, deltaMs, deltaQueries }; + await live.runtime.dispose(); +} + +// Memory per active engine: distinct synthetic users with the owner's metadata. +if (sql && shared) { + const users = Array.from({ length: ENGINES }, (_, index) => `bench-user-${index + 1}`); + await shared.runPromise( + Effect.gen(function* () { + for (const table of [ + "projection_projects", + "projection_threads", + "projection_thread_sessions", + "projection_turns", + "projection_thread_proposed_plans", + "projection_state", + ]) { + const columns = (yield* sql<{ name: string }>` + SELECT column_name AS name FROM information_schema.columns + WHERE table_name = ${table} AND column_name NOT IN ('user_id', 'row_id') + ORDER BY ordinal_position + `).map((row) => row.name); + const list = sql.literal(columns.join(", ")); + yield* sql`DELETE FROM ${sql(table)} WHERE user_id LIKE 'bench-user-%'`; + for (const user of users) { + yield* sql` + INSERT INTO ${sql(table)} (user_id, ${list}) + SELECT ${user}, ${list} FROM ${sql(table)} WHERE user_id = ${owner} + `; + } + } + yield* sql`DELETE FROM hub_users WHERE user_id LIKE 'bench-user-%'`; + for (const user of users) { + yield* sql` + INSERT INTO hub_users (user_id, last_event_sequence) + SELECT ${user}, last_event_sequence FROM hub_users WHERE user_id = ${owner} + `; + } + }), + ); + gc(); + gc(); + const before = process.memoryUsage(); + const engines: EngineRuntime[] = []; + for (const user of users) { + engines.push((await activate(user)).runtime); + } + gc(); + gc(); + const after = process.memoryUsage(); + out( + `memory: ${ENGINES} active engines rssDelta=${mib(after.rss - before.rss)}MiB ` + + `heapUsedDelta=${mib(after.heapUsed - before.heapUsed)}MiB ` + + `perEngine rss=${mib((after.rss - before.rss) / ENGINES)}MiB heap=${mib((after.heapUsed - before.heapUsed) / ENGINES)}MiB ` + + `baseRss=${mib(before.rss)}MiB`, + ); + results.memory = { engines: ENGINES, before, after }; + // Parallel activation of all synthetic users at once (cold-start storm). + for (const runtime of engines) await runtime.dispose(); + const storm = await time(() => Promise.all(users.map((user) => activate(user)))); + out(`parallel activation of ${ENGINES} users: ${storm.ms.toFixed(1)}ms total`); + for (const run of storm.value) await run.runtime.dispose(); + await shared.runPromise( + Effect.gen(function* () { + for (const table of [ + "projection_projects", + "projection_threads", + "projection_thread_sessions", + "projection_turns", + "projection_thread_proposed_plans", + "projection_state", + "hub_users", + ]) { + yield* sql`DELETE FROM ${sql(table)} WHERE user_id LIKE 'bench-user-%'`; + } + }), + ); + await shared.dispose(); +} else { + gc(); + gc(); + const before = process.memoryUsage(); + const engines: EngineRuntime[] = []; + for (let index = 0; index < ENGINES; index += 1) engines.push((await activate(owner)).runtime); + gc(); + gc(); + const after = process.memoryUsage(); + out( + `memory: ${ENGINES} active engines rssDelta=${mib(after.rss - before.rss)}MiB ` + + `heapUsedDelta=${mib(after.heapUsed - before.heapUsed)}MiB ` + + `perEngine rss=${mib((after.rss - before.rss) / ENGINES)}MiB heap=${mib((after.heapUsed - before.heapUsed) / ENGINES)}MiB`, + ); + for (const runtime of engines) await runtime.dispose(); +} + +out(`RESULTS_JSON ${JSON.stringify(results)}`); diff --git a/apps/server/scripts/hub-proto/compare-backends.ts b/apps/server/scripts/hub-proto/compare-backends.ts new file mode 100644 index 000000000000..188f3f5103f6 --- /dev/null +++ b/apps/server/scripts/hub-proto/compare-backends.ts @@ -0,0 +1,159 @@ +// Compares the SQLite and Postgres read paths over the same imported history. +// Prints only equal/different plus counts, never row contents. +// +// node scripts/hub-proto/compare-backends.ts [skip-thread-ids,] +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { ThreadId, type OrchestrationThreadDetailWindow } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as ManagedRuntime from "effect/ManagedRuntime"; +import * as Option from "effect/Option"; + +import { ServerConfig } from "../../src/config.ts"; +import { OrchestrationProjectionSnapshotQueryLive } from "../../src/orchestration/Layers/ProjectionSnapshotQuery.ts"; +import { ProjectionSnapshotQuery } from "../../src/orchestration/Services/ProjectionSnapshotQuery.ts"; +import * as ThreadBackgroundLiveness from "../../src/orchestration/ThreadBackgroundLiveness.ts"; +import * as ThreadPlanProgress from "../../src/orchestration/ThreadPlanProgress.ts"; +import { makeSqlitePersistenceLive } from "../../src/persistence/Layers/Sqlite.ts"; +import { + makeHubSharedLayer, + makeHubUserEngineLayer, + type HubSharedServices, +} from "../../src/persistence/Postgres/HubEngine.ts"; +import { RepositoryIdentityResolver } from "../../src/project/RepositoryIdentityResolver.ts"; + +const [sqliteFile, pgUrl, owner, threadArg, skipArg] = process.argv.slice(2); +if (!sqliteFile || !pgUrl || !owner || !threadArg) { + throw new Error("usage: compare-backends.ts [skip,]"); +} +const threadIds = threadArg.split(",").map((id) => ThreadId.make(id)); +// Threads the benchmark wrote to differ by design; leave them out of whole-model checks. +const skipThreads = new Set(skipArg ? skipArg.split(",") : []); +const out = (line: string) => process.stdout.write(`${line}\n`); + +const sqliteQuery = ManagedRuntime.make( + OrchestrationProjectionSnapshotQueryLive.pipe( + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide( + Layer.succeed(RepositoryIdentityResolver, { resolve: () => Effect.succeed(null) }), + ), + Layer.provide(makeSqlitePersistenceLive(sqliteFile)), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-compare-" })), + Layer.provide(NodeServices.layer), + ), +); +const shared = ManagedRuntime.make(makeHubSharedLayer({ url: pgUrl })); +const sharedContext = await shared.runPromise(Effect.context()); +const pgQuery = ManagedRuntime.make( + makeHubUserEngineLayer({ userId: owner, shared: sharedContext }), +); + +const both = async ( + read: (query: ProjectionSnapshotQuery["Service"]) => Effect.Effect, +) => { + const sqlite = await sqliteQuery.runPromise( + Effect.flatMap(Effect.service(ProjectionSnapshotQuery), read), + ); + const pg = await pgQuery.runPromise( + Effect.flatMap(Effect.service(ProjectionSnapshotQuery), read), + ); + return { sqlite, pg }; +}; + +// Drops fields that legitimately differ between the two stores after the +// benchmark appended events (sequences, max updatedAt) and the skipped threads. +const normalize = (value: unknown): unknown => + JSON.parse(JSON.stringify(value), (key, inner) => { + if (key === "snapshotSequence" || key === "threadSequence") return undefined; + if (Array.isArray(inner)) { + return inner.filter( + (item) => + !( + item && + typeof item === "object" && + skipThreads.has((item as { id?: string }).id ?? "") + ), + ); + } + return inner; + }); + +let failures = 0; +const check = (label: string, sqlite: unknown, pg: unknown, counts: string) => { + const left = JSON.stringify(normalize(sqlite)); + const right = JSON.stringify(normalize(pg)); + const equal = left === right; + if (!equal) failures += 1; + out(`${equal ? "equal" : "DIFFERENT"}: ${label} (${counts}; ${left.length} bytes)`); +}; + +const readModel = await both((query) => query.getCommandReadModel()); +check( + "getCommandReadModel", + { ...readModel.sqlite, updatedAt: undefined }, + { ...readModel.pg, updatedAt: undefined }, + `threads=${readModel.pg.threads.length}`, +); +const shell = await both((query) => query.getShellSnapshot()); +check( + "getShellSnapshot", + { ...shell.sqlite, updatedAt: undefined }, + { ...shell.pg, updatedAt: undefined }, + `threads=${shell.pg.threads.length}`, +); +const archived = await both((query) => query.getArchivedShellSnapshot()); +check( + "getArchivedShellSnapshot", + { ...archived.sqlite, updatedAt: undefined }, + { ...archived.pg, updatedAt: undefined }, + `threads=${archived.pg.threads.length}`, +); +const counts = await both((query) => query.getCounts()); +check("getCounts", counts.sqlite, counts.pg, `threads=${counts.pg.threadCount}`); + +for (const [index, threadId] of threadIds.entries()) { + const full = await both((query) => query.getThreadDetailSnapshot(threadId)); + const fullPg = Option.getOrThrow(full.pg).thread; + check( + `thread#${index + 1} full`, + full.sqlite, + full.pg, + `messages=${fullPg.messages.length} activities=${fullPg.activities.length}`, + ); + const byId = await both((query) => query.getThreadDetailById(threadId)); + check(`thread#${index + 1} getThreadDetailById`, byId.sqlite, byId.pg, "raw activities"); + // Walk every page with the client's page sizes and compare each page. + for (const turnLimit of [10, 3]) { + let window: OrchestrationThreadDetailWindow = { turnLimit }; + for (let page = 1; page <= 200; page += 1) { + const current = window; + const result = await both((query) => query.getThreadDetailSnapshot(threadId, current)); + const pgSnapshot = Option.getOrThrow(result.pg); + check( + `thread#${index + 1} turnLimit=${turnLimit} page ${page}`, + result.sqlite, + result.pg, + `messages=${pgSnapshot.thread.messages.length} activities=${pgSnapshot.thread.activities.length} hasMore=${pgSnapshot.page?.hasMore}`, + ); + const cursor = pgSnapshot.page?.beforeCursor; + if (!pgSnapshot.page?.hasMore || !cursor) break; + window = { turnLimit, beforeCursor: cursor }; + } + } +} + +const search = await both((query) => query.searchThreads({ query: "the", limit: 50 })); +check("searchThreads('the')", search.sqlite, search.pg, `matches=${search.pg.matches.length}`); +const searchUpper = await both((query) => query.searchThreads({ query: "THE", limit: 50 })); +check( + "searchThreads('THE')", + searchUpper.sqlite, + searchUpper.pg, + `matches=${searchUpper.pg.matches.length}`, +); + +out(failures === 0 ? "ALL EQUAL" : `${failures} DIFFERENCES`); +await pgQuery.dispose(); +await shared.dispose(); +await sqliteQuery.dispose(); diff --git a/apps/server/scripts/hub-proto/import-sqlite.sh b/apps/server/scripts/hub-proto/import-sqlite.sh new file mode 100755 index 000000000000..4ce0b9e144f4 --- /dev/null +++ b/apps/server/scripts/hub-proto/import-sqlite.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Streams a T3 SQLite state database into the hub Postgres schema as one user. +# Rows go through a pipe (sqlite3 CSV -> psql \copy); nothing is printed. +# +# import-sqlite.sh +set -euo pipefail + +SQLITE_FILE=$1 +USER_ID=$2 +PG_URL=$3 + +TABLES=( + projection_projects + projection_threads + projection_thread_sessions + projection_turns + projection_thread_proposed_plans + projection_pending_approvals + projection_state + projection_thread_messages + projection_thread_activities + orchestration_events +) + +for table in "${TABLES[@]}"; do + # row_id is a Postgres identity column; everything else keeps SQLite's order. + columns=$(sqlite3 "file:${SQLITE_FILE}?mode=ro" \ + "SELECT group_concat(name, ', ') FROM pragma_table_info('${table}') WHERE name <> 'row_id'") + sqlite3 -csv -nullvalue '\N' -newline $'\n' "file:${SQLITE_FILE}?mode=ro" \ + "SELECT '${USER_ID}', ${columns} FROM ${table}" | + psql "$PG_URL" -q -v ON_ERROR_STOP=1 \ + -c "\\copy ${table} (user_id, ${columns}) FROM STDIN WITH (FORMAT csv, NULL '\\N')" +done + +# Continue the user's event sequence after the imported history. +psql "$PG_URL" -q -v ON_ERROR_STOP=1 -c " + INSERT INTO hub_users (user_id, last_event_sequence) + SELECT '${USER_ID}', COALESCE(MAX(last_applied_sequence), 0) + FROM projection_state WHERE user_id = '${USER_ID}' + ON CONFLICT (user_id) DO UPDATE SET last_event_sequence = EXCLUDED.last_event_sequence" +psql "$PG_URL" -q -v ON_ERROR_STOP=1 -c "ANALYZE" diff --git a/apps/server/scripts/hub-proto/latency-proxy.ts b/apps/server/scripts/hub-proto/latency-proxy.ts new file mode 100644 index 000000000000..52f0a850cf93 --- /dev/null +++ b/apps/server/scripts/hub-proto/latency-proxy.ts @@ -0,0 +1,31 @@ +// TCP proxy that delays every chunk in both directions, to approximate the +// network round trip between a hub machine and PlanetScale. +// +// node scripts/hub-proto/latency-proxy.ts +// @effect-diagnostics globalTimers:off +import * as NodeNet from "node:net"; + +const [listenPort, targetPort, delayArg] = process.argv.slice(2).map(Number); +if (!listenPort || !targetPort || delayArg === undefined) { + throw new Error("usage: latency-proxy.ts "); +} +const delayMs = delayArg; + +const pipeDelayed = (from: NodeNet.Socket, to: NodeNet.Socket) => { + from.on("data", (chunk) => { + setTimeout(() => to.write(chunk), delayMs); + }); + from.on("end", () => setTimeout(() => to.end(), delayMs)); + from.on("error", () => to.destroy()); +}; + +NodeNet.createServer((client) => { + client.setNoDelay(true); + const upstream = NodeNet.connect({ host: "127.0.0.1", port: targetPort }, () => { + upstream.setNoDelay(true); + }); + pipeDelayed(client, upstream); + pipeDelayed(upstream, client); +}).listen(listenPort, "127.0.0.1", () => { + process.stdout.write(`latency proxy :${listenPort} -> :${targetPort} (+${delayMs}ms each way)\n`); +}); diff --git a/apps/server/scripts/hub-proto/migrate-temp-sqlite.ts b/apps/server/scripts/hub-proto/migrate-temp-sqlite.ts new file mode 100644 index 000000000000..38f2958ebf50 --- /dev/null +++ b/apps/server/scripts/hub-proto/migrate-temp-sqlite.ts @@ -0,0 +1,18 @@ +// Creates a freshly migrated SQLite database so the final schema can be +// inspected with `sqlite3 .schema`. Usage: node migrate-temp-sqlite.ts +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import { makeSqlitePersistenceLive } from "../../src/persistence/Layers/Sqlite.ts"; + +const target = process.argv[2]; +if (!target) throw new Error("usage: migrate-temp-sqlite.ts "); + +await Effect.runPromise( + Layer.build(makeSqlitePersistenceLive(target)).pipe( + Effect.scoped, + Effect.provide(NodeServices.layer), + ), +); +process.stdout.write(`migrated ${target}\n`); From 2a7f9589990308668b35f68c0910aa9f84c9134e Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 21:49:53 +0000 Subject: [PATCH 04/58] docs: define hub and runner modes for thread machines Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/internals/thread-machines.md | 115 ++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 docs/internals/thread-machines.md diff --git a/docs/internals/thread-machines.md b/docs/internals/thread-machines.md new file mode 100644 index 000000000000..04ada5089536 --- /dev/null +++ b/docs/internals/thread-machines.md @@ -0,0 +1,115 @@ +# Thread machines: hub and runner modes + +Managed Aldo is moving from one persistent machine per user to Cursor-style cloud agents: +the T3 interface, history, settings, and projects are served without any machine, and each +thread's work runs on its own machine that is created, resumed, or recreated on demand. The +platform design is `docs/cloud-agents-architecture.md` in the remote-dev repository. This +document is the contract inside T3. + +Standalone T3 behavior is unchanged. Both new modes are opt-in and only managed Aldo +staging enables them. + +## Server modes + +`ServerConfig.serverMode` is one of: + +| Mode | Selected by | Owns | +| ------------ | ------------------------ | ------------------------------------------------------------------------------------------- | +| `standalone` | default | Everything, as today | +| `hub` | `T3CODE_SERVER_MODE=hub` | Orchestration, projections, settings, auth, attachments, provider session directory, caches | +| `runner` | `t3 runner` command | One thread's checkout: provider drivers, git, checkpoints, terminals, files, setup scripts | + +A hub never runs git, provider CLIs, terminals, or repository code, and never reads a +checkout path. A runner never persists orchestration state. + +### Hub configuration + +| Variable | Meaning | +| ------------------------------- | ------------------------------------------------------------------------------ | +| `T3CODE_SERVER_MODE=hub` | Enables hub mode | +| `T3CODE_HUB_DATABASE_URL` | Postgres URL for the hub schema (a role without `BYPASSRLS`) | +| `T3CODE_HUB_DATABASE_ADMIN_URL` | Optional; used only to apply hub migrations. Defaults to the runtime URL | +| `T3CODE_HUB_TENANT_ID` | The Aldo user ID whose rows this process owns. One hub process serves one user | +| `T3CODE_HUB_SECRET_KEY` | Base64 32-byte key encrypting per-user secrets at rest in Postgres | +| `T3CODE_HUB_MACHINES_URL` | Base URL of the machine directory (served by the platform host process) | +| `T3CODE_HUB_MACHINES_TOKEN` | Bearer for the machine directory | + +The existing managed variables (`T3CODE_MANAGED_DEVPC`, `WORKSPACE_GATEWAY_TOKEN`, port, +base dir) keep their meaning. The base dir holds only disposable caches and logs in hub +mode; losing it loses nothing. + +### Runner configuration + +The runner is `t3 runner` with the usual base dir and port flags plus: + +| Variable | Meaning | +| ------------------------- | -------------------------------------- | +| `T3CODE_RUNNER_TOKEN` | Bearer the hub must present | +| `T3CODE_RUNNER_THREAD_ID` | The single thread this machine serves | +| `T3CODE_RUNNER_CHECKOUT` | Absolute checkout path for that thread | + +The runner listens on loopback. The platform exposes it to the hub through its own +authenticated tunnel; the runner never accepts other threads' calls. + +## Checkout paths and routing + +In hub mode every thread gets its own checkout path on its own machine: + +``` +/workspace/t/ +``` + +Thread creation in hub mode sets `worktreePath` to that path, and projects get a virtual +`workspaceRoot` of `/workspace/p/` that nothing reads. Because a checkout path +names exactly one thread, every existing `cwd`-keyed call (checkpoints, VCS, workspace +files, review) routes to that thread's runner without contract changes. Calls whose `cwd` +does not parse as a thread checkout fail with a typed error; the hub never falls back to its +local filesystem. + +## Machine directory + +The hub resolves a thread's runner through the machine directory at +`T3CODE_HUB_MACHINES_URL`. All requests carry `Authorization: Bearer +T3CODE_HUB_MACHINES_TOKEN` and JSON bodies. + +``` +POST /threads/{threadId}/machine + { "projectId", "repository": { "url", "ref" } | null, "branch", "checkout", "wake": true } +GET /threads/{threadId}/machine +POST /threads/{threadId}/machine/idle +POST /threads/{threadId}/machine/release +``` + +`POST` ensures a machine exists and, when `wake` is true, resumes or recreates it. `GET` +never wakes. Both return: + +```json +{ + "state": "none | preparing | starting | running | paused | saved | failed", + "runner": { "url": "wss://…", "token": "…", "expiresAt": "…" }, + "bootId": "…", + "detail": "human-readable progress or error" +} +``` + +`runner` is present only in `running`. A changed `bootId` means the machine restarted; +the hub reconciles sessions and turns that were active on the previous boot. `idle` tells +the platform no session or turn needs the machine; the platform decides when to pause or +save it. `release` retires the machine when the thread is archived or deleted. + +## Runner protocol + +`packages/contracts/src/runner.ts` defines `RunnerRpcGroup`, versioned independently of +the client protocol. It carries provider session control, the checkpoint group, routed +workspace, VCS, git, and terminal services, text generation, and the event stream. +Runners persist provider events in a SQLite outbox before acknowledging them; the hub +saves its per-machine cursor in the same transaction as the projection writes and +deduplicates by `(threadId, runner sequence)`. + +## What the hub persists + +Everything a standalone server keeps in its state directory moves to Postgres, scoped by +`user_id` with row-level security as a backstop: orchestration events, projections, command +receipts, provider session runtime, auth sessions and pairing links, settings, keybindings, +secrets (encrypted with `T3CODE_HUB_SECRET_KEY`), the environment ID, attachments, per-turn +diffs, repository identity, and the last reported VCS status per thread. From f78dc2c0f8f32a6ebd9fa6f29507d0b20e78c82c Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 21:53:26 +0000 Subject: [PATCH 05/58] feat(server): select standalone, hub, or runner server mode explicitly Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/cli/config.ts | 83 ++++++++++++++++++- apps/server/src/cli/runner.ts | 1 + apps/server/src/config.ts | 43 +++++++++- .../ProviderInstanceRegistryHydration.ts | 2 +- apps/server/src/server.ts | 16 ++-- 5 files changed, 130 insertions(+), 15 deletions(-) diff --git a/apps/server/src/cli/config.ts b/apps/server/src/cli/config.ts index 8b82924d3247..d34383d464a1 100644 --- a/apps/server/src/cli/config.ts +++ b/apps/server/src/cli/config.ts @@ -131,6 +131,34 @@ const EnvServerConfig = Config.all({ Config.option, Config.map(Option.getOrUndefined), ), + serverMode: Config.schema(ServerConfig.ServerMode, "T3CODE_SERVER_MODE").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), + hubDatabaseUrl: Config.string("T3CODE_HUB_DATABASE_URL").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), + hubDatabaseAdminUrl: Config.string("T3CODE_HUB_DATABASE_ADMIN_URL").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), + hubTenantId: Config.string("T3CODE_HUB_TENANT_ID").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), + hubSecretKey: Config.string("T3CODE_HUB_SECRET_KEY").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), + hubMachinesUrl: Config.string("T3CODE_HUB_MACHINES_URL").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), + hubMachinesToken: Config.string("T3CODE_HUB_MACHINES_TOKEN").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), runnerUrl: Config.string("T3CODE_RUNNER_URL").pipe( Config.option, Config.map(Option.getOrUndefined), @@ -139,6 +167,14 @@ const EnvServerConfig = Config.all({ Config.option, Config.map(Option.getOrUndefined), ), + runnerThreadId: Config.string("T3CODE_RUNNER_THREAD_ID").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), + runnerCheckout: Config.string("T3CODE_RUNNER_CHECKOUT").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), bootstrapFd: Config.int("T3CODE_BOOTSTRAP_FD").pipe( Config.option, Config.map(Option.getOrUndefined), @@ -225,12 +261,47 @@ const loadPersistedObservabilitySettings = Effect.fn(function* (settingsPath: st return parsePersistedServerObservabilitySettings(raw); }); +const resolveHubServerConfig = (env: { + readonly hubDatabaseUrl?: string | undefined; + readonly hubDatabaseAdminUrl?: string | undefined; + readonly hubTenantId?: string | undefined; + readonly hubSecretKey?: string | undefined; + readonly hubMachinesUrl?: string | undefined; + readonly hubMachinesToken?: string | undefined; + readonly runnerUrl?: string | undefined; +}) => + Effect.gen(function* () { + const missing = [ + ["T3CODE_HUB_DATABASE_URL", env.hubDatabaseUrl], + ["T3CODE_HUB_TENANT_ID", env.hubTenantId], + ["T3CODE_HUB_SECRET_KEY", env.hubSecretKey], + ] + .filter(([, value]) => !value) + .map(([name]) => name); + if (!env.runnerUrl && (!env.hubMachinesUrl || !env.hubMachinesToken)) { + missing.push("T3CODE_HUB_MACHINES_URL and T3CODE_HUB_MACHINES_TOKEN"); + } + if (missing.length > 0) { + return yield* Effect.die(new Error(`Hub mode requires ${missing.join(", ")}.`)); + } + return { + databaseUrl: env.hubDatabaseUrl!, + databaseAdminUrl: env.hubDatabaseAdminUrl, + tenantId: env.hubTenantId!, + secretKey: env.hubSecretKey!, + machinesUrl: env.hubMachinesUrl, + machinesToken: env.hubMachinesToken, + } satisfies ServerConfig.HubServerConfig; + }); + export const resolveServerConfig = ( flags: CliServerFlags, cliLogLevel: Option.Option, options?: { readonly startupPresentation?: ServerConfig.StartupPresentation; readonly forceAutoBootstrapProjectFromCwd?: boolean; + /** Set by commands that fix the mode, such as `t3 runner`. */ + readonly serverMode?: ServerConfig.ServerMode; }, ) => Effect.gen(function* () { @@ -366,6 +437,8 @@ export const resolveServerConfig = ( () => (mode === "desktop" ? "127.0.0.1" : undefined), ); const logLevel = Option.getOrElse(cliLogLevel, () => env.logLevel); + const serverMode = options?.serverMode ?? env.serverMode ?? "standalone"; + const hub = serverMode === "hub" ? yield* resolveHubServerConfig(env) : undefined; const config: ServerConfig.ServerConfig["Service"] = { logLevel, @@ -399,8 +472,14 @@ export const resolveServerConfig = ( museCodeEnabled: env.museCodeEnabled ?? !env.managedDevPc, primeAgentSubscriptionOAuthEnabled: env.primeAgentSubscriptionOAuthEnabled, managedGatewayToken: env.managedGatewayToken, - runnerUrl: env.runnerUrl, - runnerToken: env.runnerToken, + // Hub and runner settings are omitted entirely in standalone mode so the + // standalone configuration stays identical to upstream T3. + ...(serverMode === "standalone" ? {} : { serverMode }), + ...(hub ? { hub } : {}), + ...(env.runnerUrl ? { runnerUrl: env.runnerUrl } : {}), + ...(env.runnerToken ? { runnerToken: env.runnerToken } : {}), + ...(env.runnerThreadId ? { runnerThreadId: env.runnerThreadId } : {}), + ...(env.runnerCheckout ? { runnerCheckout: env.runnerCheckout } : {}), startupPresentation, desktopBootstrapToken, desktopTelemetryFd, diff --git a/apps/server/src/cli/runner.ts b/apps/server/src/cli/runner.ts index e4b8ab496542..db87ac3f2703 100644 --- a/apps/server/src/cli/runner.ts +++ b/apps/server/src/cli/runner.ts @@ -14,6 +14,7 @@ export const runnerCommand = Command.make("runner", { ...sharedServerCommandFlag const config = yield* resolveServerConfig(flags, logLevel, { startupPresentation: "headless", forceAutoBootstrapProjectFromCwd: false, + serverMode: "runner", }); return yield* runRunner.pipe(Effect.provideService(ServerConfig, config)); }), diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index ca7231182e05..fc8a09c0b07a 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -57,6 +57,28 @@ export interface DeriveServerPathsOptions { /** * ServerConfig - Service tag for server runtime configuration. */ +export const ServerMode = Schema.Literals(["standalone", "hub", "runner"]); +export type ServerMode = typeof ServerMode.Type; + +/** Standalone unless a hub or runner mode was selected. */ +export const serverModeOf = (config: { + readonly serverMode?: ServerMode | undefined; +}): ServerMode => config.serverMode ?? "standalone"; + +export interface HubServerConfig { + /** Postgres URL for the hub schema, using a role without BYPASSRLS. */ + readonly databaseUrl: string; + /** Optional role used only to apply hub migrations. */ + readonly databaseAdminUrl?: string | undefined; + /** The Aldo user whose rows this process owns. */ + readonly tenantId: string; + /** Base64 32-byte key that encrypts per-user secrets at rest. */ + readonly secretKey: string; + /** Machine directory base URL; absent only with a development `runnerUrl`. */ + readonly machinesUrl?: string | undefined; + readonly machinesToken?: string | undefined; +} + export class ServerConfig extends Context.Service< ServerConfig, ServerDerivedPaths & { @@ -99,13 +121,26 @@ export class ServerConfig extends Context.Service< */ readonly managedGatewayToken?: string | undefined; /** - * Hub mode (prototype): when set, this server owns no checkout. Provider - * sessions, checkpoints and workspace validation run on the runner at this - * WebSocket URL (see `runner/`). + * `standalone` owns everything (the default). `hub` owns orchestration and + * persistence but no checkout; thread work runs on per-thread runners. + * `runner` serves one thread's checkout to a hub. See + * docs/internals/thread-machines.md. + */ + readonly serverMode?: ServerMode | undefined; + /** Hub-mode settings; present only when `serverMode` is `hub`. */ + readonly hub?: HubServerConfig | undefined; + /** + * Development override for hub mode: a single runner WebSocket URL used + * for every thread instead of resolving runners through the machine + * directory. */ readonly runnerUrl?: string | undefined; - /** Shared bearer secret presented to the runner (hub) or required from hubs (runner). */ + /** Bearer presented to runners (hub) or required from hubs (runner). */ readonly runnerToken?: string | undefined; + /** Runner mode: the only thread this runner serves. */ + readonly runnerThreadId?: string | undefined; + /** Runner mode: the absolute checkout path of that thread. */ + readonly runnerCheckout?: string | undefined; readonly startupPresentation: StartupPresentation; readonly desktopBootstrapToken: string | undefined; readonly desktopTelemetryFd?: number | undefined; diff --git a/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts b/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts index 89e5c40ecedf..350dc8011144 100644 --- a/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts +++ b/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts @@ -199,7 +199,7 @@ export const ProviderInstanceRegistryHydrationLive: Layer.Layer< // Hub mode: every provider runs on the thread's runner. The hub keeps the // same driver kinds and instance ids, but each adapter forwards over the // runner protocol instead of spawning a provider CLI locally. - return serverConfig.runnerUrl + return serverConfig.serverMode === "hub" ? makeProviderInstanceRegistryHydration(drivers.map(makeRemoteProviderDriver)) : makeProviderInstanceRegistryHydration(drivers); }), diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index d99d9f2cd4d0..19b4a480682e 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -171,14 +171,14 @@ const PtyAdapterLive = Layer.unwrap( * Picks the checkout-bound implementation (`local`) or the hub substitute * (`hub`) depending on whether this server delegates to a runner. */ -const byRunnerMode = ( +const byServerMode = ( local: Layer.Layer, hub: Layer.Layer, ): Layer.Layer => Layer.unwrap( Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; - return (config.runnerUrl ? hub : local) as Layer.Layer; + return (config.serverMode === "hub" ? hub : local) as Layer.Layer; }), ); @@ -291,7 +291,7 @@ export const PlatformServicesLive = Layer.unwrap( }), ); -const CheckoutGitProbeLayerLive = byRunnerMode( +const CheckoutGitProbeLayerLive = byServerMode( Layer.succeed(CheckoutGitProbe, localCheckoutGitProbe), HubLayers.remoteCheckoutGitProbeLayer, ); @@ -383,7 +383,7 @@ const VcsLayerLive = Layer.empty.pipe( Layer.provideMerge(GrokReviewLayerLive), Layer.provideMerge(SourceControlRepositoryServiceLayerLive), Layer.provideMerge( - byRunnerMode( + byServerMode( VcsStatusBroadcaster.layer.pipe(Layer.provide(GitWorkflowLayerLive)), HubLayers.hubVcsStatusBroadcasterLayer, ), @@ -393,7 +393,7 @@ const VcsLayerLive = Layer.empty.pipe( const CheckpointingLayerLive = Layer.empty.pipe( Layer.provideMerge(CheckpointDiffQuery.layer), Layer.provideMerge( - byRunnerMode( + byServerMode( CheckpointStore.layer.pipe(Layer.provide(VcsDriverRegistryLayerLive)), HubLayers.remoteCheckpointStoreLayer, ), @@ -420,8 +420,8 @@ const WorkspaceFileSystemLayerLive = WorkspaceFileSystem.layer.pipe( ); const WorkspaceLayerLive = Layer.mergeAll( - byRunnerMode(WorkspacePaths.layer, HubLayers.remoteWorkspacePathsLayer), - byRunnerMode(WorkspaceEntriesLayerLive, HubLayers.hubWorkspaceEntriesLayer), + byServerMode(WorkspacePaths.layer, HubLayers.remoteWorkspacePathsLayer), + byServerMode(WorkspaceEntriesLayerLive, HubLayers.hubWorkspaceEntriesLayer), WorkspaceFileSystemLayerLive, ); @@ -493,7 +493,7 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( Layer.provideMerge(WorkspaceLayerLive), Layer.provideMerge(ProjectFaviconResolverLayerLive), Layer.provideMerge( - byRunnerMode(RepositoryIdentityResolver.layer, HubLayers.hubRepositoryIdentityResolverLayer), + byServerMode(RepositoryIdentityResolver.layer, HubLayers.hubRepositoryIdentityResolverLayer), ), Layer.provideMerge(ServerEnvironmentLayerLive), Layer.provideMerge(AuthLayerLive), From 4ff0cbbc14ceb11dcd9e0a6c5155a2729fddf0fa Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 22:20:26 +0000 Subject: [PATCH 06/58] feat(server): run hub persistence on tenant-scoped Postgres with migrations and RLS Hub mode (a hub database URL in the server config) now keeps state in Postgres instead of state.sqlite. A HubDatabase reference, provided at the server and auth CLI roots, carries the tenant client; the SQLite persistence layer hands it out, and every repository layer picks its Postgres port through localOrHub, so standalone composition and types are unchanged. The tenant client applies SET LOCAL hub.user_id to every transaction and wraps autocommit statements in one, so row-level security works behind a transaction-mode pooler. Hub migrations are ordered numbered entries applied per transaction under an advisory lock, with the admin URL when configured, and the runtime role is granted access afterwards. Port AuthSessions, AuthPairingLinks and ProviderSessionRuntime, and replace the prototype baseline with migration 001, which adds hub documents, secrets, attachments, the project repository identity column, and forced tenant policies on every table. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scripts/hub-proto/bench-activation.ts | 364 --------------- .../scripts/hub-proto/compare-backends.ts | 26 +- .../server/scripts/hub-proto/import-sqlite.sh | 41 -- apps/server/src/auth/EnvironmentAuth.ts | 3 + .../Layers/ProjectionSnapshotQuery.ts | 8 +- .../src/persistence/AuthPairingLinks.ts | 7 +- apps/server/src/persistence/AuthSessions.ts | 7 +- .../Layers/OrchestrationCommandReceipts.ts | 8 +- .../Layers/OrchestrationEventStore.ts | 7 +- .../server/src/persistence/Layers/Postgres.ts | 72 --- .../Layers/ProjectionCheckpoints.ts | 8 +- .../Layers/ProjectionPendingApprovals.ts | 8 +- .../persistence/Layers/ProjectionProjects.ts | 8 +- .../src/persistence/Layers/ProjectionState.ts | 8 +- .../Layers/ProjectionThreadActivities.ts | 8 +- .../Layers/ProjectionThreadMessages.ts | 8 +- .../Layers/ProjectionThreadProposedPlans.ts | 8 +- .../Layers/ProjectionThreadSessions.ts | 8 +- .../persistence/Layers/ProjectionThreads.ts | 8 +- .../src/persistence/Layers/ProjectionTurns.ts | 8 +- apps/server/src/persistence/Layers/Sqlite.ts | 17 +- .../persistence/Postgres/AuthPairingLinks.ts | 346 ++++++++++++++ .../src/persistence/Postgres/AuthSessions.ts | 439 ++++++++++++++++++ .../src/persistence/Postgres/HubClient.ts | 164 +++++++ .../persistence/Postgres/HubDatabase.test.ts | 185 ++++++++ .../src/persistence/Postgres/HubDatabase.ts | 90 ++++ .../persistence/Postgres/HubDatabaseLive.ts | 88 ++++ .../persistence/Postgres/HubEngine.test.ts | 429 ----------------- .../src/persistence/Postgres/HubEngine.ts | 92 ---- .../src/persistence/Postgres/HubMigrator.ts | 84 ++++ .../Postgres/ProviderSessionRuntime.ts | 304 ++++++++++++ .../persistence/Postgres/hubTestDatabase.ts | 112 +++++ .../001_HubBaseline.ts} | 267 ++++++----- .../persistence/Postgres/migrations/index.ts | 20 + .../src/persistence/ProviderSessionRuntime.ts | 7 +- apps/server/src/server.ts | 3 + 36 files changed, 2111 insertions(+), 1159 deletions(-) delete mode 100644 apps/server/scripts/hub-proto/bench-activation.ts delete mode 100755 apps/server/scripts/hub-proto/import-sqlite.sh delete mode 100644 apps/server/src/persistence/Layers/Postgres.ts create mode 100644 apps/server/src/persistence/Postgres/AuthPairingLinks.ts create mode 100644 apps/server/src/persistence/Postgres/AuthSessions.ts create mode 100644 apps/server/src/persistence/Postgres/HubClient.ts create mode 100644 apps/server/src/persistence/Postgres/HubDatabase.test.ts create mode 100644 apps/server/src/persistence/Postgres/HubDatabase.ts create mode 100644 apps/server/src/persistence/Postgres/HubDatabaseLive.ts delete mode 100644 apps/server/src/persistence/Postgres/HubEngine.test.ts delete mode 100644 apps/server/src/persistence/Postgres/HubEngine.ts create mode 100644 apps/server/src/persistence/Postgres/HubMigrator.ts create mode 100644 apps/server/src/persistence/Postgres/ProviderSessionRuntime.ts create mode 100644 apps/server/src/persistence/Postgres/hubTestDatabase.ts rename apps/server/src/persistence/Postgres/{Schema.ts => migrations/001_HubBaseline.ts} (60%) create mode 100644 apps/server/src/persistence/Postgres/migrations/index.ts diff --git a/apps/server/scripts/hub-proto/bench-activation.ts b/apps/server/scripts/hub-proto/bench-activation.ts deleted file mode 100644 index c434d07ff8db..000000000000 --- a/apps/server/scripts/hub-proto/bench-activation.ts +++ /dev/null @@ -1,364 +0,0 @@ -// Hub prototype benchmark. Prints only counts, sizes, and timings. -// -// node --expose-gc scripts/hub-proto/bench-activation.ts pg -// node --expose-gc scripts/hub-proto/bench-activation.ts sqlite - -// -// pg mode clones the owner's metadata for synthetic users to measure memory per -// active engine, and counts round trips by wrapping node-postgres' Client.query. -import * as NodeModule from "node:module"; -import * as NodePerfHooks from "node:perf_hooks"; - -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { - CommandId, - EventId, - MessageId, - ThreadId, - type OrchestrationReadModel, -} from "@t3tools/contracts"; -import * as Context from "effect/Context"; -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; -import * as ManagedRuntime from "effect/ManagedRuntime"; -import * as Option from "effect/Option"; -import * as SqlClient from "effect/unstable/sql/SqlClient"; - -import { ServerConfig } from "../../src/config.ts"; -import { OrchestrationEngineLive } from "../../src/orchestration/Layers/OrchestrationEngine.ts"; -import { OrchestrationProjectionPipelineLive } from "../../src/orchestration/Layers/ProjectionPipeline.ts"; -import { OrchestrationProjectionSnapshotQueryLive } from "../../src/orchestration/Layers/ProjectionSnapshotQuery.ts"; -import { OrchestrationEngineService } from "../../src/orchestration/Services/OrchestrationEngine.ts"; -import { ProjectionSnapshotQuery } from "../../src/orchestration/Services/ProjectionSnapshotQuery.ts"; -import * as ThreadBackgroundLiveness from "../../src/orchestration/ThreadBackgroundLiveness.ts"; -import * as ThreadPlanProgress from "../../src/orchestration/ThreadPlanProgress.ts"; -import { OrchestrationCommandReceiptRepositoryLive } from "../../src/persistence/Layers/OrchestrationCommandReceipts.ts"; -import { OrchestrationEventStoreLive } from "../../src/persistence/Layers/OrchestrationEventStore.ts"; -import { makeSqlitePersistenceLive } from "../../src/persistence/Layers/Sqlite.ts"; -import { - makeHubSharedLayer, - makeHubUserEngineLayer, - type HubSharedServices, -} from "../../src/persistence/Postgres/HubEngine.ts"; -import { RepositoryIdentityResolver } from "../../src/project/RepositoryIdentityResolver.ts"; - -const [mode, target, ownerArg, threadArg] = process.argv.slice(2); -if ((mode !== "pg" && mode !== "sqlite") || !target || !threadArg) { - throw new Error("usage: bench-activation.ts pg|sqlite "); -} -const owner = ownerArg ?? "owner"; -const threadIds = threadArg.split(",").map((id) => ThreadId.make(id)); -const ACTIVATIONS = Number(process.env.BENCH_ACTIVATIONS ?? 30); -const ENGINES = Number(process.env.BENCH_ENGINES ?? 20); -const DETAIL_RUNS = Number(process.env.BENCH_DETAIL_RUNS ?? 10); -const DISPATCHES = Number(process.env.BENCH_DISPATCHES ?? 50); - -const out = (line: string) => process.stdout.write(`${line}\n`); -const gc = () => (globalThis as { gc?: () => void }).gc?.(); -const mib = (bytes: number) => Math.round((bytes / 1048576) * 10) / 10; -const percentile = (values: ReadonlyArray, p: number) => { - const sorted = values.toSorted((a, b) => a - b); - return sorted[Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1)] ?? NaN; -}; -const summary = (values: ReadonlyArray) => - `p50=${percentile(values, 50).toFixed(1)}ms p90=${percentile(values, 90).toFixed(1)}ms min=${Math.min(...values).toFixed(1)}ms max=${Math.max(...values).toFixed(1)}ms n=${values.length}`; -const time = async (run: () => Promise) => { - const start = NodePerfHooks.performance.now(); - const value = await run(); - return { ms: NodePerfHooks.performance.now() - start, value }; -}; - -// Round-trip counter over node-postgres (resolved through @effect/sql-pg). -let pgQueries = 0; -if (mode === "pg") { - const requirePg = NodeModule.createRequire(import.meta.resolve("@effect/sql-pg/PgClient")); - const Pg = requirePg("pg") as { - Client: { prototype: { query: (...args: unknown[]) => unknown } }; - }; - const query = Pg.Client.prototype.query; - Pg.Client.prototype.query = function (this: unknown, ...args: unknown[]) { - pgQueries += 1; - return query.apply(this, args); - }; -} - -const NoRepositoryIdentityLive = Layer.succeed(RepositoryIdentityResolver, { - resolve: () => Effect.succeed(null), -}); - -type EngineRuntime = ManagedRuntime.ManagedRuntime< - OrchestrationEngineService | ProjectionSnapshotQuery, - unknown ->; - -const sqliteEngineLayer = (file: string) => - Layer.mergeAll( - OrchestrationEngineLive.pipe( - Layer.provide(OrchestrationProjectionSnapshotQueryLive), - Layer.provide(OrchestrationProjectionPipelineLive), - ), - OrchestrationProjectionSnapshotQueryLive, - ).pipe( - Layer.provideMerge(ThreadBackgroundLiveness.layer), - Layer.provide(ThreadPlanProgress.layer), - Layer.provide(OrchestrationEventStoreLive), - Layer.provideMerge(OrchestrationCommandReceiptRepositoryLive), - Layer.provide(NoRepositoryIdentityLive), - Layer.provide(makeSqlitePersistenceLive(file)), - Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-bench-" })), - Layer.provideMerge(NodeServices.layer), - ); - -const shared = - mode === "pg" - ? ManagedRuntime.make(makeHubSharedLayer({ url: target, maxConnections: 10 })) - : null; -const sharedContext = shared ? await shared.runPromise(Effect.context()) : null; -const sql = sharedContext ? Context.get(sharedContext, SqlClient.SqlClient) : null; - -const makeEngine = (userId: string): EngineRuntime => - ManagedRuntime.make( - sharedContext - ? makeHubUserEngineLayer({ userId, shared: sharedContext }) - : sqliteEngineLayer(target), - ) as EngineRuntime; - -const activate = async (userId: string) => { - const runtime = makeEngine(userId); - const queriesBefore = pgQueries; - const { ms } = await time(() => runtime.runPromise(Effect.service(OrchestrationEngineService))); - return { runtime, ms, queries: pgQueries - queriesBefore }; -}; - -const results: Record = { mode }; -// BENCH_SECTIONS=memory measures engine memory in a fresh process (one warm-up -// activation first), so earlier snapshot reads do not pre-grow the heap. -const onlyMemory = process.env.BENCH_SECTIONS === "memory"; -if (onlyMemory) { - const warmup = await activate(owner); - await warmup.runtime.dispose(); -} else { - // Activation (bootstrap + command read model), cold first then repeated. - const cold = await activate(owner); - const readModel = (await cold.runtime.runPromise( - Effect.flatMap(Effect.service(ProjectionSnapshotQuery), (query) => query.getCommandReadModel()), - )) as OrchestrationReadModel; - out( - `read model: projects=${readModel.projects.length} threads=${readModel.threads.length} ` + - `latestTurns=${readModel.threads.filter((thread) => thread.latestTurn !== null).length} ` + - `sessions=${readModel.threads.filter((thread) => thread.session !== null).length} ` + - `plans=${readModel.threads.reduce((n, thread) => n + thread.proposedPlans.length, 0)} ` + - `snapshotSequence=${readModel.snapshotSequence} jsonBytes=${JSON.stringify(readModel).length}`, - ); - out(`cold activation: ${cold.ms.toFixed(1)}ms queries=${cold.queries}`); - await cold.runtime.dispose(); - const activationMs: number[] = []; - let activationQueries = 0; - for (let index = 0; index < ACTIVATIONS; index += 1) { - const run = await activate(owner); - activationMs.push(run.ms); - activationQueries = run.queries; - await run.runtime.dispose(); - } - out(`warm activation: ${summary(activationMs)} queries/activation=${activationQueries}`); - results.activation = { coldMs: cold.ms, warm: activationMs, queries: activationQueries }; - - // getCommandReadModel alone, on a live engine. - const live = await activate(owner); - const query = await live.runtime.runPromise(Effect.service(ProjectionSnapshotQuery)); - const readModelMs: number[] = []; - for (let index = 0; index < ACTIVATIONS; index += 1) { - readModelMs.push((await time(() => live.runtime.runPromise(query.getCommandReadModel()))).ms); - } - out(`getCommandReadModel: ${summary(readModelMs)}`); - const shellMs: number[] = []; - let shellThreads = 0; - for (let index = 0; index < ACTIVATIONS; index += 1) { - const run = await time(() => live.runtime.runPromise(query.getShellSnapshot())); - shellMs.push(run.ms); - shellThreads = run.value.threads.length; - } - out(`getShellSnapshot: ${summary(shellMs)} threads=${shellThreads}`); - results.readModelMs = readModelMs; - results.shellMs = shellMs; - - // Thread detail loads: the client's first page (turnLimit 10) and the full thread. - const details: Record = {}; - for (const [index, threadId] of threadIds.entries()) { - for (const window of [{ turnLimit: 10 }, undefined] as const) { - const samples: number[] = []; - let counts = ""; - for (let run = 0; run < DETAIL_RUNS; run += 1) { - const queriesBefore = pgQueries; - const { ms, value } = await time(() => - live.runtime.runPromise(query.getThreadDetailSnapshot(threadId, window)), - ); - samples.push(ms); - const thread = Option.getOrThrow(value).thread; - counts = - `messages=${thread.messages.length} activities=${thread.activities.length} ` + - `checkpoints=${thread.checkpoints.length} queries=${pgQueries - queriesBefore}`; - } - const label = `thread#${index + 1} ${window ? "page(turnLimit=10)" : "full"}`; - out(`detail ${label}: ${summary(samples)} ${counts}`); - details[label] = { samples, counts }; - } - } - results.details = details; - - // Command dispatch on the owner's largest thread: activity appends and streaming deltas. - const engine = await live.runtime.runPromise(Effect.service(OrchestrationEngineService)); - const dispatchThread = threadIds[0]!; - const appendMs: number[] = []; - const appendQueries: number[] = []; - const createdAt = "2026-09-25T12:00:00.000Z"; - for (let index = 0; index < DISPATCHES; index += 1) { - const queriesBefore = pgQueries; - appendMs.push( - ( - await time(() => - live.runtime.runPromise( - engine.dispatch({ - type: "thread.activity.append", - commandId: CommandId.make(`bench-activity-${process.pid}-${index}`), - threadId: dispatchThread, - activity: { - id: EventId.make(`bench-activity-${process.pid}-${index}`), - tone: "tool", - kind: "bench.synthetic", - summary: "synthetic benchmark activity", - payload: { index }, - turnId: null, - createdAt, - }, - createdAt, - }), - ), - ) - ).ms, - ); - appendQueries.push(pgQueries - queriesBefore); - } - out( - `dispatch thread.activity.append: ${summary(appendMs)} queries/command=${percentile(appendQueries, 50)}`, - ); - const deltaMs: number[] = []; - const deltaQueries: number[] = []; - for (let index = 0; index < DISPATCHES; index += 1) { - const queriesBefore = pgQueries; - deltaMs.push( - ( - await time(() => - live.runtime.runPromise( - engine.dispatch({ - type: "thread.message.assistant.delta", - commandId: CommandId.make(`bench-delta-${process.pid}-${index}`), - threadId: dispatchThread, - messageId: MessageId.make(`bench-message-${process.pid}`), - delta: "x", - createdAt, - }), - ), - ) - ).ms, - ); - deltaQueries.push(pgQueries - queriesBefore); - } - out( - `dispatch thread.message.assistant.delta: ${summary(deltaMs)} queries/command=${percentile(deltaQueries, 50)}`, - ); - results.dispatch = { appendMs, appendQueries, deltaMs, deltaQueries }; - await live.runtime.dispose(); -} - -// Memory per active engine: distinct synthetic users with the owner's metadata. -if (sql && shared) { - const users = Array.from({ length: ENGINES }, (_, index) => `bench-user-${index + 1}`); - await shared.runPromise( - Effect.gen(function* () { - for (const table of [ - "projection_projects", - "projection_threads", - "projection_thread_sessions", - "projection_turns", - "projection_thread_proposed_plans", - "projection_state", - ]) { - const columns = (yield* sql<{ name: string }>` - SELECT column_name AS name FROM information_schema.columns - WHERE table_name = ${table} AND column_name NOT IN ('user_id', 'row_id') - ORDER BY ordinal_position - `).map((row) => row.name); - const list = sql.literal(columns.join(", ")); - yield* sql`DELETE FROM ${sql(table)} WHERE user_id LIKE 'bench-user-%'`; - for (const user of users) { - yield* sql` - INSERT INTO ${sql(table)} (user_id, ${list}) - SELECT ${user}, ${list} FROM ${sql(table)} WHERE user_id = ${owner} - `; - } - } - yield* sql`DELETE FROM hub_users WHERE user_id LIKE 'bench-user-%'`; - for (const user of users) { - yield* sql` - INSERT INTO hub_users (user_id, last_event_sequence) - SELECT ${user}, last_event_sequence FROM hub_users WHERE user_id = ${owner} - `; - } - }), - ); - gc(); - gc(); - const before = process.memoryUsage(); - const engines: EngineRuntime[] = []; - for (const user of users) { - engines.push((await activate(user)).runtime); - } - gc(); - gc(); - const after = process.memoryUsage(); - out( - `memory: ${ENGINES} active engines rssDelta=${mib(after.rss - before.rss)}MiB ` + - `heapUsedDelta=${mib(after.heapUsed - before.heapUsed)}MiB ` + - `perEngine rss=${mib((after.rss - before.rss) / ENGINES)}MiB heap=${mib((after.heapUsed - before.heapUsed) / ENGINES)}MiB ` + - `baseRss=${mib(before.rss)}MiB`, - ); - results.memory = { engines: ENGINES, before, after }; - // Parallel activation of all synthetic users at once (cold-start storm). - for (const runtime of engines) await runtime.dispose(); - const storm = await time(() => Promise.all(users.map((user) => activate(user)))); - out(`parallel activation of ${ENGINES} users: ${storm.ms.toFixed(1)}ms total`); - for (const run of storm.value) await run.runtime.dispose(); - await shared.runPromise( - Effect.gen(function* () { - for (const table of [ - "projection_projects", - "projection_threads", - "projection_thread_sessions", - "projection_turns", - "projection_thread_proposed_plans", - "projection_state", - "hub_users", - ]) { - yield* sql`DELETE FROM ${sql(table)} WHERE user_id LIKE 'bench-user-%'`; - } - }), - ); - await shared.dispose(); -} else { - gc(); - gc(); - const before = process.memoryUsage(); - const engines: EngineRuntime[] = []; - for (let index = 0; index < ENGINES; index += 1) engines.push((await activate(owner)).runtime); - gc(); - gc(); - const after = process.memoryUsage(); - out( - `memory: ${ENGINES} active engines rssDelta=${mib(after.rss - before.rss)}MiB ` + - `heapUsedDelta=${mib(after.heapUsed - before.heapUsed)}MiB ` + - `perEngine rss=${mib((after.rss - before.rss) / ENGINES)}MiB heap=${mib((after.heapUsed - before.heapUsed) / ENGINES)}MiB`, - ); - for (const runtime of engines) await runtime.dispose(); -} - -out(`RESULTS_JSON ${JSON.stringify(results)}`); diff --git a/apps/server/scripts/hub-proto/compare-backends.ts b/apps/server/scripts/hub-proto/compare-backends.ts index 188f3f5103f6..25f9441791f4 100644 --- a/apps/server/scripts/hub-proto/compare-backends.ts +++ b/apps/server/scripts/hub-proto/compare-backends.ts @@ -15,11 +15,9 @@ import { ProjectionSnapshotQuery } from "../../src/orchestration/Services/Projec import * as ThreadBackgroundLiveness from "../../src/orchestration/ThreadBackgroundLiveness.ts"; import * as ThreadPlanProgress from "../../src/orchestration/ThreadPlanProgress.ts"; import { makeSqlitePersistenceLive } from "../../src/persistence/Layers/Sqlite.ts"; -import { - makeHubSharedLayer, - makeHubUserEngineLayer, - type HubSharedServices, -} from "../../src/persistence/Postgres/HubEngine.ts"; +import { layerConfig as ServerPersistenceLive } from "../../src/persistence/Layers/Sqlite.ts"; +import { HubDatabase } from "../../src/persistence/Postgres/HubDatabase.ts"; +import { makeHubDatabase } from "../../src/persistence/Postgres/HubDatabaseLive.ts"; import { RepositoryIdentityResolver } from "../../src/project/RepositoryIdentityResolver.ts"; const [sqliteFile, pgUrl, owner, threadArg, skipArg] = process.argv.slice(2); @@ -43,10 +41,21 @@ const sqliteQuery = ManagedRuntime.make( Layer.provide(NodeServices.layer), ), ); -const shared = ManagedRuntime.make(makeHubSharedLayer({ url: pgUrl })); -const sharedContext = await shared.runPromise(Effect.context()); +// The hub reads through the same live layer as the server, selected by HubDatabase. const pgQuery = ManagedRuntime.make( - makeHubUserEngineLayer({ userId: owner, shared: sharedContext }), + OrchestrationProjectionSnapshotQueryLive.pipe( + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide( + Layer.succeed(RepositoryIdentityResolver, { resolve: () => Effect.succeed(null) }), + ), + Layer.provide(ServerPersistenceLive), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-compare-" })), + Layer.provide(NodeServices.layer), + Layer.provide( + Layer.effect(HubDatabase, makeHubDatabase({ databaseUrl: pgUrl, tenantId: owner })), + ), + ), ); const both = async ( @@ -155,5 +164,4 @@ check( out(failures === 0 ? "ALL EQUAL" : `${failures} DIFFERENCES`); await pgQuery.dispose(); -await shared.dispose(); await sqliteQuery.dispose(); diff --git a/apps/server/scripts/hub-proto/import-sqlite.sh b/apps/server/scripts/hub-proto/import-sqlite.sh deleted file mode 100755 index 4ce0b9e144f4..000000000000 --- a/apps/server/scripts/hub-proto/import-sqlite.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# Streams a T3 SQLite state database into the hub Postgres schema as one user. -# Rows go through a pipe (sqlite3 CSV -> psql \copy); nothing is printed. -# -# import-sqlite.sh -set -euo pipefail - -SQLITE_FILE=$1 -USER_ID=$2 -PG_URL=$3 - -TABLES=( - projection_projects - projection_threads - projection_thread_sessions - projection_turns - projection_thread_proposed_plans - projection_pending_approvals - projection_state - projection_thread_messages - projection_thread_activities - orchestration_events -) - -for table in "${TABLES[@]}"; do - # row_id is a Postgres identity column; everything else keeps SQLite's order. - columns=$(sqlite3 "file:${SQLITE_FILE}?mode=ro" \ - "SELECT group_concat(name, ', ') FROM pragma_table_info('${table}') WHERE name <> 'row_id'") - sqlite3 -csv -nullvalue '\N' -newline $'\n' "file:${SQLITE_FILE}?mode=ro" \ - "SELECT '${USER_ID}', ${columns} FROM ${table}" | - psql "$PG_URL" -q -v ON_ERROR_STOP=1 \ - -c "\\copy ${table} (user_id, ${columns}) FROM STDIN WITH (FORMAT csv, NULL '\\N')" -done - -# Continue the user's event sequence after the imported history. -psql "$PG_URL" -q -v ON_ERROR_STOP=1 -c " - INSERT INTO hub_users (user_id, last_event_sequence) - SELECT '${USER_ID}', COALESCE(MAX(last_applied_sequence), 0) - FROM projection_state WHERE user_id = '${USER_ID}' - ON CONFLICT (user_id) DO UPDATE SET last_event_sequence = EXCLUDED.last_event_sequence" -psql "$PG_URL" -q -v ON_ERROR_STOP=1 -c "ANALYZE" diff --git a/apps/server/src/auth/EnvironmentAuth.ts b/apps/server/src/auth/EnvironmentAuth.ts index 08838cb7b780..e0e02c292ad0 100644 --- a/apps/server/src/auth/EnvironmentAuth.ts +++ b/apps/server/src/auth/EnvironmentAuth.ts @@ -37,6 +37,7 @@ import * as ServerSecretStore from "./ServerSecretStore.ts"; import * as SessionStore from "./SessionStore.ts"; import { verifyRequestDpopProof } from "./dpop.ts"; import { layerConfig as SqlitePersistenceLayer } from "../persistence/Layers/Sqlite.ts"; +import * as HubDatabase from "../persistence/Postgres/HubDatabase.ts"; export const DEFAULT_SESSION_SUBJECT = "cli-issued-session"; export const INTERNAL_ADMINISTRATIVE_BOOTSTRAP_SUBJECT = "administrative-bootstrap"; @@ -1037,4 +1038,6 @@ export const storageLayer = Layer.mergeAll(ServerSecretStore.layer, SqlitePersis export const runtimeLayer = layer.pipe( Layer.provideMerge(storageLayer), Layer.provideMerge(ServerEnvironment.identityLayer), + // Hub mode keeps auth state in the tenant's Postgres database. + Layer.provideMerge(HubDatabase.layerConfig), ); diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index ea808f18d3a7..fb1b727cc030 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -68,6 +68,8 @@ import { type ProjectionThreadDetailQuery, type ProjectionSnapshotQueryShape, } from "../Services/ProjectionSnapshotQuery.ts"; +import { localOrHub } from "../../persistence/Postgres/HubDatabase.ts"; +import { PgProjectionSnapshotQueryLive } from "../../persistence/Postgres/ProjectionSnapshotQuery.ts"; const decodeReadModel = Schema.decodeUnknownEffect(OrchestrationReadModel); const decodeShellSnapshot = Schema.decodeUnknownEffect(OrchestrationShellSnapshot); @@ -3120,7 +3122,7 @@ pending_approval_requests AS ( } satisfies ProjectionSnapshotQueryShape; }); -export const OrchestrationProjectionSnapshotQueryLive = Layer.effect( - ProjectionSnapshotQuery, - makeProjectionSnapshotQuery, +export const OrchestrationProjectionSnapshotQueryLive = localOrHub( + Layer.effect(ProjectionSnapshotQuery, makeProjectionSnapshotQuery), + PgProjectionSnapshotQueryLive, ); diff --git a/apps/server/src/persistence/AuthPairingLinks.ts b/apps/server/src/persistence/AuthPairingLinks.ts index e54c977e7ab7..dd19b419f5fb 100644 --- a/apps/server/src/persistence/AuthPairingLinks.ts +++ b/apps/server/src/persistence/AuthPairingLinks.ts @@ -14,6 +14,8 @@ import { type PersistenceErrorCorrelation, PersistenceSqlError, } from "./Errors.ts"; +import { localOrHub } from "./Postgres/HubDatabase.ts"; +import * as PgRepository from "./Postgres/AuthPairingLinks.ts"; export const AuthPairingLinkRecord = Schema.Struct({ id: Schema.String, @@ -353,4 +355,7 @@ export const make = Effect.gen(function* () { } satisfies AuthPairingLinkRepository["Service"]; }); -export const layer = Layer.effect(AuthPairingLinkRepository, make); +export const layer = localOrHub( + Layer.effect(AuthPairingLinkRepository, make), + Layer.effect(AuthPairingLinkRepository, PgRepository.make), +); diff --git a/apps/server/src/persistence/AuthSessions.ts b/apps/server/src/persistence/AuthSessions.ts index 579d3a608190..b5ab2628ae76 100644 --- a/apps/server/src/persistence/AuthSessions.ts +++ b/apps/server/src/persistence/AuthSessions.ts @@ -20,6 +20,8 @@ import { type PersistenceErrorCorrelation, PersistenceSqlError, } from "./Errors.ts"; +import { localOrHub } from "./Postgres/HubDatabase.ts"; +import * as PgRepository from "./Postgres/AuthSessions.ts"; export const AuthSessionClientMetadataRecord = Schema.Struct({ label: Schema.NullOr(Schema.String), @@ -451,4 +453,7 @@ export const make = Effect.gen(function* () { } satisfies AuthSessionRepository["Service"]; }); -export const layer = Layer.effect(AuthSessionRepository, make); +export const layer = localOrHub( + Layer.effect(AuthSessionRepository, make), + Layer.effect(AuthSessionRepository, PgRepository.make), +); diff --git a/apps/server/src/persistence/Layers/OrchestrationCommandReceipts.ts b/apps/server/src/persistence/Layers/OrchestrationCommandReceipts.ts index 989f5d78f8d8..1fbef988df15 100644 --- a/apps/server/src/persistence/Layers/OrchestrationCommandReceipts.ts +++ b/apps/server/src/persistence/Layers/OrchestrationCommandReceipts.ts @@ -11,6 +11,8 @@ import { OrchestrationCommandReceiptRepository, type OrchestrationCommandReceiptRepositoryShape, } from "../Services/OrchestrationCommandReceipts.ts"; +import { localOrHub } from "../Postgres/HubDatabase.ts"; +import { PgOrchestrationCommandReceiptRepositoryLive } from "../Postgres/OrchestrationCommandReceipts.ts"; const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -84,7 +86,7 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { } satisfies OrchestrationCommandReceiptRepositoryShape; }); -export const OrchestrationCommandReceiptRepositoryLive = Layer.effect( - OrchestrationCommandReceiptRepository, - makeOrchestrationCommandReceiptRepository, +export const OrchestrationCommandReceiptRepositoryLive = localOrHub( + Layer.effect(OrchestrationCommandReceiptRepository, makeOrchestrationCommandReceiptRepository), + PgOrchestrationCommandReceiptRepositoryLive, ); diff --git a/apps/server/src/persistence/Layers/OrchestrationEventStore.ts b/apps/server/src/persistence/Layers/OrchestrationEventStore.ts index cde45de064e7..a4f9313b0304 100644 --- a/apps/server/src/persistence/Layers/OrchestrationEventStore.ts +++ b/apps/server/src/persistence/Layers/OrchestrationEventStore.ts @@ -28,6 +28,8 @@ import { OrchestrationEventStore, type OrchestrationEventStoreShape, } from "../Services/OrchestrationEventStore.ts"; +import { localOrHub } from "../Postgres/HubDatabase.ts"; +import { PgOrchestrationEventStoreLive } from "../Postgres/OrchestrationEventStore.ts"; const decodeEvent = Schema.decodeUnknownEffect(OrchestrationEvent); const UnknownFromJsonString = Schema.fromJsonString(Schema.Unknown); @@ -403,4 +405,7 @@ const makeEventStore = Effect.gen(function* () { } satisfies OrchestrationEventStoreShape; }); -export const OrchestrationEventStoreLive = Layer.effect(OrchestrationEventStore, makeEventStore); +export const OrchestrationEventStoreLive = localOrHub( + Layer.effect(OrchestrationEventStore, makeEventStore), + PgOrchestrationEventStoreLive, +); diff --git a/apps/server/src/persistence/Layers/Postgres.ts b/apps/server/src/persistence/Layers/Postgres.ts deleted file mode 100644 index b4cb13e10874..000000000000 --- a/apps/server/src/persistence/Layers/Postgres.ts +++ /dev/null @@ -1,72 +0,0 @@ -import * as PgClient from "@effect/sql-pg/PgClient"; -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; -import * as Redacted from "effect/Redacted"; -import * as SqlClient from "effect/unstable/sql/SqlClient"; - -import { HUB_BASELINE_STATEMENTS, HUB_SCHEMA_VERSION } from "../Postgres/Schema.ts"; - -export interface HubPostgresConfig { - readonly url: string; - readonly maxConnections?: number; - readonly applicationName?: string; -} - -// node-postgres returns int8/numeric as strings. Hub sequences, COUNT(*) and -// MAX(sequence) are int8, and the shared row schemas decode numbers, so parse -// them here. Values stay far below 2^53 (sequences are per user). -const identity = (value: string) => value; -const toNumber = (value: string) => Number(value); -const HUB_TYPE_PARSERS: Record unknown> = { - 16: (value) => value === "t", // bool - 20: toNumber, // int8 - 21: toNumber, // int2 - 23: toNumber, // int4 - 26: toNumber, // oid - 700: toNumber, // float4 - 701: toNumber, // float8 - 1700: toNumber, // numeric (SUM of int8) - 114: (value) => JSON.parse(value), // json - 3802: (value) => JSON.parse(value), // jsonb -}; - -export const hubPgTypes = { - getTypeParser: (oid: number, format?: string) => - format === "binary" ? identity : (HUB_TYPE_PARSERS[oid] ?? identity), -} as unknown as NonNullable; - -export const makeHubPgClientLayer = (config: HubPostgresConfig) => - PgClient.layer({ - url: Redacted.make(config.url), - maxConnections: config.maxConnections ?? 10, - applicationName: config.applicationName ?? "t3-hub", - types: hubPgTypes, - }); - -// Arbitrary constant; serializes concurrent hub processes applying the baseline. -const HUB_SCHEMA_LOCK_KEY = 7_330_001; - -/** - * Applies the baseline idempotently. Transaction-scoped advisory locks are safe - * behind PgBouncer transaction pooling; session-level locks are not. - */ -export const ensureHubSchema = Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - yield* sql.withTransaction( - Effect.gen(function* () { - yield* sql`SELECT pg_advisory_xact_lock(${HUB_SCHEMA_LOCK_KEY})`; - for (const statement of HUB_BASELINE_STATEMENTS) { - yield* sql.unsafe(statement); - } - yield* sql` - INSERT INTO hub_schema_version (version) - VALUES (${HUB_SCHEMA_VERSION}) - ON CONFLICT (version) DO NOTHING - `; - }), - ); -}); - -/** Shared pool plus schema, provided once per hub process. */ -export const makeHubPostgresPersistenceLive = (config: HubPostgresConfig) => - Layer.provideMerge(Layer.effectDiscard(ensureHubSchema), makeHubPgClientLayer(config)); diff --git a/apps/server/src/persistence/Layers/ProjectionCheckpoints.ts b/apps/server/src/persistence/Layers/ProjectionCheckpoints.ts index cfa5658d0e9c..95c66cf656af 100644 --- a/apps/server/src/persistence/Layers/ProjectionCheckpoints.ts +++ b/apps/server/src/persistence/Layers/ProjectionCheckpoints.ts @@ -16,6 +16,8 @@ import { ProjectionCheckpointRepository, type ProjectionCheckpointRepositoryShape, } from "../Services/ProjectionCheckpoints.ts"; +import { localOrHub } from "../Postgres/HubDatabase.ts"; +import { PgProjectionCheckpointRepositoryLive } from "../Postgres/ProjectionCheckpoints.ts"; const ProjectionCheckpointDbRowSchema = ProjectionCheckpoint.mapFields( Struct.assign({ @@ -211,7 +213,7 @@ const makeProjectionCheckpointRepository = Effect.gen(function* () { } satisfies ProjectionCheckpointRepositoryShape; }); -export const ProjectionCheckpointRepositoryLive = Layer.effect( - ProjectionCheckpointRepository, - makeProjectionCheckpointRepository, +export const ProjectionCheckpointRepositoryLive = localOrHub( + Layer.effect(ProjectionCheckpointRepository, makeProjectionCheckpointRepository), + PgProjectionCheckpointRepositoryLive, ); diff --git a/apps/server/src/persistence/Layers/ProjectionPendingApprovals.ts b/apps/server/src/persistence/Layers/ProjectionPendingApprovals.ts index 3b159a9e1715..6c3cf4a56b6b 100644 --- a/apps/server/src/persistence/Layers/ProjectionPendingApprovals.ts +++ b/apps/server/src/persistence/Layers/ProjectionPendingApprovals.ts @@ -12,6 +12,8 @@ import { ProjectionPendingApprovalRepository, type ProjectionPendingApprovalRepositoryShape, } from "../Services/ProjectionPendingApprovals.ts"; +import { localOrHub } from "../Postgres/HubDatabase.ts"; +import { PgProjectionPendingApprovalRepositoryLive } from "../Postgres/ProjectionPendingApprovals.ts"; const makeProjectionPendingApprovalRepository = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -148,7 +150,7 @@ const makeProjectionPendingApprovalRepository = Effect.gen(function* () { } satisfies ProjectionPendingApprovalRepositoryShape; }); -export const ProjectionPendingApprovalRepositoryLive = Layer.effect( - ProjectionPendingApprovalRepository, - makeProjectionPendingApprovalRepository, +export const ProjectionPendingApprovalRepositoryLive = localOrHub( + Layer.effect(ProjectionPendingApprovalRepository, makeProjectionPendingApprovalRepository), + PgProjectionPendingApprovalRepositoryLive, ); diff --git a/apps/server/src/persistence/Layers/ProjectionProjects.ts b/apps/server/src/persistence/Layers/ProjectionProjects.ts index ba133bb24a47..e2a724d4db9d 100644 --- a/apps/server/src/persistence/Layers/ProjectionProjects.ts +++ b/apps/server/src/persistence/Layers/ProjectionProjects.ts @@ -14,6 +14,8 @@ import { ProjectionProjectRepository, type ProjectionProjectRepositoryShape, } from "../Services/ProjectionProjects.ts"; +import { localOrHub } from "../Postgres/HubDatabase.ts"; +import { PgProjectionProjectRepositoryLive } from "../Postgres/ProjectionProjects.ts"; const ProjectionProjectDbRow = ProjectionProject.mapFields( Struct.assign({ @@ -147,7 +149,7 @@ const makeProjectionProjectRepository = Effect.gen(function* () { } satisfies ProjectionProjectRepositoryShape; }); -export const ProjectionProjectRepositoryLive = Layer.effect( - ProjectionProjectRepository, - makeProjectionProjectRepository, +export const ProjectionProjectRepositoryLive = localOrHub( + Layer.effect(ProjectionProjectRepository, makeProjectionProjectRepository), + PgProjectionProjectRepositoryLive, ); diff --git a/apps/server/src/persistence/Layers/ProjectionState.ts b/apps/server/src/persistence/Layers/ProjectionState.ts index f630fe51e03c..5ea7e09d3cc6 100644 --- a/apps/server/src/persistence/Layers/ProjectionState.ts +++ b/apps/server/src/persistence/Layers/ProjectionState.ts @@ -13,6 +13,8 @@ import { GetProjectionStateInput, ProjectionState, } from "../Services/ProjectionState.ts"; +import { localOrHub } from "../Postgres/HubDatabase.ts"; +import { PgProjectionStateRepositoryLive } from "../Postgres/ProjectionState.ts"; const MinLastAppliedSequenceRowSchema = Schema.Struct({ minLastAppliedSequence: Schema.NullOr(NonNegativeInt), @@ -112,7 +114,7 @@ const makeProjectionStateRepository = Effect.gen(function* () { } satisfies ProjectionStateRepositoryShape; }); -export const ProjectionStateRepositoryLive = Layer.effect( - ProjectionStateRepository, - makeProjectionStateRepository, +export const ProjectionStateRepositoryLive = localOrHub( + Layer.effect(ProjectionStateRepository, makeProjectionStateRepository), + PgProjectionStateRepositoryLive, ); diff --git a/apps/server/src/persistence/Layers/ProjectionThreadActivities.ts b/apps/server/src/persistence/Layers/ProjectionThreadActivities.ts index fa3c948e4f3d..e9a23251ba51 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreadActivities.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreadActivities.ts @@ -15,6 +15,8 @@ import { ProjectionThreadActivityRepository, type ProjectionThreadActivityRepositoryShape, } from "../Services/ProjectionThreadActivities.ts"; +import { localOrHub } from "../Postgres/HubDatabase.ts"; +import { PgProjectionThreadActivityRepositoryLive } from "../Postgres/ProjectionThreadActivities.ts"; const ProjectionThreadActivityDbRowSchema = ProjectionThreadActivity.mapFields( Struct.assign({ @@ -199,7 +201,7 @@ const makeProjectionThreadActivityRepository = Effect.gen(function* () { } satisfies ProjectionThreadActivityRepositoryShape; }); -export const ProjectionThreadActivityRepositoryLive = Layer.effect( - ProjectionThreadActivityRepository, - makeProjectionThreadActivityRepository, +export const ProjectionThreadActivityRepositoryLive = localOrHub( + Layer.effect(ProjectionThreadActivityRepository, makeProjectionThreadActivityRepository), + PgProjectionThreadActivityRepositoryLive, ); diff --git a/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts b/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts index 85e854dc6606..531186e6ef50 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreadMessages.ts @@ -17,6 +17,8 @@ import { ListProjectionThreadMessagesInput, ProjectionThreadMessage, } from "../Services/ProjectionThreadMessages.ts"; +import { localOrHub } from "../Postgres/HubDatabase.ts"; +import { PgProjectionThreadMessageRepositoryLive } from "../Postgres/ProjectionThreadMessages.ts"; const ProjectionThreadMessageDbRowSchema = ProjectionThreadMessage.mapFields( Struct.assign({ @@ -235,7 +237,7 @@ const makeProjectionThreadMessageRepository = Effect.gen(function* () { } satisfies ProjectionThreadMessageRepositoryShape; }); -export const ProjectionThreadMessageRepositoryLive = Layer.effect( - ProjectionThreadMessageRepository, - makeProjectionThreadMessageRepository, +export const ProjectionThreadMessageRepositoryLive = localOrHub( + Layer.effect(ProjectionThreadMessageRepository, makeProjectionThreadMessageRepository), + PgProjectionThreadMessageRepositoryLive, ); diff --git a/apps/server/src/persistence/Layers/ProjectionThreadProposedPlans.ts b/apps/server/src/persistence/Layers/ProjectionThreadProposedPlans.ts index 63aed1a16704..2d7bdf6675b9 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreadProposedPlans.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreadProposedPlans.ts @@ -11,6 +11,8 @@ import { ProjectionThreadProposedPlanRepository, type ProjectionThreadProposedPlanRepositoryShape, } from "../Services/ProjectionThreadProposedPlans.ts"; +import { localOrHub } from "../Postgres/HubDatabase.ts"; +import { PgProjectionThreadProposedPlanRepositoryLive } from "../Postgres/ProjectionThreadProposedPlans.ts"; const makeProjectionThreadProposedPlanRepository = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -105,7 +107,7 @@ const makeProjectionThreadProposedPlanRepository = Effect.gen(function* () { } satisfies ProjectionThreadProposedPlanRepositoryShape; }); -export const ProjectionThreadProposedPlanRepositoryLive = Layer.effect( - ProjectionThreadProposedPlanRepository, - makeProjectionThreadProposedPlanRepository, +export const ProjectionThreadProposedPlanRepositoryLive = localOrHub( + Layer.effect(ProjectionThreadProposedPlanRepository, makeProjectionThreadProposedPlanRepository), + PgProjectionThreadProposedPlanRepositoryLive, ); diff --git a/apps/server/src/persistence/Layers/ProjectionThreadSessions.ts b/apps/server/src/persistence/Layers/ProjectionThreadSessions.ts index dcb750983a00..c0f2386fcf91 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreadSessions.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreadSessions.ts @@ -12,6 +12,8 @@ import { DeleteProjectionThreadSessionInput, GetProjectionThreadSessionInput, } from "../Services/ProjectionThreadSessions.ts"; +import { localOrHub } from "../Postgres/HubDatabase.ts"; +import { PgProjectionThreadSessionRepositoryLive } from "../Postgres/ProjectionThreadSessions.ts"; const makeProjectionThreadSessionRepository = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -106,7 +108,7 @@ const makeProjectionThreadSessionRepository = Effect.gen(function* () { } satisfies ProjectionThreadSessionRepositoryShape; }); -export const ProjectionThreadSessionRepositoryLive = Layer.effect( - ProjectionThreadSessionRepository, - makeProjectionThreadSessionRepository, +export const ProjectionThreadSessionRepositoryLive = localOrHub( + Layer.effect(ProjectionThreadSessionRepository, makeProjectionThreadSessionRepository), + PgProjectionThreadSessionRepositoryLive, ); diff --git a/apps/server/src/persistence/Layers/ProjectionThreads.ts b/apps/server/src/persistence/Layers/ProjectionThreads.ts index d5653a2c8b42..81e401ac7369 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreads.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreads.ts @@ -15,6 +15,8 @@ import { type ProjectionThreadRepositoryShape, } from "../Services/ProjectionThreads.ts"; import { ModelSelection, ThreadLinkedPullRequest } from "@t3tools/contracts"; +import { localOrHub } from "../Postgres/HubDatabase.ts"; +import { PgProjectionThreadRepositoryLive } from "../Postgres/ProjectionThreads.ts"; const ProjectionThreadDbRow = ProjectionThread.mapFields( Struct.assign({ @@ -234,7 +236,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { } satisfies ProjectionThreadRepositoryShape; }); -export const ProjectionThreadRepositoryLive = Layer.effect( - ProjectionThreadRepository, - makeProjectionThreadRepository, +export const ProjectionThreadRepositoryLive = localOrHub( + Layer.effect(ProjectionThreadRepository, makeProjectionThreadRepository), + PgProjectionThreadRepositoryLive, ); diff --git a/apps/server/src/persistence/Layers/ProjectionTurns.ts b/apps/server/src/persistence/Layers/ProjectionTurns.ts index 3f9dd97494cc..dbf3aa129e16 100644 --- a/apps/server/src/persistence/Layers/ProjectionTurns.ts +++ b/apps/server/src/persistence/Layers/ProjectionTurns.ts @@ -20,6 +20,8 @@ import { ProjectionTurnRepository, type ProjectionTurnRepositoryShape, } from "../Services/ProjectionTurns.ts"; +import { localOrHub } from "../Postgres/HubDatabase.ts"; +import { PgProjectionTurnRepositoryLive } from "../Postgres/ProjectionTurns.ts"; const ProjectionTurnDbRowSchema = ProjectionTurn.mapFields( Struct.assign({ @@ -380,7 +382,7 @@ const makeProjectionTurnRepository = Effect.gen(function* () { } satisfies ProjectionTurnRepositoryShape; }); -export const ProjectionTurnRepositoryLive = Layer.effect( - ProjectionTurnRepository, - makeProjectionTurnRepository, +export const ProjectionTurnRepositoryLive = localOrHub( + Layer.effect(ProjectionTurnRepository, makeProjectionTurnRepository), + PgProjectionTurnRepositoryLive, ); diff --git a/apps/server/src/persistence/Layers/Sqlite.ts b/apps/server/src/persistence/Layers/Sqlite.ts index ec1ffdefac0f..d59201588d90 100644 --- a/apps/server/src/persistence/Layers/Sqlite.ts +++ b/apps/server/src/persistence/Layers/Sqlite.ts @@ -7,6 +7,7 @@ import type { SqlError } from "effect/unstable/sql/SqlError"; import { runMigrations } from "../Migrations.ts"; import { ServerConfig } from "../../config.ts"; +import { HubDatabase } from "../Postgres/HubDatabase.ts"; type RuntimeSqliteLayerConfig = { readonly filename: string; @@ -65,9 +66,23 @@ export const SqlitePersistenceMemory = Layer.provideMerge( makeRuntimeSqliteLayer({ filename: ":memory:" }), ); +/** + * The server's persistence: `state.sqlite` in its state directory, or, in hub + * mode, the hub's tenant-scoped Postgres client (see `HubDatabase`). + */ export const layerConfig = Layer.unwrap( Effect.gen(function* () { - const { dbPath } = yield* ServerConfig; + const hubDatabase = yield* HubDatabase; + if (hubDatabase !== undefined) { + return Layer.succeed(SqlClient.SqlClient, hubDatabase.sql); + } + const { dbPath, hub } = yield* ServerConfig; + if (hub?.databaseUrl) { + // Never fall back to a local SQLite file in hub mode. + return yield* Effect.die( + new Error("Hub persistence is not available: provide HubDatabase.layerConfig at the root."), + ); + } return makeSqlitePersistenceLive(dbPath); }), ); diff --git a/apps/server/src/persistence/Postgres/AuthPairingLinks.ts b/apps/server/src/persistence/Postgres/AuthPairingLinks.ts new file mode 100644 index 000000000000..a9e7c9d254a2 --- /dev/null +++ b/apps/server/src/persistence/Postgres/AuthPairingLinks.ts @@ -0,0 +1,346 @@ +/** + * Postgres port of `AuthPairingLinkRepository` for the hub (see + * `../AuthPairingLinks.ts`). Every statement is scoped to the process tenant. + * Keep the SQL in step with the SQLite repository; the differences are the + * `user_id` predicates. + */ +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; + +import { AuthEnvironmentScopes } from "@t3tools/contracts"; + +import { + type AuthPairingLinkRepositoryError, + PersistenceDecodeError, + type PersistenceErrorCorrelation, + PersistenceSqlError, +} from "../Errors.ts"; +import type { AuthPairingLinkRepository } from "../AuthPairingLinks.ts"; +import { HubTenant } from "./HubTenant.ts"; + +const AuthPairingLinkRecord = Schema.Struct({ + id: Schema.String, + credential: Schema.String, + method: Schema.Literals(["desktop-bootstrap", "one-time-token"]), + scopes: Schema.fromJsonString(AuthEnvironmentScopes), + subject: Schema.String, + label: Schema.NullOr(Schema.String), + proofKeyThumbprint: Schema.NullOr(Schema.String), + createdAt: Schema.DateTimeUtcFromString, + expiresAt: Schema.DateTimeUtcFromString, + consumedAt: Schema.NullOr(Schema.DateTimeUtcFromString), + revokedAt: Schema.NullOr(Schema.DateTimeUtcFromString), +}); +type AuthPairingLinkRecord = typeof AuthPairingLinkRecord.Type; + +const CreateAuthPairingLinkInput = Schema.Struct({ + id: Schema.String, + credential: Schema.String, + method: Schema.Literals(["desktop-bootstrap", "one-time-token"]), + scopes: AuthEnvironmentScopes, + subject: Schema.String, + label: Schema.NullOr(Schema.String), + proofKeyThumbprint: Schema.NullOr(Schema.String), + createdAt: Schema.DateTimeUtcFromString, + expiresAt: Schema.DateTimeUtcFromString, +}); +type CreateAuthPairingLinkInput = typeof CreateAuthPairingLinkInput.Type; + +const ConsumeAuthPairingLinkInput = Schema.Struct({ + credential: Schema.String, + proofKeyThumbprint: Schema.NullOr(Schema.String), + consumedAt: Schema.DateTimeUtcFromString, + now: Schema.DateTimeUtcFromString, +}); +type ConsumeAuthPairingLinkInput = typeof ConsumeAuthPairingLinkInput.Type; + +const ListActiveAuthPairingLinksInput = Schema.Struct({ + now: Schema.DateTimeUtcFromString, +}); +type ListActiveAuthPairingLinksInput = typeof ListActiveAuthPairingLinksInput.Type; + +const RevokeAuthPairingLinkInput = Schema.Struct({ + id: Schema.String, + revokedAt: Schema.DateTimeUtcFromString, +}); +type RevokeAuthPairingLinkInput = typeof RevokeAuthPairingLinkInput.Type; + +const GetAuthPairingLinkByCredentialInput = Schema.Struct({ + credential: Schema.String, +}); +type GetAuthPairingLinkByCredentialInput = typeof GetAuthPairingLinkByCredentialInput.Type; + +const AuthPairingLinkRawDbRow = Schema.Struct({ + id: Schema.String, + credential: Schema.Unknown, + method: Schema.Unknown, + scopes: Schema.Unknown, + subject: Schema.Unknown, + label: Schema.Unknown, + proofKeyThumbprint: Schema.Unknown, + createdAt: Schema.Unknown, + expiresAt: Schema.Unknown, + consumedAt: Schema.Unknown, + revokedAt: Schema.Unknown, +}); + +const decodeAuthPairingLinkDbRow = Schema.decodeUnknownEffect(AuthPairingLinkRecord); + +function toPersistenceSqlOrDecodeError( + sqlOperation: string, + decodeOperation: string, + correlation?: PersistenceErrorCorrelation, +) { + return (cause: unknown): AuthPairingLinkRepositoryError => + Schema.isSchemaError(cause) + ? PersistenceDecodeError.fromSchemaError(decodeOperation, cause, correlation) + : new PersistenceSqlError({ + operation: sqlOperation, + ...(correlation === undefined ? {} : { correlation }), + cause, + }); +} + +export const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const createPairingLinkRow = SqlSchema.void({ + Request: CreateAuthPairingLinkInput, + execute: (input) => + sql` + INSERT INTO auth_pairing_links ( + user_id, + id, + credential, + method, + scopes, + subject, + label, + proof_key_thumbprint, + created_at, + expires_at, + consumed_at, + revoked_at + ) + VALUES ( + ${userId}, + ${input.id}, + ${input.credential}, + ${input.method}, + ${JSON.stringify(input.scopes)}, + ${input.subject}, + ${input.label}, + ${input.proofKeyThumbprint}, + ${input.createdAt}, + ${input.expiresAt}, + NULL, + NULL + ) + `, + }); + + const consumeAvailablePairingLinkRow = SqlSchema.findOneOption({ + Request: ConsumeAuthPairingLinkInput, + Result: AuthPairingLinkRawDbRow, + execute: ({ credential, proofKeyThumbprint, consumedAt, now }) => + sql` + UPDATE auth_pairing_links + SET consumed_at = ${consumedAt} + WHERE user_id = ${userId} + AND credential = ${credential} + AND revoked_at IS NULL + AND consumed_at IS NULL + AND expires_at > ${now} + AND ( + proof_key_thumbprint IS NULL + OR proof_key_thumbprint = ${proofKeyThumbprint} + ) + RETURNING + id AS "id", + credential AS "credential", + method AS "method", + scopes AS "scopes", + subject AS "subject", + label AS "label", + proof_key_thumbprint AS "proofKeyThumbprint", + created_at AS "createdAt", + expires_at AS "expiresAt", + consumed_at AS "consumedAt", + revoked_at AS "revokedAt" + `, + }); + + const listActivePairingLinkRows = SqlSchema.findAll({ + Request: ListActiveAuthPairingLinksInput, + Result: AuthPairingLinkRawDbRow, + execute: ({ now }) => + sql` + SELECT + id AS "id", + credential AS "credential", + method AS "method", + scopes AS "scopes", + subject AS "subject", + label AS "label", + proof_key_thumbprint AS "proofKeyThumbprint", + created_at AS "createdAt", + expires_at AS "expiresAt", + consumed_at AS "consumedAt", + revoked_at AS "revokedAt" + FROM auth_pairing_links + WHERE user_id = ${userId} + AND revoked_at IS NULL + AND consumed_at IS NULL + AND expires_at > ${now} + ORDER BY created_at DESC, id DESC + `, + }); + + const revokePairingLinkRow = SqlSchema.findAll({ + Request: RevokeAuthPairingLinkInput, + Result: Schema.Struct({ id: Schema.String }), + execute: ({ id, revokedAt }) => + sql` + UPDATE auth_pairing_links + SET revoked_at = ${revokedAt} + WHERE user_id = ${userId} + AND id = ${id} + AND revoked_at IS NULL + AND consumed_at IS NULL + RETURNING id AS "id" + `, + }); + + const getPairingLinkRowByCredential = SqlSchema.findOneOption({ + Request: GetAuthPairingLinkByCredentialInput, + Result: AuthPairingLinkRawDbRow, + execute: ({ credential }) => + sql` + SELECT + id AS "id", + credential AS "credential", + method AS "method", + scopes AS "scopes", + subject AS "subject", + label AS "label", + proof_key_thumbprint AS "proofKeyThumbprint", + created_at AS "createdAt", + expires_at AS "expiresAt", + consumed_at AS "consumedAt", + revoked_at AS "revokedAt" + FROM auth_pairing_links + WHERE user_id = ${userId} + AND credential = ${credential} + `, + }); + + const create: AuthPairingLinkRepository["Service"]["create"] = (input) => + createPairingLinkRow(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "AuthPairingLinkRepository.create:query", + "AuthPairingLinkRepository.create:encodeRequest", + { pairingLinkId: input.id }, + ), + ), + ); + + const consumeAvailable: AuthPairingLinkRepository["Service"]["consumeAvailable"] = (input) => + consumeAvailablePairingLinkRow(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "AuthPairingLinkRepository.consumeAvailable:query", + "AuthPairingLinkRepository.consumeAvailable:decodeRow", + ), + ), + Effect.flatMap((rowOption) => + Option.match(rowOption, { + onNone: () => Effect.succeed(Option.none()), + onSome: (row) => + decodeAuthPairingLinkDbRow(row).pipe( + Effect.mapError((cause) => + PersistenceDecodeError.fromSchemaError( + "AuthPairingLinkRepository.consumeAvailable:decodeRow", + cause, + { pairingLinkId: row.id }, + ), + ), + Effect.map(Option.some), + ), + }), + ), + ); + + const listActive: AuthPairingLinkRepository["Service"]["listActive"] = (input) => + listActivePairingLinkRows(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "AuthPairingLinkRepository.listActive:query", + "AuthPairingLinkRepository.listActive:decodeRows", + ), + ), + Effect.flatMap((rows) => + Effect.forEach(rows, (row) => + decodeAuthPairingLinkDbRow(row).pipe( + Effect.mapError((cause) => + PersistenceDecodeError.fromSchemaError( + "AuthPairingLinkRepository.listActive:decodeRows", + cause, + { pairingLinkId: row.id }, + ), + ), + ), + ), + ), + ); + + const revoke: AuthPairingLinkRepository["Service"]["revoke"] = (input) => + revokePairingLinkRow(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "AuthPairingLinkRepository.revoke:query", + "AuthPairingLinkRepository.revoke:decodeRows", + { pairingLinkId: input.id }, + ), + ), + Effect.map((rows) => rows.length > 0), + ); + + const getByCredential: AuthPairingLinkRepository["Service"]["getByCredential"] = (input) => + getPairingLinkRowByCredential(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "AuthPairingLinkRepository.getByCredential:query", + "AuthPairingLinkRepository.getByCredential:decodeRow", + ), + ), + Effect.flatMap((rowOption) => + Option.match(rowOption, { + onNone: () => Effect.succeed(Option.none()), + onSome: (row) => + decodeAuthPairingLinkDbRow(row).pipe( + Effect.mapError((cause) => + PersistenceDecodeError.fromSchemaError( + "AuthPairingLinkRepository.getByCredential:decodeRow", + cause, + { pairingLinkId: row.id }, + ), + ), + Effect.map(Option.some), + ), + }), + ), + ); + + return { + create, + consumeAvailable, + listActive, + revoke, + getByCredential, + } satisfies AuthPairingLinkRepository["Service"]; +}); diff --git a/apps/server/src/persistence/Postgres/AuthSessions.ts b/apps/server/src/persistence/Postgres/AuthSessions.ts new file mode 100644 index 000000000000..44c0ec709c56 --- /dev/null +++ b/apps/server/src/persistence/Postgres/AuthSessions.ts @@ -0,0 +1,439 @@ +/** + * Postgres port of `AuthSessionRepository` for the hub (see `../AuthSessions.ts`). + * Every statement is scoped to the process tenant. Keep the SQL in step with + * the SQLite repository; the differences are the `user_id` predicates. + */ +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; + +import { + AuthClientMetadataDeviceType, + AuthEnvironmentScopes, + AuthSessionId, + ClientSurface, + ServerAuthSessionMethod, +} from "@t3tools/contracts"; + +import { + type AuthSessionRepositoryError, + PersistenceDecodeError, + type PersistenceErrorCorrelation, + PersistenceSqlError, +} from "../Errors.ts"; +import type { AuthSessionRepository } from "../AuthSessions.ts"; +import { HubTenant } from "./HubTenant.ts"; + +const AuthSessionClientMetadataRecord = Schema.Struct({ + label: Schema.NullOr(Schema.String), + ipAddress: Schema.NullOr(Schema.String), + userAgent: Schema.NullOr(Schema.String), + deviceType: AuthClientMetadataDeviceType, + os: Schema.NullOr(Schema.String), + browser: Schema.NullOr(Schema.String), +}); +type AuthSessionClientMetadataRecord = typeof AuthSessionClientMetadataRecord.Type; + +const AuthSessionRecord = Schema.Struct({ + sessionId: AuthSessionId, + subject: Schema.String, + scopes: AuthEnvironmentScopes, + method: ServerAuthSessionMethod, + client: AuthSessionClientMetadataRecord, + issuedAt: Schema.DateTimeUtcFromString, + expiresAt: Schema.DateTimeUtcFromString, + lastConnectedAt: Schema.NullOr(Schema.DateTimeUtcFromString), + revokedAt: Schema.NullOr(Schema.DateTimeUtcFromString), +}); +type AuthSessionRecord = typeof AuthSessionRecord.Type; + +const CreateAuthSessionInput = Schema.Struct({ + sessionId: AuthSessionId, + subject: Schema.String, + scopes: AuthEnvironmentScopes, + method: ServerAuthSessionMethod, + client: AuthSessionClientMetadataRecord, + issuedAt: Schema.DateTimeUtcFromString, + expiresAt: Schema.DateTimeUtcFromString, +}); +type CreateAuthSessionInput = typeof CreateAuthSessionInput.Type; + +const GetAuthSessionByIdInput = Schema.Struct({ + sessionId: AuthSessionId, +}); +type GetAuthSessionByIdInput = typeof GetAuthSessionByIdInput.Type; + +const ListActiveAuthSessionsInput = Schema.Struct({ + now: Schema.DateTimeUtcFromString, +}); +type ListActiveAuthSessionsInput = typeof ListActiveAuthSessionsInput.Type; + +const RevokeAuthSessionInput = Schema.Struct({ + sessionId: AuthSessionId, + revokedAt: Schema.DateTimeUtcFromString, +}); +type RevokeAuthSessionInput = typeof RevokeAuthSessionInput.Type; + +const RevokeOtherAuthSessionsInput = Schema.Struct({ + currentSessionId: AuthSessionId, + revokedAt: Schema.DateTimeUtcFromString, +}); +type RevokeOtherAuthSessionsInput = typeof RevokeOtherAuthSessionsInput.Type; + +const SetAuthSessionLastConnectedAtInput = Schema.Struct({ + sessionId: AuthSessionId, + lastConnectedAt: Schema.DateTimeUtcFromString, +}); +type SetAuthSessionLastConnectedAtInput = typeof SetAuthSessionLastConnectedAtInput.Type; + +const SetAuthSessionClientConnectionInput = Schema.Struct({ + sessionId: AuthSessionId, + surface: Schema.NullOr(ClientSurface), + appVersion: Schema.NullOr(Schema.String), +}); +type SetAuthSessionClientConnectionInput = typeof SetAuthSessionClientConnectionInput.Type; + +const AuthSessionDbRow = Schema.Struct({ + sessionId: AuthSessionId, + subject: Schema.String, + scopes: Schema.fromJsonString(AuthEnvironmentScopes), + method: ServerAuthSessionMethod, + clientLabel: Schema.NullOr(Schema.String), + clientIpAddress: Schema.NullOr(Schema.String), + clientUserAgent: Schema.NullOr(Schema.String), + clientDeviceType: Schema.Literals(["desktop", "mobile", "tablet", "bot", "unknown"]), + clientOs: Schema.NullOr(Schema.String), + clientBrowser: Schema.NullOr(Schema.String), + issuedAt: Schema.DateTimeUtcFromString, + expiresAt: Schema.DateTimeUtcFromString, + lastConnectedAt: Schema.NullOr(Schema.DateTimeUtcFromString), + revokedAt: Schema.NullOr(Schema.DateTimeUtcFromString), +}); + +const AuthSessionRawDbRow = Schema.Struct({ + sessionId: Schema.String, + subject: Schema.Unknown, + scopes: Schema.Unknown, + method: Schema.Unknown, + clientLabel: Schema.Unknown, + clientIpAddress: Schema.Unknown, + clientUserAgent: Schema.Unknown, + clientDeviceType: Schema.Unknown, + clientOs: Schema.Unknown, + clientBrowser: Schema.Unknown, + issuedAt: Schema.Unknown, + expiresAt: Schema.Unknown, + lastConnectedAt: Schema.Unknown, + revokedAt: Schema.Unknown, +}); + +const decodeAuthSessionDbRow = Schema.decodeUnknownEffect(AuthSessionDbRow); + +function toAuthSessionRecord(row: typeof AuthSessionDbRow.Type): AuthSessionRecord { + return { + sessionId: row.sessionId, + subject: row.subject, + scopes: row.scopes, + method: row.method, + client: { + label: row.clientLabel, + ipAddress: row.clientIpAddress, + userAgent: row.clientUserAgent, + deviceType: row.clientDeviceType, + os: row.clientOs, + browser: row.clientBrowser, + }, + issuedAt: row.issuedAt, + expiresAt: row.expiresAt, + lastConnectedAt: row.lastConnectedAt, + revokedAt: row.revokedAt, + }; +} + +function toPersistenceSqlOrDecodeError( + sqlOperation: string, + decodeOperation: string, + correlation?: PersistenceErrorCorrelation, +) { + return (cause: unknown): AuthSessionRepositoryError => + Schema.isSchemaError(cause) + ? PersistenceDecodeError.fromSchemaError(decodeOperation, cause, correlation) + : new PersistenceSqlError({ + operation: sqlOperation, + ...(correlation === undefined ? {} : { correlation }), + cause, + }); +} + +export const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const createSessionRow = SqlSchema.void({ + Request: CreateAuthSessionInput, + execute: (input) => + sql` + INSERT INTO auth_sessions ( + user_id, + session_id, + subject, + scopes, + method, + client_label, + client_ip_address, + client_user_agent, + client_device_type, + client_os, + client_browser, + issued_at, + expires_at, + revoked_at + ) + VALUES ( + ${userId}, + ${input.sessionId}, + ${input.subject}, + ${JSON.stringify(input.scopes)}, + ${input.method}, + ${input.client.label}, + ${input.client.ipAddress}, + ${input.client.userAgent}, + ${input.client.deviceType}, + ${input.client.os}, + ${input.client.browser}, + ${input.issuedAt}, + ${input.expiresAt}, + NULL + ) + `, + }); + + const getSessionRowById = SqlSchema.findOneOption({ + Request: GetAuthSessionByIdInput, + Result: AuthSessionRawDbRow, + execute: ({ sessionId }) => + sql` + SELECT + session_id AS "sessionId", + subject AS "subject", + scopes AS "scopes", + method AS "method", + client_label AS "clientLabel", + client_ip_address AS "clientIpAddress", + client_user_agent AS "clientUserAgent", + client_device_type AS "clientDeviceType", + client_os AS "clientOs", + client_browser AS "clientBrowser", + issued_at AS "issuedAt", + expires_at AS "expiresAt", + last_connected_at AS "lastConnectedAt", + revoked_at AS "revokedAt" + FROM auth_sessions + WHERE user_id = ${userId} + AND session_id = ${sessionId} + `, + }); + + const listActiveSessionRows = SqlSchema.findAll({ + Request: ListActiveAuthSessionsInput, + Result: AuthSessionRawDbRow, + execute: ({ now }) => + sql` + SELECT + session_id AS "sessionId", + subject AS "subject", + scopes AS "scopes", + method AS "method", + client_label AS "clientLabel", + client_ip_address AS "clientIpAddress", + client_user_agent AS "clientUserAgent", + client_device_type AS "clientDeviceType", + client_os AS "clientOs", + client_browser AS "clientBrowser", + issued_at AS "issuedAt", + expires_at AS "expiresAt", + last_connected_at AS "lastConnectedAt", + revoked_at AS "revokedAt" + FROM auth_sessions + WHERE user_id = ${userId} + AND revoked_at IS NULL + AND expires_at > ${now} + ORDER BY issued_at DESC, session_id DESC + `, + }); + + const setLastConnectedAtRow = SqlSchema.void({ + Request: SetAuthSessionLastConnectedAtInput, + execute: ({ sessionId, lastConnectedAt }) => + sql` + UPDATE auth_sessions + SET last_connected_at = ${lastConnectedAt} + WHERE user_id = ${userId} + AND session_id = ${sessionId} + AND revoked_at IS NULL + `, + }); + + // COALESCE keeps the previous value when a client reports only one field, so + // a partial report never nulls out data a fuller client stored earlier. + const setClientConnectionRow = SqlSchema.void({ + Request: SetAuthSessionClientConnectionInput, + execute: ({ sessionId, surface, appVersion }) => + sql` + UPDATE auth_sessions + SET client_surface = COALESCE(${surface}, client_surface), + client_app_version = COALESCE(${appVersion}, client_app_version) + WHERE user_id = ${userId} + AND session_id = ${sessionId} + AND revoked_at IS NULL + `, + }); + + const revokeSessionRows = SqlSchema.findAll({ + Request: RevokeAuthSessionInput, + Result: Schema.Struct({ sessionId: AuthSessionId }), + execute: ({ sessionId, revokedAt }) => + sql` + UPDATE auth_sessions + SET revoked_at = ${revokedAt} + WHERE user_id = ${userId} + AND session_id = ${sessionId} + AND revoked_at IS NULL + RETURNING session_id AS "sessionId" + `, + }); + + const revokeOtherSessionRows = SqlSchema.findAll({ + Request: RevokeOtherAuthSessionsInput, + Result: Schema.Struct({ sessionId: AuthSessionId }), + execute: ({ currentSessionId, revokedAt }) => + sql` + UPDATE auth_sessions + SET revoked_at = ${revokedAt} + WHERE user_id = ${userId} + AND session_id <> ${currentSessionId} + AND revoked_at IS NULL + RETURNING session_id AS "sessionId" + `, + }); + + const create: AuthSessionRepository["Service"]["create"] = (input) => + createSessionRow(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "AuthSessionRepository.create:query", + "AuthSessionRepository.create:encodeRequest", + { sessionId: input.sessionId }, + ), + ), + ); + + const getById: AuthSessionRepository["Service"]["getById"] = (input) => + getSessionRowById(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "AuthSessionRepository.getById:query", + "AuthSessionRepository.getById:decodeRow", + { sessionId: input.sessionId }, + ), + ), + Effect.flatMap((rowOption) => + Option.match(rowOption, { + onNone: () => Effect.succeed(Option.none()), + onSome: (row) => + decodeAuthSessionDbRow(row).pipe( + Effect.mapError((cause) => + PersistenceDecodeError.fromSchemaError( + "AuthSessionRepository.getById:decodeRow", + cause, + { sessionId: input.sessionId }, + ), + ), + Effect.map((decodedRow) => Option.some(toAuthSessionRecord(decodedRow))), + ), + }), + ), + ); + + const listActive: AuthSessionRepository["Service"]["listActive"] = (input) => + listActiveSessionRows(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "AuthSessionRepository.listActive:query", + "AuthSessionRepository.listActive:decodeRows", + ), + ), + Effect.flatMap((rows) => + Effect.forEach(rows, (row) => + decodeAuthSessionDbRow(row).pipe( + Effect.mapError((cause) => + PersistenceDecodeError.fromSchemaError( + "AuthSessionRepository.listActive:decodeRows", + cause, + { sessionId: row.sessionId }, + ), + ), + Effect.map(toAuthSessionRecord), + ), + ), + ), + ); + + const revoke: AuthSessionRepository["Service"]["revoke"] = (input) => + revokeSessionRows(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "AuthSessionRepository.revoke:query", + "AuthSessionRepository.revoke:decodeRows", + { sessionId: input.sessionId }, + ), + ), + Effect.map((rows) => rows.length > 0), + ); + + const revokeAllExcept: AuthSessionRepository["Service"]["revokeAllExcept"] = (input) => + revokeOtherSessionRows(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "AuthSessionRepository.revokeAllExcept:query", + "AuthSessionRepository.revokeAllExcept:decodeRows", + { currentSessionId: input.currentSessionId }, + ), + ), + Effect.map((rows) => rows.map((row) => row.sessionId)), + ); + + const setLastConnectedAt: AuthSessionRepository["Service"]["setLastConnectedAt"] = (input) => + setLastConnectedAtRow(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "AuthSessionRepository.setLastConnectedAt:query", + "AuthSessionRepository.setLastConnectedAt:encodeRequest", + { sessionId: input.sessionId }, + ), + ), + ); + + const setClientConnection: AuthSessionRepository["Service"]["setClientConnection"] = (input) => + setClientConnectionRow(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "AuthSessionRepository.setClientConnection:query", + "AuthSessionRepository.setClientConnection:encodeRequest", + { sessionId: input.sessionId }, + ), + ), + ); + + return { + create, + getById, + listActive, + revoke, + revokeAllExcept, + setLastConnectedAt, + setClientConnection, + } satisfies AuthSessionRepository["Service"]; +}); diff --git a/apps/server/src/persistence/Postgres/HubClient.ts b/apps/server/src/persistence/Postgres/HubClient.ts new file mode 100644 index 000000000000..fe33794a6dee --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubClient.ts @@ -0,0 +1,164 @@ +/** + * Postgres clients for the hub. + * + * The runtime client applies the tenant to every unit of work so the + * row-level-security policies (see migration 001) can check it: transactions + * begin with `SET LOCAL hub.user_id = ''`, and statements issued outside + * a transaction run in their own tenant transaction. Nothing is set per + * session, so the client is safe behind a transaction-mode pooler (PgBouncer): + * the setting never outlives the transaction that carries it. + * + * `SET LOCAL` is a utility statement and takes no snapshot, so a transaction + * may still start with `SET TRANSACTION ISOLATION LEVEL ...` after it. + * + * @module HubClient + */ +import * as PgClient from "@effect/sql-pg/PgClient"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Redacted from "effect/Redacted"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import type * as SqlConnection from "effect/unstable/sql/SqlConnection"; +import type { SqlError } from "effect/unstable/sql/SqlError"; + +/** The transaction-local setting the row-level-security policies compare with `user_id`. */ +export const HUB_TENANT_SETTING = "hub.user_id"; + +// Aldo user ids are opaque but plain (UUIDs, `user_...`). Restricting the +// alphabet keeps the tenant safe to embed as a literal in `SET LOCAL`, which +// cannot take a bind parameter. +const HUB_TENANT_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:@-]{0,199}$/; + +export const isValidHubTenantId = (tenantId: string): boolean => + HUB_TENANT_ID_PATTERN.test(tenantId); + +export class HubTenantIdError extends Error { + constructor() { + super( + "T3CODE_HUB_TENANT_ID must be 1-200 characters of letters, digits, '_', '.', ':', '@', or '-'.", + ); + this.name = "HubTenantIdError"; + } +} + +export const hubTenantSettingStatement = (tenantId: string): string => { + if (!isValidHubTenantId(tenantId)) { + throw new HubTenantIdError(); + } + return `SET LOCAL ${HUB_TENANT_SETTING} = '${tenantId}'`; +}; + +// node-postgres returns int8/numeric as strings and bytea as hex text. Hub +// sequences, COUNT(*) and MAX(sequence) are int8, and the shared row schemas +// decode numbers, so parse them here. Values stay far below 2^53 (sequences +// are per user). +const identity = (value: string) => value; +const toNumber = (value: string) => Number(value); +const HUB_TYPE_PARSERS: Record unknown> = { + 16: (value) => value === "t", // bool + 17: (value) => Buffer.from(value.slice(2), "hex"), // bytea (\x...) + 20: toNumber, // int8 + 21: toNumber, // int2 + 23: toNumber, // int4 + 26: toNumber, // oid + 700: toNumber, // float4 + 701: toNumber, // float8 + 1700: toNumber, // numeric (SUM of int8) + 114: (value) => JSON.parse(value), // json + 3802: (value) => JSON.parse(value), // jsonb +}; + +export const hubPgTypes = { + getTypeParser: (oid: number, format?: string) => + format === "binary" ? identity : (HUB_TYPE_PARSERS[oid] ?? identity), +} as unknown as NonNullable; + +export interface HubPoolOptions { + readonly url: string; + readonly maxConnections?: number; + readonly applicationName?: string; +} + +/** A plain pooled client: migrations and administration only (no tenant). */ +export const makeHubPool = (options: HubPoolOptions) => + PgClient.make({ + url: Redacted.make(options.url), + // Many hub processes share one database, and PlanetScale's direct port + // allows few connections. One user's engine is a single writer, so a + // handful of connections is enough. + maxConnections: options.maxConnections ?? 4, + idleTimeout: "30 seconds", + connectTimeout: "10 seconds", + applicationName: options.applicationName ?? "t3-hub", + types: hubPgTypes, + }); + +const withTenantTransaction = ( + connection: SqlConnection.Connection, + begin: string, + effect: Effect.Effect, +) => + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + yield* connection.executeUnprepared(begin, [], undefined); + const exit = yield* Effect.exit(restore(effect)); + if (Exit.isSuccess(exit)) { + yield* connection.executeUnprepared("COMMIT", [], undefined); + return exit.value; + } + yield* connection.executeUnprepared("ROLLBACK", [], undefined).pipe(Effect.ignore); + return yield* exit; + }), + ); + +/** + * Wraps a pooled client so every statement and transaction carries the tenant. + * + * Transactions cost no extra round trip (`BEGIN` and `SET LOCAL` travel as one + * simple query). A statement outside a transaction costs two more (the tenant + * `BEGIN` and `COMMIT`), which is the price of the RLS backstop behind a pooler. + */ +export const makeTenantSqlClient = (base: PgClient.PgClient, tenantId: string) => + Effect.gen(function* () { + const setTenant = hubTenantSettingStatement(tenantId); + const begin = `BEGIN; ${setTenant}`; + + const autocommit = (connection: SqlConnection.Connection): SqlConnection.Connection => ({ + execute: (sql, params, transformRows) => + withTenantTransaction(connection, begin, connection.execute(sql, params, transformRows)), + executeRaw: (sql, params) => + withTenantTransaction(connection, begin, connection.executeRaw(sql, params)), + executeValues: (sql, params) => + withTenantTransaction(connection, begin, connection.executeValues(sql, params)), + executeValuesUnprepared: (sql, params) => + withTenantTransaction(connection, begin, connection.executeValuesUnprepared(sql, params)), + executeUnprepared: (sql, params, transformRows) => + withTenantTransaction( + connection, + begin, + connection.executeUnprepared(sql, params, transformRows), + ), + executeStream: (sql, params, transformRows) => + Stream.unwrap( + Effect.acquireRelease(connection.executeUnprepared(begin, [], undefined), (_, exit) => + connection + .executeUnprepared(Exit.isSuccess(exit) ? "COMMIT" : "ROLLBACK", [], undefined) + .pipe(Effect.ignore), + ).pipe(Effect.as(connection.executeStream(sql, params, transformRows))), + ), + }); + + return yield* SqlClient.make({ + // `reserve` pins one pooled connection for the statement, so the tenant + // transaction and the statement share it. + acquirer: Effect.map(base.reserve, autocommit), + transactionAcquirer: base.reserve, + compiler: PgClient.makeCompiler(), + spanAttributes: [ + ["db.system.name", "postgresql"], + ["db.namespace", base.config.database ?? base.config.username ?? "postgres"], + ], + beginTransaction: begin, + }); + }); diff --git a/apps/server/src/persistence/Postgres/HubDatabase.test.ts b/apps/server/src/persistence/Postgres/HubDatabase.test.ts new file mode 100644 index 000000000000..53d89b258c33 --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubDatabase.test.ts @@ -0,0 +1,185 @@ +/** + * Hub database foundation: migrations, the tenant-scoped client, and the + * row-level-security backstop. Runs only with T3_HUB_TEST_DATABASE_URL (see + * `hubTestDatabase.ts`). + */ +import { assert, describe, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Reactivity from "effect/unstable/reactivity/Reactivity"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { HubDatabase } from "./HubDatabase.ts"; +import { makeHubPool, makeTenantSqlClient } from "./HubClient.ts"; +import { orderHubMigrations, runHubMigrations, type HubMigrationEntry } from "./HubMigrator.ts"; +import { hubMigrations } from "./migrations/index.ts"; +import { + hubTestDatabaseLayer, + hubTestDatabaseUrl, + makeHubTestSchema, + type HubTestSchema, +} from "./hubTestDatabase.ts"; + +const withSchema = (use: (schema: HubTestSchema) => Effect.Effect) => + Effect.scoped(Effect.flatMap(makeHubTestSchema(hubTestDatabaseUrl!), use)).pipe( + Effect.provide(Reactivity.layer), + ); + +const tenantSetting = (sql: SqlClient.SqlClient) => + sql<{ readonly tenant: string | null }>` + SELECT current_setting('hub.user_id', true) AS tenant + `.pipe(Effect.map((rows) => rows[0]?.tenant ?? null)); + +describe.skipIf(hubTestDatabaseUrl === undefined)("hub database", () => { + it.effect("applies migrations once, in id order, under concurrent starts", () => + withSchema((schema) => + Effect.gen(function* () { + const applied = yield* Effect.all( + Array.from({ length: 4 }, () => + Effect.scoped( + Effect.gen(function* () { + const pool = yield* makeHubPool({ url: schema.adminUrl, maxConnections: 1 }); + return yield* runHubMigrations(hubMigrations).pipe( + Effect.provideService(SqlClient.SqlClient, pool), + ); + }), + ), + ), + { concurrency: "unbounded" }, + ); + assert.deepStrictEqual( + applied.flat().map((entry) => entry.id), + hubMigrations.map((entry) => entry.id), + ); + + // A later migration with a lower id than an applied one still applies. + const pool = yield* makeHubPool({ url: schema.adminUrl, maxConnections: 1 }); + const extra: ReadonlyArray = [ + ...hubMigrations, + { + id: 60, + name: "RunnerProbe", + migration: Effect.flatMap(SqlClient.SqlClient, (sql) => + sql`CREATE TABLE runner_probe (id integer)`.pipe(Effect.asVoid), + ), + }, + { + id: 49, + name: "HubProbe", + migration: Effect.flatMap(SqlClient.SqlClient, (sql) => + sql`CREATE TABLE hub_probe (id integer)`.pipe(Effect.asVoid), + ), + }, + ]; + const second = yield* runHubMigrations(extra).pipe( + Effect.provideService(SqlClient.SqlClient, pool), + ); + assert.deepStrictEqual( + second.map((entry) => entry.id), + [49, 60], + ); + const rerun = yield* runHubMigrations(extra).pipe( + Effect.provideService(SqlClient.SqlClient, pool), + ); + assert.deepStrictEqual(rerun, []); + }), + ), + ); + + it("rejects duplicate migration ids", () => { + const migration = Effect.void; + assert.throws(() => + orderHubMigrations([ + { id: 2, name: "A", migration }, + { id: 2, name: "B", migration }, + ]), + ); + }); + + it.effect("applies the tenant to statements and transactions", () => + withSchema((schema) => + Effect.gen(function* () { + const database = yield* HubDatabase; + assert.isDefined(database); + const sql = database!.sql; + + assert.strictEqual(yield* tenantSetting(sql), "tenant-a"); + assert.strictEqual(yield* sql.withTransaction(tenantSetting(sql)), "tenant-a"); + + // The tenant is set with SET LOCAL, so a snapshot transaction can still + // choose its isolation level first. + const isolation = yield* sql.withTransaction( + Effect.gen(function* () { + yield* sql`SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY`; + const rows = yield* sql<{ + readonly level: string; + }>`SELECT current_setting('transaction_isolation') AS level`; + return rows[0]?.level; + }), + ); + assert.strictEqual(isolation, "repeatable read"); + + // The setting never outlives its transaction on the pooled connection. + const plain = yield* makeHubPool({ url: schema.runtimeUrl, maxConnections: 1 }); + const leaked = yield* tenantSetting(plain); + assert.isTrue(leaked === null || leaked === ""); + }).pipe(Effect.provide(hubTestDatabaseLayer(schema, "tenant-a"))), + ), + ); + + it.effect("fails closed for a role without BYPASSRLS", () => + withSchema((schema) => + Effect.gen(function* () { + const database = yield* HubDatabase; + const tenantA = database!.sql; + const runtime = yield* makeHubPool({ url: schema.runtimeUrl, maxConnections: 2 }); + + const [role] = yield* runtime<{ + readonly rolsuper: boolean; + readonly rolbypassrls: boolean; + }>`SELECT rolsuper, rolbypassrls FROM pg_roles WHERE rolname = current_user`; + assert.isFalse(role?.rolsuper); + assert.isFalse(role?.rolbypassrls); + + yield* tenantA` + INSERT INTO hub_documents (user_id, name, contents) + VALUES ('tenant-a', 'settings.json', '{}') + `; + const tenantB = yield* makeTenantSqlClient(runtime, "tenant-b"); + + const count = (sql: SqlClient.SqlClient) => + sql<{ readonly count: number }>`SELECT count(*) AS count FROM hub_documents`.pipe( + Effect.map((rows) => rows[0]?.count ?? -1), + ); + assert.strictEqual(yield* count(tenantA), 1); + assert.strictEqual(yield* count(tenantB), 0); + // No tenant set: the policy compares with NULL and matches nothing. + assert.strictEqual(yield* count(runtime), 0); + + const forged = yield* tenantB` + INSERT INTO hub_documents (user_id, name, contents) + VALUES ('tenant-a', 'keybindings.json', '[]') + `.pipe(Effect.exit); + assert.isTrue(Exit.isFailure(forged)); + const untenanted = yield* runtime` + INSERT INTO hub_documents (user_id, name, contents) + VALUES ('tenant-a', 'anonymous-id', 'x') + `.pipe(Effect.exit); + assert.isTrue(Exit.isFailure(untenanted)); + + const updated = yield* tenantB<{ readonly name: string }>` + UPDATE hub_documents SET contents = 'stolen' RETURNING name + `; + assert.strictEqual(updated.length, 0); + const deleted = yield* tenantB<{ readonly name: string }>` + DELETE FROM hub_documents RETURNING name + `; + assert.strictEqual(deleted.length, 0); + const [row] = yield* tenantA<{ readonly contents: string }>` + SELECT contents FROM hub_documents WHERE user_id = 'tenant-a' + `; + assert.strictEqual(row?.contents, "{}"); + }).pipe(Effect.provide(hubTestDatabaseLayer(schema, "tenant-a"))), + ), + ); +}); diff --git a/apps/server/src/persistence/Postgres/HubDatabase.ts b/apps/server/src/persistence/Postgres/HubDatabase.ts new file mode 100644 index 000000000000..6e6cc3253eaf --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubDatabase.ts @@ -0,0 +1,90 @@ +/** + * HubDatabase - the tenant-scoped Postgres database of a hub process. + * + * Set only in hub mode (`T3CODE_SERVER_MODE=hub` with a hub database URL), where + * one process serves exactly one tenant and its base directory is disposable. + * Everything a standalone server keeps in SQLite or in state-directory files + * switches to Postgres when it is set: + * + * - repository layers pick their Postgres port through `localOrHub`; + * - the SQLite persistence layer hands out this client instead of opening + * `state.sqlite`; + * - file-backed stores (settings, keybindings, secrets, environment id, + * attachments) read it directly. + * + * It is a reference rather than a required service so that standalone layer + * types, tests, and behavior stay exactly as upstream T3 has them. The server + * and the auth CLI provide `layerConfig` once at their root. + * + * @module HubDatabase + */ +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { ServerConfig } from "../../config.ts"; +import { HubTenant } from "./HubTenant.ts"; + +export interface HubDatabaseShape { + /** The Aldo user this process serves. */ + readonly tenantId: string; + /** Applies the tenant to every statement and transaction (the RLS backstop). */ + readonly sql: SqlClient.SqlClient; +} + +export class HubDatabase extends Context.Reference( + "t3/persistence/Postgres/HubDatabase", + { defaultValue: () => undefined }, +) {} + +/** + * Hub mode: applies hub migrations and connects the tenant client. Otherwise + * provides nothing, so standalone composition is unchanged. + */ +export const layerConfig = Layer.unwrap( + Effect.gen(function* () { + const { hub } = yield* ServerConfig; + if (!hub?.databaseUrl) { + return Layer.empty; + } + // Loaded lazily so a standalone server never loads the Postgres driver. + const { makeHubDatabase } = yield* Effect.promise(() => import("./HubDatabaseLive.ts")); + return Layer.effect( + HubDatabase, + makeHubDatabase({ + databaseUrl: hub.databaseUrl, + databaseAdminUrl: hub.databaseAdminUrl, + tenantId: hub.tenantId, + }), + ); + }), +); + +/** + * Chooses a service implementation by persistence backend: `local` (SQLite or + * files) normally, `hub` when this process is a hub. The hub layer receives + * the tenant client as its `SqlClient` and the tenant as `HubTenant`. + */ +export const localOrHub = ( + local: Layer.Layer, + hub: Layer.Layer, +): Layer.Layer> => + Layer.unwrap( + Effect.gen(function* () { + const database = yield* HubDatabase; + const selected: Layer.Layer> = + database === undefined + ? local + : hub.pipe( + Layer.provide( + Layer.succeedContext( + Context.make(HubTenant, { userId: database.tenantId }).pipe( + Context.add(SqlClient.SqlClient, database.sql), + ), + ), + ), + ); + return selected; + }), + ); diff --git a/apps/server/src/persistence/Postgres/HubDatabaseLive.ts b/apps/server/src/persistence/Postgres/HubDatabaseLive.ts new file mode 100644 index 000000000000..d5d6d44d08e6 --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubDatabaseLive.ts @@ -0,0 +1,88 @@ +/** + * Builds a hub's database: applies hub migrations (with the admin URL when one + * is configured), grants the runtime role access, and connects the + * tenant-scoped runtime client. Loaded only in hub mode. + * + * @module HubDatabaseLive + */ +import * as Effect from "effect/Effect"; +import * as Reactivity from "effect/unstable/reactivity/Reactivity"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import type { HubDatabaseShape } from "./HubDatabase.ts"; +import { + HubTenantIdError, + isValidHubTenantId, + makeHubPool, + makeTenantSqlClient, +} from "./HubClient.ts"; +import { runHubMigrations } from "./HubMigrator.ts"; +import { hubMigrations } from "./migrations/index.ts"; + +export interface HubDatabaseOptions { + readonly databaseUrl: string; + readonly databaseAdminUrl?: string | undefined; + readonly tenantId: string; +} + +const quoteIdentifier = (identifier: string) => `"${identifier.replaceAll('"', '""')}"`; + +/** + * Lets a separate runtime role use the tables the admin role owns. Idempotent; + * a failure is logged rather than fatal because a platform may manage grants + * itself, and a missing grant still fails loudly on first use. + */ +const grantRuntimeRole = (admin: SqlClient.SqlClient, runtimeRole: string) => + Effect.gen(function* () { + const [row] = yield* admin<{ + readonly adminRole: string; + readonly schema: string; + }>`SELECT current_user AS "adminRole", current_schema() AS "schema"`; + if (!row || row.adminRole === runtimeRole) { + return; + } + const role = quoteIdentifier(runtimeRole); + const schema = quoteIdentifier(row.schema); + yield* admin.unsafe(`GRANT USAGE ON SCHEMA ${schema} TO ${role}`); + yield* admin.unsafe( + `GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA ${schema} TO ${role}`, + ); + yield* admin.unsafe(`GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA ${schema} TO ${role}`); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not grant the hub runtime role access to hub tables.", { + cause, + }), + ), + ); + +export const makeHubDatabase = (options: HubDatabaseOptions) => + Effect.gen(function* () { + if (!isValidHubTenantId(options.tenantId)) { + return yield* Effect.die(new HubTenantIdError()); + } + + const runtimePool = yield* makeHubPool({ url: options.databaseUrl }); + const [runtime] = yield* runtimePool<{ readonly role: string }>`SELECT current_user AS role`; + + yield* Effect.scoped( + Effect.gen(function* () { + const admin = options.databaseAdminUrl + ? yield* makeHubPool({ + url: options.databaseAdminUrl, + maxConnections: 1, + applicationName: "t3-hub-migrate", + }) + : runtimePool; + yield* runHubMigrations(hubMigrations).pipe( + Effect.provideService(SqlClient.SqlClient, admin), + ); + if (options.databaseAdminUrl && runtime) { + yield* grantRuntimeRole(admin, runtime.role); + } + }), + ); + + const sql = yield* makeTenantSqlClient(runtimePool, options.tenantId); + return { tenantId: options.tenantId, sql } satisfies HubDatabaseShape; + }).pipe(Effect.provide(Reactivity.layer)); diff --git a/apps/server/src/persistence/Postgres/HubEngine.test.ts b/apps/server/src/persistence/Postgres/HubEngine.test.ts deleted file mode 100644 index ba2771b81ec2..000000000000 --- a/apps/server/src/persistence/Postgres/HubEngine.test.ts +++ /dev/null @@ -1,429 +0,0 @@ -/** - * Hub prototype: two users' orchestration engines in one process over one - * Postgres database. Runs only when T3_HUB_TEST_DATABASE_URL points at a - * disposable database, e.g. - * T3_HUB_TEST_DATABASE_URL=postgres://postgres@127.0.0.1:54339/hub - */ -import { - CommandId, - DEFAULT_PROVIDER_INTERACTION_MODE, - EventId, - MessageId, - ProjectId, - ProviderInstanceId, - ThreadId, - type OrchestrationCommand, - type OrchestrationEvent, -} from "@t3tools/contracts"; -import * as NodeCrypto from "node:crypto"; - -import * as PgClient from "@effect/sql-pg/PgClient"; -import { assert, expect, it } from "@effect/vitest"; -import * as Context from "effect/Context"; -import * as Effect from "effect/Effect"; -import * as Exit from "effect/Exit"; -import * as Layer from "effect/Layer"; -import * as Option from "effect/Option"; -import * as Redacted from "effect/Redacted"; -import * as Scope from "effect/Scope"; -import * as Stream from "effect/Stream"; -import * as SqlClient from "effect/unstable/sql/SqlClient"; - -import { OrchestrationEngineService } from "../../orchestration/Services/OrchestrationEngine.ts"; -import { ProjectionSnapshotQuery } from "../../orchestration/Services/ProjectionSnapshotQuery.ts"; -import { hubPgTypes } from "../Layers/Postgres.ts"; -import { makeHubSharedLayer, makeHubUserEngineLayer, type HubSharedServices } from "./HubEngine.ts"; -import { HUB_TENANT_TABLES, hubRowLevelSecurityStatements } from "./Schema.ts"; - -const databaseUrl = process.env.T3_HUB_TEST_DATABASE_URL; -const runId = NodeCrypto.randomUUID().slice(0, 8); -const userA = `test-user-a-${runId}`; -const userB = `test-user-b-${runId}`; -const createdAt = "2026-09-25T00:00:00.000Z"; -const at = (second: number) => `2026-09-25T00:00:${String(second).padStart(2, "0")}.000Z`; -const modelSelection = { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5-codex" }; - -// Both users deliberately reuse the same project, thread, message, and command -// ids: isolation must come from the tenant key, not from id uniqueness. -const commandsFor = (label: string): ReadonlyArray => [ - { - type: "project.create", - commandId: CommandId.make("cmd-project-create"), - projectId: ProjectId.make("project-shared"), - title: `Project of ${label}`, - workspaceRoot: `/workspace/${label}`, - defaultModelSelection: modelSelection, - createdAt: at(1), - }, - { - type: "thread.create", - commandId: CommandId.make("cmd-thread-create"), - threadId: ThreadId.make("thread-shared"), - projectId: ProjectId.make("project-shared"), - title: `Thread of ${label}`, - modelSelection, - interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, - runtimeMode: "approval-required", - branch: null, - worktreePath: null, - createdAt: at(2), - }, - { - type: "thread.turn.start", - commandId: CommandId.make("cmd-turn-start"), - threadId: ThreadId.make("thread-shared"), - message: { - messageId: MessageId.make("msg-user-1"), - role: "user", - text: `hello from ${label}`, - attachments: [], - }, - interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, - runtimeMode: "approval-required", - createdAt: at(3), - }, - { - type: "thread.activity.append", - commandId: CommandId.make("cmd-activity-1"), - threadId: ThreadId.make("thread-shared"), - activity: { - id: EventId.make("activity-1"), - tone: "tool", - kind: "tool.completed", - summary: `tool ran for ${label}`, - payload: { owner: label }, - turnId: null, - createdAt: at(4), - }, - createdAt: at(5), - }, - { - type: "thread.message.assistant.delta", - commandId: CommandId.make("cmd-assistant-delta-1"), - threadId: ThreadId.make("thread-shared"), - messageId: MessageId.make("msg-assistant-1"), - delta: `reply to ${label}`, - createdAt: at(6), - }, - { - type: "thread.message.assistant.complete", - commandId: CommandId.make("cmd-assistant-complete-1"), - threadId: ThreadId.make("thread-shared"), - messageId: MessageId.make("msg-assistant-1"), - createdAt: at(7), - }, -]; - -const extraThreadFor = (label: string): OrchestrationCommand => ({ - type: "thread.create", - commandId: CommandId.make("cmd-thread-extra-create"), - threadId: ThreadId.make("thread-extra"), - projectId: ProjectId.make("project-shared"), - title: `Extra thread of ${label}`, - modelSelection, - interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, - runtimeMode: "approval-required", - branch: null, - worktreePath: null, - createdAt, -}); - -type UserEngine = { - readonly run: ( - effect: Effect.Effect, - ) => Effect.Effect; - readonly scope: Scope.Closeable; -}; - -// Builds one user's engine (bootstrap + command read model) over the shared -// pool and platform services, in its own scope so it can be evicted. -const activateUser = (userId: string) => - Effect.gen(function* () { - const shared = yield* Effect.context(); - const scope = yield* Scope.make(); - const context = yield* Layer.buildWithScope(makeHubUserEngineLayer({ userId, shared }), scope); - return { - scope, - run: (effect) => Effect.provideContext(effect, context), - } satisfies UserEngine; - }); - -const readEvents = (user: UserEngine, fromSequenceExclusive = 0) => - user.run( - Effect.gen(function* () { - const engine = yield* OrchestrationEngineService; - const chunk = yield* Stream.runCollect(engine.readEvents(fromSequenceExclusive)); - return Array.from(chunk) as OrchestrationEvent[]; - }), - ); - -const dispatch = (user: UserEngine, command: OrchestrationCommand) => - user.run( - Effect.flatMap(Effect.service(OrchestrationEngineService), (engine) => - engine.dispatch(command), - ), - ); - -const withQuery = ( - user: UserEngine, - read: (query: ProjectionSnapshotQuery["Service"]) => Effect.Effect, -) => user.run(Effect.flatMap(Effect.service(ProjectionSnapshotQuery), read)); - -const deleteUsers = (users: ReadonlyArray) => - Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - for (const table of [...HUB_TENANT_TABLES, "hub_users"]) { - yield* sql`DELETE FROM ${sql(table)} WHERE ${sql.in("user_id", users)}`; - } - }); - -if (databaseUrl === undefined) { - it.skip("hub engines on Postgres (set T3_HUB_TEST_DATABASE_URL)", () => {}); -} else { - it.layer(makeHubSharedLayer({ url: databaseUrl, maxConnections: 4 }), { - excludeTestServices: true, - })("hub engines on Postgres", (it) => { - it.effect("isolates projects, threads, events, and sequences per user in one process", () => - Effect.gen(function* () { - yield* deleteUsers([userA, userB]); - const a = yield* activateUser(userA); - const b = yield* activateUser(userB); - - // Interleave the two users' commands through their own engines. - const commandsA = commandsFor("alice"); - const commandsB = commandsFor("bob"); - const sequencesA: number[] = []; - const sequencesB: number[] = []; - for (let index = 0; index < commandsA.length; index += 1) { - sequencesA.push((yield* dispatch(a, commandsA[index]!)).sequence); - sequencesB.push((yield* dispatch(b, commandsB[index]!)).sequence); - } - sequencesA.push((yield* dispatch(a, extraThreadFor("alice"))).sequence); - - const eventsA = yield* readEvents(a); - const eventsB = yield* readEvents(b); - - // Per-user sequences start at 1 and are contiguous, so clients resume per user. - assert.deepStrictEqual( - eventsA.map((event) => event.sequence), - Array.from({ length: eventsA.length }, (_, index) => index + 1), - ); - assert.deepStrictEqual( - eventsB.map((event) => event.sequence), - Array.from({ length: eventsB.length }, (_, index) => index + 1), - ); - assert.strictEqual(sequencesA.at(-1), eventsA.length); - assert.strictEqual(sequencesB.at(-1), eventsB.length); - assert.isAbove(eventsA.length, eventsB.length); - - // Events never cross users, even though ids are identical. - // @effect-diagnostics-next-line preferSchemaOverJson:off - expect(JSON.stringify(eventsA)).not.toContain("bob"); - // @effect-diagnostics-next-line preferSchemaOverJson:off - expect(JSON.stringify(eventsB)).not.toContain("alice"); - - // Resume-by-sequence reads only the caller's later events. - const resumeFrom = 3; - const resumedA = yield* readEvents(a, resumeFrom); - assert.deepStrictEqual( - resumedA.map((event) => event.sequence), - eventsA.filter((event) => event.sequence > resumeFrom).map((event) => event.sequence), - ); - - // Shell snapshots. - const shellA = yield* withQuery(a, (query) => query.getShellSnapshot()); - const shellB = yield* withQuery(b, (query) => query.getShellSnapshot()); - assert.deepStrictEqual( - shellA.projects.map((project) => project.title), - ["Project of alice"], - ); - assert.deepStrictEqual( - shellB.projects.map((project) => project.title), - ["Project of bob"], - ); - assert.deepStrictEqual(shellA.threads.map((thread) => thread.title).toSorted(), [ - "Extra thread of alice", - "Thread of alice", - ]); - assert.deepStrictEqual( - shellB.threads.map((thread) => thread.title), - ["Thread of bob"], - ); - assert.strictEqual(shellA.snapshotSequence, eventsA.length); - assert.strictEqual(shellB.snapshotSequence, eventsB.length); - - // Thread detail snapshots: full and windowed. - for (const window of [undefined, { turnLimit: 5 }]) { - const threadA = Option.getOrThrow( - yield* withQuery(a, (query) => - query.getThreadDetailSnapshot(ThreadId.make("thread-shared"), window), - ), - ).thread; - const threadB = Option.getOrThrow( - yield* withQuery(b, (query) => - query.getThreadDetailSnapshot(ThreadId.make("thread-shared"), window), - ), - ).thread; - assert.deepStrictEqual( - threadA.messages.map((message) => message.text), - ["hello from alice", "reply to alice"], - ); - assert.deepStrictEqual( - threadB.messages.map((message) => message.text), - ["hello from bob", "reply to bob"], - ); - assert.deepStrictEqual( - threadA.activities.map((activity) => activity.summary), - ["tool ran for alice"], - ); - assert.deepStrictEqual( - threadB.activities.map((activity) => activity.summary), - ["tool ran for bob"], - ); - } - - // Counts and command receipts are tenant-scoped too: replaying user A's - // command id against user B's engine is B's own replay, not a conflict. - assert.deepStrictEqual(yield* withQuery(a, (query) => query.getCounts()), { - projectCount: 1, - threadCount: 2, - }); - assert.deepStrictEqual(yield* withQuery(b, (query) => query.getCounts()), { - projectCount: 1, - threadCount: 1, - }); - assert.strictEqual((yield* dispatch(b, commandsB[0]!)).replayed, true); - - // Evicting and reactivating A rebuilds the same command read model. - const readModelBefore = yield* withQuery(a, (query) => query.getCommandReadModel()); - yield* Scope.close(a.scope, Exit.void); - const a2 = yield* activateUser(userA); - const readModelAfter = yield* withQuery(a2, (query) => query.getCommandReadModel()); - assert.deepStrictEqual(readModelAfter, readModelBefore); - assert.strictEqual(readModelAfter.snapshotSequence, eventsA.length); - assert.strictEqual(readModelAfter.threads.length, 2); - - yield* Scope.close(a2.scope, Exit.void); - yield* Scope.close(b.scope, Exit.void); - yield* deleteUsers([userA, userB]); - }), - ); - - it.effect("row-level security backstop fails closed without a tenant context", () => - Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - yield* deleteUsers([userA, userB]); - yield* sql` - INSERT INTO projection_projects (user_id, project_id, title, workspace_root, scripts_json, created_at, updated_at) - VALUES - (${userA}, 'rls-project', 'rls a', '/a', '[]', ${createdAt}, ${createdAt}), - (${userB}, 'rls-project', 'rls b', '/b', '[]', ${createdAt}, ${createdAt}) - `; - yield* sql` - DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'hub_rls_probe') THEN - CREATE ROLE hub_rls_probe NOLOGIN; - END IF; - END $$ - `; - yield* sql`GRANT SELECT, INSERT ON projection_projects TO hub_rls_probe`; - for (const statement of hubRowLevelSecurityStatements(["projection_projects"])) { - yield* sql.unsafe(statement); - } - - // A non-owner role inside a transaction, optionally with a tenant context. - const asProbe = (tenant: string | null, effect: Effect.Effect) => - sql.withTransaction( - Effect.gen(function* () { - yield* sql`SET LOCAL ROLE hub_rls_probe`; - if (tenant !== null) { - yield* sql`SELECT set_config('hub.user_id', ${tenant}, true)`; - } - return yield* effect; - }), - ); - const titles = (tenant: string | null) => - asProbe( - tenant, - sql<{ title: string }>` - SELECT title FROM projection_projects WHERE project_id = 'rls-project' ORDER BY title - `.pipe(Effect.map((rows) => rows.map((row) => row.title))), - ); - - yield* Effect.gen(function* () { - // No tenant context: nothing is visible, even without a user_id predicate. - assert.deepStrictEqual(yield* titles(null), []); - assert.deepStrictEqual(yield* titles(userA), ["rls a"]); - assert.deepStrictEqual(yield* titles(userB), ["rls b"]); - // Writes for another tenant are rejected by WITH CHECK. - const forged = yield* asProbe( - userA, - sql` - INSERT INTO projection_projects (user_id, project_id, title, workspace_root, scripts_json, created_at, updated_at) - VALUES (${userB}, 'rls-forged', 'forged', '/b', '[]', ${createdAt}, ${createdAt}) - `, - ).pipe(Effect.exit); - assert.isTrue(Exit.isFailure(forged)); - // The setting is transaction-local, so the next transaction starts clean. - assert.deepStrictEqual(yield* titles(null), []); - }).pipe( - Effect.ensuring( - sql`ALTER TABLE projection_projects DISABLE ROW LEVEL SECURITY`.pipe(Effect.orDie), - ), - ); - yield* deleteUsers([userA, userB]); - }), - ); - - it.effect( - "shows why schema-per-user cannot rely on session search_path over a shared connection", - () => - Effect.gen(function* () { - // One pooled connection stands in for a PgBouncer server connection that - // transaction pooling hands to whichever client runs the next transaction. - const single = yield* Layer.build( - Layer.fresh( - PgClient.layer({ - url: Redacted.make(databaseUrl), - maxConnections: 1, - types: hubPgTypes, - }), - ), - ); - const db = Context.get(single, SqlClient.SqlClient); - for (const schema of ["hub_probe_a", "hub_probe_b"]) { - yield* db`CREATE SCHEMA IF NOT EXISTS ${db(schema)}`; - yield* db`CREATE TABLE IF NOT EXISTS ${db(schema)}.probe_threads (title text)`; - yield* db`TRUNCATE ${db(schema)}.probe_threads`; - yield* db`INSERT INTO ${db(schema)}.probe_threads VALUES (${schema})`; - } - // Tenant A sets its schema for the session... - yield* db`SET search_path TO hub_probe_a`; - // ...and tenant B's next statement on that connection reads A's data. - const leaked = yield* db<{ title: string }>`SELECT title FROM probe_threads`; - yield* db`RESET search_path`; - // SET LOCAL is scoped to one transaction, which is safe behind a pooler, - // but then every statement must run inside such a transaction. - const scoped = yield* db.withTransaction( - Effect.gen(function* () { - yield* db`SET LOCAL search_path TO hub_probe_b`; - return yield* db<{ title: string }>`SELECT title FROM probe_threads`; - }), - ); - const after = yield* db<{ search_path: string }>`SHOW search_path`; - yield* db`DROP SCHEMA hub_probe_a CASCADE`; - yield* db`DROP SCHEMA hub_probe_b CASCADE`; - - assert.deepStrictEqual( - leaked.map((row) => row.title), - ["hub_probe_a"], - ); - assert.deepStrictEqual( - scoped.map((row) => row.title), - ["hub_probe_b"], - ); - assert.strictEqual(after[0]?.search_path, '"$user", public'); - }), - ); - }); -} diff --git a/apps/server/src/persistence/Postgres/HubEngine.ts b/apps/server/src/persistence/Postgres/HubEngine.ts deleted file mode 100644 index 6e185a9c2894..000000000000 --- a/apps/server/src/persistence/Postgres/HubEngine.ts +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Per-user orchestration engine over the shared hub Postgres pool. - * - * The hub keeps T3's one-writer engine model per user: each active user gets an - * engine (command queue, in-memory command read model, event PubSub) whose - * repositories are scoped by `HubTenant`. All engines in a process share one - * `SqlClient`, so the pool size is independent of the number of active users. - */ -import * as NodeServices from "@effect/platform-node/NodeServices"; -import * as Context from "effect/Context"; -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; -import * as SqlClient from "effect/unstable/sql/SqlClient"; - -import { ServerConfig } from "../../config.ts"; -import { makeHubPostgresPersistenceLive, type HubPostgresConfig } from "../Layers/Postgres.ts"; -import { OrchestrationEngineLive } from "../../orchestration/Layers/OrchestrationEngine.ts"; -import { makeOrchestrationProjectionPipeline } from "../../orchestration/Layers/ProjectionPipeline.ts"; -import { OrchestrationProjectionPipeline } from "../../orchestration/Services/ProjectionPipeline.ts"; -import * as ThreadBackgroundLiveness from "../../orchestration/ThreadBackgroundLiveness.ts"; -import * as ThreadPlanProgress from "../../orchestration/ThreadPlanProgress.ts"; -import { RepositoryIdentityResolver } from "../../project/RepositoryIdentityResolver.ts"; -import { hubTenantLayer } from "./HubTenant.ts"; -import { PgOrchestrationCommandReceiptRepositoryLive } from "./OrchestrationCommandReceipts.ts"; -import { PgOrchestrationEventStoreLive } from "./OrchestrationEventStore.ts"; -import { PgProjectionPendingApprovalRepositoryLive } from "./ProjectionPendingApprovals.ts"; -import { PgProjectionProjectRepositoryLive } from "./ProjectionProjects.ts"; -import { PgProjectionSnapshotQueryLive } from "./ProjectionSnapshotQuery.ts"; -import { PgProjectionStateRepositoryLive } from "./ProjectionState.ts"; -import { PgProjectionThreadActivityRepositoryLive } from "./ProjectionThreadActivities.ts"; -import { PgProjectionThreadMessageRepositoryLive } from "./ProjectionThreadMessages.ts"; -import { PgProjectionThreadProposedPlanRepositoryLive } from "./ProjectionThreadProposedPlans.ts"; -import { PgProjectionThreadSessionRepositoryLive } from "./ProjectionThreadSessions.ts"; -import { PgProjectionThreadRepositoryLive } from "./ProjectionThreads.ts"; -import { PgProjectionTurnRepositoryLive } from "./ProjectionTurns.ts"; - -export const PgOrchestrationProjectionPipelineLive = Layer.effect( - OrchestrationProjectionPipeline, - makeOrchestrationProjectionPipeline(), -).pipe( - Layer.provideMerge(PgProjectionProjectRepositoryLive), - Layer.provideMerge(PgProjectionThreadRepositoryLive), - Layer.provideMerge(PgProjectionThreadMessageRepositoryLive), - Layer.provideMerge(PgProjectionThreadProposedPlanRepositoryLive), - Layer.provideMerge(PgProjectionThreadActivityRepositoryLive), - Layer.provideMerge(PgProjectionThreadSessionRepositoryLive), - Layer.provideMerge(PgProjectionTurnRepositoryLive), - Layer.provideMerge(PgProjectionPendingApprovalRepositoryLive), - Layer.provideMerge(PgProjectionStateRepositoryLive), -); - -// The hub has no repositories on disk. Repository identity moves onto the -// project record (see the design doc); until then projects report none. -const NoRepositoryIdentityLive = Layer.succeed(RepositoryIdentityResolver, { - resolve: () => Effect.succeed(null), -}); - -/** Process-wide services every user engine shares: the pool and the platform. */ -export type HubSharedServices = SqlClient.SqlClient | NodeServices.NodeServices; - -export const makeHubSharedLayer = (config: HubPostgresConfig) => - Layer.merge(makeHubPostgresPersistenceLive(config), NodeServices.layer); - -/** - * Engine + snapshot query for one user over the process-wide services. - * ServerConfig is only used for attachment side effects, which move to object - * storage in the hub; a per-user temp config keeps the prototype self-contained. - */ -export const makeHubUserEngineLayer = (input: { - readonly userId: string; - readonly shared: Context.Context; -}) => { - const serverConfigLayer = ServerConfig.layerTest(process.cwd(), { - prefix: `t3-hub-${input.userId.replace(/[^a-zA-Z0-9_-]/g, "_")}-`, - }); - return Layer.mergeAll( - OrchestrationEngineLive.pipe( - Layer.provide(PgProjectionSnapshotQueryLive), - Layer.provide(PgOrchestrationProjectionPipelineLive), - ), - PgProjectionSnapshotQueryLive, - ).pipe( - Layer.provideMerge(ThreadBackgroundLiveness.layer), - Layer.provide(ThreadPlanProgress.layer), - Layer.provide(PgOrchestrationEventStoreLive), - Layer.provideMerge(PgOrchestrationCommandReceiptRepositoryLive), - Layer.provide(NoRepositoryIdentityLive), - Layer.provide(hubTenantLayer(input.userId)), - Layer.provideMerge(serverConfigLayer), - Layer.provideMerge(Layer.succeedContext(input.shared)), - ); -}; diff --git a/apps/server/src/persistence/Postgres/HubMigrator.ts b/apps/server/src/persistence/Postgres/HubMigrator.ts new file mode 100644 index 000000000000..8770135d1165 --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubMigrator.ts @@ -0,0 +1,84 @@ +/** + * HubMigrator - ordered, idempotent hub schema migrations. + * + * Each pending migration runs in its own transaction that first takes a + * transaction-scoped advisory lock, re-reads the applied set, applies the + * lowest pending id, and records it in `hub_schema_migrations`. Concurrent hub + * processes therefore serialize and never apply a migration twice, and the lock + * is released with the transaction, which keeps it safe behind a + * transaction-mode pooler (a session-level advisory lock is not). + * + * Migrations are applied by id, not by list position, and an id missing below + * an applied one is still applied: separately owned ranges (001-049 hub + * persistence, 050-099 thread machines) can land in any order. + * + * @module HubMigrator + */ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import type { SqlError } from "effect/unstable/sql/SqlError"; + +export type HubMigration = Effect.Effect; + +export interface HubMigrationEntry { + readonly id: number; + readonly name: string; + readonly migration: HubMigration; +} + +// Arbitrary constant shared by every hub process. +const HUB_MIGRATION_LOCK_KEY = 7_330_001; + +/** Sorts entries by id and rejects duplicate or invalid ids. */ +export const orderHubMigrations = ( + entries: ReadonlyArray, +): ReadonlyArray => { + const seen = new Set(); + for (const entry of entries) { + if (!Number.isSafeInteger(entry.id) || entry.id <= 0) { + throw new Error(`Hub migration ${entry.name} has an invalid id ${entry.id}.`); + } + if (seen.has(entry.id)) { + throw new Error(`Hub migration id ${entry.id} is used more than once.`); + } + seen.add(entry.id); + } + return entries.toSorted((left, right) => left.id - right.id); +}; + +/** Applies every pending migration; returns the ones this call applied. */ +export const runHubMigrations = (entries: ReadonlyArray) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const ordered = orderHubMigrations(entries); + const applied: Array> = []; + + const applyNext = sql.withTransaction( + Effect.gen(function* () { + yield* sql.unsafe(`SELECT pg_advisory_xact_lock(${HUB_MIGRATION_LOCK_KEY})`); + yield* sql.unsafe(`CREATE TABLE IF NOT EXISTS hub_schema_migrations ( + id integer PRIMARY KEY, + name text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now() + )`); + const rows = yield* sql<{ readonly id: number }>`SELECT id FROM hub_schema_migrations`; + const done = new Set(rows.map((row) => Number(row.id))); + const next = ordered.find((entry) => !done.has(entry.id)); + if (next === undefined) { + return undefined; + } + yield* next.migration; + yield* sql`INSERT INTO hub_schema_migrations (id, name) VALUES (${next.id}, ${next.name})`; + return next; + }), + ); + + while (true) { + const next = yield* applyNext; + if (next === undefined) { + return applied; + } + applied.push({ id: next.id, name: next.name }); + yield* Effect.logInfo("Applied hub migration.", { id: next.id, name: next.name }); + } + }); diff --git a/apps/server/src/persistence/Postgres/ProviderSessionRuntime.ts b/apps/server/src/persistence/Postgres/ProviderSessionRuntime.ts new file mode 100644 index 000000000000..3d3aea2da365 --- /dev/null +++ b/apps/server/src/persistence/Postgres/ProviderSessionRuntime.ts @@ -0,0 +1,304 @@ +/** + * Postgres port of `ProviderSessionRuntimeRepository` for the hub (see + * `../ProviderSessionRuntime.ts`): the provider session directory the hub keeps + * for every thread, including resume cursors. Every statement is scoped to the + * process tenant. Keep the SQL in step with the SQLite repository; the + * differences are the `user_id` predicates. + */ +import * as Arr from "effect/Array"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Result from "effect/Result"; +import * as Schema from "effect/Schema"; +import * as Struct from "effect/Struct"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; + +import { + IsoDateTime, + ProviderInstanceId, + ProviderSessionRuntimeStatus, + RuntimeMode, + ThreadId, +} from "@t3tools/contracts"; + +import { + PersistenceDecodeError, + type PersistenceErrorCorrelation, + PersistenceSqlError, + type ProviderSessionRuntimeRepositoryError, +} from "../Errors.ts"; +import type { ProviderSessionRuntimeRepository } from "../ProviderSessionRuntime.ts"; +import { HubTenant } from "./HubTenant.ts"; + +/** + * ProviderSessionRuntimeRepository - Repository interface for provider runtime sessions. + * + * Owns persistence operations for provider runtime metadata and resume cursors. + * + * @module ProviderSessionRuntimeRepository + */ + +const ProviderSessionRuntime = Schema.Struct({ + threadId: ThreadId, + providerName: Schema.String, + /** + * User-defined routing key for the configured provider instance that + * owns this session. Nullable only at the storage/migration boundary: + * rows persisted before the driver/instance split carry only + * `providerName`. Repository consumers must materialize a concrete + * instance id before routing. + */ + providerInstanceId: Schema.NullOr(ProviderInstanceId), + adapterKey: Schema.String, + runtimeMode: RuntimeMode, + status: ProviderSessionRuntimeStatus, + lastSeenAt: IsoDateTime, + resumeCursor: Schema.NullOr(Schema.Unknown), + runtimePayload: Schema.NullOr(Schema.Unknown), +}); +type ProviderSessionRuntime = typeof ProviderSessionRuntime.Type; + +const GetProviderSessionRuntimeInput = Schema.Struct({ threadId: ThreadId }); +type GetProviderSessionRuntimeInput = typeof GetProviderSessionRuntimeInput.Type; + +const DeleteProviderSessionRuntimeInput = Schema.Struct({ threadId: ThreadId }); +type DeleteProviderSessionRuntimeInput = typeof DeleteProviderSessionRuntimeInput.Type; + +/** + * ProviderSessionRuntimeRepository - Service tag for provider runtime persistence. + */ + +const ProviderSessionRuntimeDbRowSchema = ProviderSessionRuntime.mapFields( + Struct.assign({ + resumeCursor: Schema.NullOr(Schema.fromJsonString(Schema.Unknown)), + runtimePayload: Schema.NullOr(Schema.fromJsonString(Schema.Unknown)), + }), +); + +const ProviderSessionRuntimeRawDbRowSchema = Schema.Struct({ + threadId: Schema.String, + providerName: Schema.Unknown, + providerInstanceId: Schema.Unknown, + adapterKey: Schema.Unknown, + runtimeMode: Schema.Unknown, + status: Schema.Unknown, + lastSeenAt: Schema.Unknown, + resumeCursor: Schema.Unknown, + runtimePayload: Schema.Unknown, +}); + +const decodeRuntimeRow = Schema.decodeUnknownEffect(ProviderSessionRuntimeDbRowSchema); + +const GetRuntimeRequestSchema = Schema.Struct({ + threadId: ThreadId, +}); + +const DeleteRuntimeRequestSchema = GetRuntimeRequestSchema; + +function toPersistenceSqlOrDecodeError( + sqlOperation: string, + decodeOperation: string, + correlation?: PersistenceErrorCorrelation, +) { + return (cause: unknown): ProviderSessionRuntimeRepositoryError => + Schema.isSchemaError(cause) + ? PersistenceDecodeError.fromSchemaError(decodeOperation, cause, correlation) + : new PersistenceSqlError({ + operation: sqlOperation, + ...(correlation === undefined ? {} : { correlation }), + cause, + }); +} + +export const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const upsertRuntimeRow = SqlSchema.void({ + Request: ProviderSessionRuntimeDbRowSchema, + execute: (runtime) => + sql` + INSERT INTO provider_session_runtime ( + user_id, + thread_id, + provider_name, + provider_instance_id, + adapter_key, + runtime_mode, + status, + last_seen_at, + resume_cursor_json, + runtime_payload_json + ) + VALUES ( + ${userId}, + ${runtime.threadId}, + ${runtime.providerName}, + ${runtime.providerInstanceId}, + ${runtime.adapterKey}, + ${runtime.runtimeMode}, + ${runtime.status}, + ${runtime.lastSeenAt}, + ${runtime.resumeCursor}, + ${runtime.runtimePayload} + ) + ON CONFLICT (user_id, thread_id) + DO UPDATE SET + provider_name = excluded.provider_name, + provider_instance_id = excluded.provider_instance_id, + adapter_key = excluded.adapter_key, + runtime_mode = excluded.runtime_mode, + status = excluded.status, + last_seen_at = excluded.last_seen_at, + resume_cursor_json = excluded.resume_cursor_json, + runtime_payload_json = excluded.runtime_payload_json + `, + }); + + const getRuntimeRowByThreadId = SqlSchema.findOneOption({ + Request: GetRuntimeRequestSchema, + Result: ProviderSessionRuntimeRawDbRowSchema, + execute: ({ threadId }) => + sql` + SELECT + thread_id AS "threadId", + provider_name AS "providerName", + provider_instance_id AS "providerInstanceId", + adapter_key AS "adapterKey", + runtime_mode AS "runtimeMode", + status, + last_seen_at AS "lastSeenAt", + resume_cursor_json AS "resumeCursor", + runtime_payload_json AS "runtimePayload" + FROM provider_session_runtime + WHERE user_id = ${userId} + AND thread_id = ${threadId} + `, + }); + + const listRuntimeRows = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProviderSessionRuntimeRawDbRowSchema, + execute: () => + sql` + SELECT + thread_id AS "threadId", + provider_name AS "providerName", + provider_instance_id AS "providerInstanceId", + adapter_key AS "adapterKey", + runtime_mode AS "runtimeMode", + status, + last_seen_at AS "lastSeenAt", + resume_cursor_json AS "resumeCursor", + runtime_payload_json AS "runtimePayload" + FROM provider_session_runtime + WHERE user_id = ${userId} + ORDER BY last_seen_at ASC, thread_id ASC + `, + }); + + const deleteRuntimeByThreadId = SqlSchema.void({ + Request: DeleteRuntimeRequestSchema, + execute: ({ threadId }) => + sql` + DELETE FROM provider_session_runtime + WHERE user_id = ${userId} + AND thread_id = ${threadId} + `, + }); + + const upsert: ProviderSessionRuntimeRepository["Service"]["upsert"] = (runtime) => + upsertRuntimeRow(runtime).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProviderSessionRuntimeRepository.upsert:query", + "ProviderSessionRuntimeRepository.upsert:encodeRequest", + { threadId: runtime.threadId }, + ), + ), + ); + + const getByThreadId: ProviderSessionRuntimeRepository["Service"]["getByThreadId"] = (input) => + getRuntimeRowByThreadId(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProviderSessionRuntimeRepository.getByThreadId:query", + "ProviderSessionRuntimeRepository.getByThreadId:decodeRow", + { threadId: input.threadId }, + ), + ), + Effect.flatMap((runtimeRowOption) => + Option.match(runtimeRowOption, { + onNone: () => Effect.succeed(Option.none()), + onSome: (row) => + decodeRuntimeRow(row).pipe( + Effect.mapError((cause) => + PersistenceDecodeError.fromSchemaError( + "ProviderSessionRuntimeRepository.getByThreadId:decodeRow", + cause, + { threadId: input.threadId }, + ), + ), + Effect.map((runtime) => Option.some(runtime)), + ), + }), + ), + ); + + const list: ProviderSessionRuntimeRepository["Service"]["list"] = () => + listRuntimeRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "ProviderSessionRuntimeRepository.list:query", + "ProviderSessionRuntimeRepository.list:decodeRows", + ), + ), + Effect.flatMap((rows) => + // Skip rows that no longer decode (e.g. written by an older build) + // instead of failing the whole list — one stale row must not disable + // every consumer that enumerates sessions, such as the reaper. + Effect.forEach(rows, (row) => + decodeRuntimeRow(row).pipe( + Effect.map(Option.some), + Effect.catch((cause) => + Effect.logWarning("provider.session.runtime.row-skipped", { + threadId: row.threadId, + error: PersistenceDecodeError.fromSchemaError( + "ProviderSessionRuntimeRepository.list:decodeRows", + cause, + { threadId: row.threadId }, + ).message, + }).pipe(Effect.as(Option.none())), + ), + ), + ), + ), + Effect.map((decoded) => + Arr.filterMap(decoded, (row) => + Option.isSome(row) ? Result.succeed(row.value) : Result.failVoid, + ), + ), + ); + + const deleteByThreadId: ProviderSessionRuntimeRepository["Service"]["deleteByThreadId"] = ( + input, + ) => + deleteRuntimeByThreadId(input).pipe( + Effect.mapError( + (cause) => + new PersistenceSqlError({ + operation: "ProviderSessionRuntimeRepository.deleteByThreadId:query", + correlation: { threadId: input.threadId }, + cause, + }), + ), + ); + + return { + upsert, + getByThreadId, + list, + deleteByThreadId, + } satisfies ProviderSessionRuntimeRepository["Service"]; +}); diff --git a/apps/server/src/persistence/Postgres/hubTestDatabase.ts b/apps/server/src/persistence/Postgres/hubTestDatabase.ts new file mode 100644 index 000000000000..fbd4dfe7e26e --- /dev/null +++ b/apps/server/src/persistence/Postgres/hubTestDatabase.ts @@ -0,0 +1,112 @@ +/** + * Disposable hub databases for Postgres-backed tests. + * + * Tests run only when `T3_HUB_TEST_DATABASE_URL` points at a database the test + * may create schemas in, e.g. a private local cluster: + * + * T3_HUB_TEST_DATABASE_URL=postgres://postgres@127.0.0.1:54359/postgres + * + * Each test schema is created with a unique name and dropped afterwards. When + * the connecting role is a superuser or has `BYPASSRLS` (a local cluster), a + * throwaway login role without those attributes is created too and used as the + * hub runtime role, so row-level security is exercised end to end. Otherwise + * the connecting role is used directly; `FORCE ROW LEVEL SECURITY` makes the + * policies apply to it as the table owner. + * + * @module hubTestDatabase + */ +import * as NodeCrypto from "node:crypto"; + +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Reactivity from "effect/unstable/reactivity/Reactivity"; + +import { HubDatabase } from "./HubDatabase.ts"; +import { makeHubDatabase } from "./HubDatabaseLive.ts"; +import { makeHubPool } from "./HubClient.ts"; + +export const hubTestDatabaseUrl = process.env.T3_HUB_TEST_DATABASE_URL?.trim() || undefined; + +export interface HubTestSchema { + readonly schema: string; + /** Owner of the schema's tables; applies migrations. */ + readonly adminUrl: string; + /** The hub runtime role; subject to row-level security. */ + readonly runtimeUrl: string; + /** Whether a separate role without BYPASSRLS was created for the runtime. */ + readonly separateRuntimeRole: boolean; +} + +const withSearchPath = (url: string, schema: string, credentials?: [string, string]) => { + const parsed = new URL(url); + parsed.searchParams.set("options", `-c search_path=${schema}`); + if (credentials) { + parsed.username = credentials[0]; + parsed.password = credentials[1]; + } + return parsed.toString(); +}; + +/** Creates a unique schema (and runtime role when possible); drops both on scope close. */ +export const makeHubTestSchema = (baseUrl: string) => + Effect.gen(function* () { + const suffix = NodeCrypto.randomBytes(6).toString("hex"); + const schema = `tm_hub_test_${suffix}`; + const admin = yield* makeHubPool({ + url: baseUrl, + maxConnections: 1, + applicationName: "t3-hub-test", + }); + const [attributes] = yield* admin<{ + readonly rolsuper: boolean; + readonly rolbypassrls: boolean; + }>`SELECT rolsuper, rolbypassrls FROM pg_roles WHERE rolname = current_user`; + const needsRuntimeRole = attributes?.rolsuper === true || attributes?.rolbypassrls === true; + const runtimeRole = `${schema}_app`; + const runtimePassword = NodeCrypto.randomBytes(12).toString("hex"); + + yield* Effect.acquireRelease( + Effect.gen(function* () { + yield* admin.unsafe(`CREATE SCHEMA ${schema}`); + if (needsRuntimeRole) { + yield* admin.unsafe( + `CREATE ROLE ${runtimeRole} LOGIN NOSUPERUSER NOBYPASSRLS PASSWORD '${runtimePassword}'`, + ); + } + }), + () => + Effect.gen(function* () { + if (needsRuntimeRole) { + yield* admin.unsafe( + `SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = '${runtimeRole}'`, + ); + } + yield* admin.unsafe(`DROP SCHEMA IF EXISTS ${schema} CASCADE`); + if (needsRuntimeRole) { + yield* admin.unsafe(`DROP OWNED BY ${runtimeRole}`); + yield* admin.unsafe(`DROP ROLE IF EXISTS ${runtimeRole}`); + } + }).pipe(Effect.orDie), + ); + + const adminUrl = withSearchPath(baseUrl, schema); + return { + schema, + adminUrl, + runtimeUrl: needsRuntimeRole + ? withSearchPath(baseUrl, schema, [runtimeRole, runtimePassword]) + : adminUrl, + separateRuntimeRole: needsRuntimeRole, + } satisfies HubTestSchema; + }).pipe(Effect.provide(Reactivity.layer)); + +/** A migrated hub database for one tenant in the given test schema. */ +export const hubTestDatabaseLayer = (schema: HubTestSchema, tenantId: string) => + Layer.effect( + HubDatabase, + makeHubDatabase({ + databaseUrl: schema.runtimeUrl, + databaseAdminUrl: schema.separateRuntimeRole ? schema.adminUrl : undefined, + tenantId, + }), + ); diff --git a/apps/server/src/persistence/Postgres/Schema.ts b/apps/server/src/persistence/Postgres/migrations/001_HubBaseline.ts similarity index 60% rename from apps/server/src/persistence/Postgres/Schema.ts rename to apps/server/src/persistence/Postgres/migrations/001_HubBaseline.ts index d133a93404f9..7f03cfc4bbc2 100644 --- a/apps/server/src/persistence/Postgres/Schema.ts +++ b/apps/server/src/persistence/Postgres/migrations/001_HubBaseline.ts @@ -1,43 +1,109 @@ /** - * Hub baseline schema for PostgreSQL. + * Hub baseline: the standalone SQLite schema after its migration 043, plus the + * state a standalone server keeps as files, keyed by tenant. * - * Derived from the SQLite schema after migration 043 (`sqlite3 .schema`), - * not from a port of the 43 migrations. Tenancy is a `user_id` column that leads - * every primary key, unique constraint, and index, so one database serves every - * hub user and every query stays safe behind a transaction-mode pooler. + * Tenancy is a `user_id` column that leads every primary key, unique + * constraint, and index. Every table has a row-level-security policy that + * compares `user_id` with the transaction-local `hub.user_id` setting (see + * `HubClient.ts`); `FORCE` makes it apply to the table owner too, so only + * superusers and `BYPASSRLS` roles skip it. * - * Dialect decisions that the queries depend on: - * - Every text column that is compared or ordered uses `COLLATE "C"` so ordering - * matches SQLite's BINARY collation (the thread-detail keyset uses "~" and "" - * sentinels that only work bytewise), whatever the database default is. - * - Nullable sort keys are indexed `NULLS FIRST`, SQLite's NULL ordering, so the - * `DESC NULLS LAST` window reads are served by a backward index scan. - * - JSON stays in `text` columns: the repositories decode with `fromJsonString` - * and the `\u0000` escapes that JSON.stringify can emit are rejected by jsonb. - * - Boolean-like columns stay integers (`is_streaming`, counts) so row decoding is - * identical to SQLite. + * Dialect decisions the queries depend on: + * - Every compared or ordered text column uses `COLLATE "C"`, matching SQLite's + * BINARY ordering (the thread-detail keyset relies on bytewise "~" and "" + * sentinels) whatever the database default collation is. + * - Nullable sort keys are indexed `NULLS FIRST`, SQLite's NULL ordering. + * - JSON stays in `text` columns: repositories decode with `fromJsonString`, + * and `jsonb` rejects the `\u0000` escapes that JSON.stringify can emit. + * - Boolean-like columns stay integers so row decoding matches SQLite. * - SQLite's global `orchestration_events.sequence AUTOINCREMENT` becomes a * per-user counter in `hub_users.last_event_sequence`, allocated inside the - * append transaction, so sequences are gap-free per user like SQLite's. + * append transaction, so sequences stay gap-free per user. */ - -export const HUB_SCHEMA_VERSION = 1; +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; const c = `COLLATE "C"`; -export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ - `CREATE TABLE IF NOT EXISTS hub_schema_version ( - version integer PRIMARY KEY, - applied_at timestamptz NOT NULL DEFAULT now() - )`, - `CREATE TABLE IF NOT EXISTS hub_users ( +export const HUB_BASELINE_TENANT_TABLES = [ + "hub_users", + "hub_documents", + "hub_secrets", + "hub_attachments", + "orchestration_events", + "orchestration_command_receipts", + "checkpoint_diff_blobs", + "provider_session_runtime", + "projection_projects", + "projection_threads", + "projection_thread_messages", + "projection_thread_activities", + "projection_thread_sessions", + "projection_turns", + "projection_pending_approvals", + "projection_state", + "projection_thread_proposed_plans", + "auth_pairing_links", + "auth_sessions", +] as const; + +/** Enables the fail-closed tenant policy on a table (idempotent). */ +export const hubTenantPolicyStatements = (table: string): ReadonlyArray => [ + `ALTER TABLE ${table} ENABLE ROW LEVEL SECURITY`, + `ALTER TABLE ${table} FORCE ROW LEVEL SECURITY`, + `DROP POLICY IF EXISTS hub_tenant_isolation ON ${table}`, + `CREATE POLICY hub_tenant_isolation ON ${table} + USING (user_id = current_setting('hub.user_id', true)) + WITH CHECK (user_id = current_setting('hub.user_id', true))`, +]; + +const statements: ReadonlyArray = [ + // Per-user event sequence counter. + `CREATE TABLE hub_users ( user_id text ${c} PRIMARY KEY, last_event_sequence bigint NOT NULL DEFAULT 0, created_at timestamptz NOT NULL DEFAULT now() )`, - // orchestration_events (001) — sequence is per user. - `CREATE TABLE IF NOT EXISTS orchestration_events ( + // Small documents a standalone server keeps as files in its state directory: + // settings.json, keybindings.json, environment-id, anonymous-id. + `CREATE TABLE hub_documents ( + user_id text ${c} NOT NULL, + name text ${c} NOT NULL, + contents text NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, name) + )`, + + // ServerSecretStore values, AES-256-GCM encrypted with T3CODE_HUB_SECRET_KEY. + `CREATE TABLE hub_secrets ( + user_id text ${c} NOT NULL, + name text ${c} NOT NULL, + format smallint NOT NULL, + nonce bytea NOT NULL, + ciphertext bytea NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, name) + )`, + + // Attachment bytes. The local attachments directory is a disposable cache. + `CREATE TABLE hub_attachments ( + user_id text ${c} NOT NULL, + relative_path text ${c} NOT NULL, + attachment_id text ${c} NOT NULL, + thread_segment text ${c} NOT NULL, + size_bytes bigint NOT NULL, + content bytea NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, relative_path) + )`, + `CREATE INDEX idx_hub_attachments_attachment_id + ON hub_attachments (user_id, attachment_id)`, + `CREATE INDEX idx_hub_attachments_thread_segment + ON hub_attachments (user_id, thread_segment, created_at)`, + + // orchestration_events (SQLite 001) — sequence is per user. + `CREATE TABLE orchestration_events ( user_id text ${c} NOT NULL, sequence bigint NOT NULL, event_id text ${c} NOT NULL, @@ -55,17 +121,17 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ PRIMARY KEY (user_id, sequence), UNIQUE (user_id, event_id) )`, - `CREATE UNIQUE INDEX IF NOT EXISTS idx_orch_events_stream_version + `CREATE UNIQUE INDEX idx_orch_events_stream_version ON orchestration_events (user_id, aggregate_kind, stream_id, stream_version)`, - `CREATE INDEX IF NOT EXISTS idx_orch_events_stream_sequence + `CREATE INDEX idx_orch_events_stream_sequence ON orchestration_events (user_id, aggregate_kind, stream_id, sequence)`, - `CREATE INDEX IF NOT EXISTS idx_orch_events_command_id + `CREATE INDEX idx_orch_events_command_id ON orchestration_events (user_id, command_id)`, - `CREATE INDEX IF NOT EXISTS idx_orch_events_correlation_id + `CREATE INDEX idx_orch_events_correlation_id ON orchestration_events (user_id, correlation_id)`, // orchestration_command_receipts (002) - `CREATE TABLE IF NOT EXISTS orchestration_command_receipts ( + `CREATE TABLE orchestration_command_receipts ( user_id text ${c} NOT NULL, command_id text ${c} NOT NULL, aggregate_kind text ${c} NOT NULL, @@ -76,26 +142,27 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ error text, PRIMARY KEY (user_id, command_id) )`, - `CREATE INDEX IF NOT EXISTS idx_orch_command_receipts_aggregate + `CREATE INDEX idx_orch_command_receipts_aggregate ON orchestration_command_receipts (user_id, aggregate_kind, aggregate_id)`, - `CREATE INDEX IF NOT EXISTS idx_orch_command_receipts_sequence + `CREATE INDEX idx_orch_command_receipts_sequence ON orchestration_command_receipts (user_id, result_sequence)`, - // checkpoint_diff_blobs (003) - `CREATE TABLE IF NOT EXISTS checkpoint_diff_blobs ( + // checkpoint_diff_blobs (003): the patch of every completed turn, so reading a + // finished diff never wakes a thread machine. + `CREATE TABLE checkpoint_diff_blobs ( user_id text ${c} NOT NULL, thread_id text ${c} NOT NULL, from_turn_count integer NOT NULL, to_turn_count integer NOT NULL, diff text NOT NULL, created_at text ${c} NOT NULL, - UNIQUE (user_id, thread_id, from_turn_count, to_turn_count) + PRIMARY KEY (user_id, thread_id, from_turn_count, to_turn_count) )`, - `CREATE INDEX IF NOT EXISTS idx_checkpoint_diff_blobs_thread_to_turn + `CREATE INDEX idx_checkpoint_diff_blobs_thread_to_turn ON checkpoint_diff_blobs (user_id, thread_id, to_turn_count)`, // provider_session_runtime (004, 009, 027) - `CREATE TABLE IF NOT EXISTS provider_session_runtime ( + `CREATE TABLE provider_session_runtime ( user_id text ${c} NOT NULL, thread_id text ${c} NOT NULL, provider_name text ${c} NOT NULL, @@ -108,15 +175,16 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ provider_instance_id text ${c}, PRIMARY KEY (user_id, thread_id) )`, - `CREATE INDEX IF NOT EXISTS idx_provider_session_runtime_status + `CREATE INDEX idx_provider_session_runtime_status ON provider_session_runtime (user_id, status)`, - `CREATE INDEX IF NOT EXISTS idx_provider_session_runtime_provider + `CREATE INDEX idx_provider_session_runtime_provider ON provider_session_runtime (user_id, provider_name)`, - `CREATE INDEX IF NOT EXISTS idx_provider_session_runtime_instance + `CREATE INDEX idx_provider_session_runtime_instance ON provider_session_runtime (user_id, provider_instance_id)`, - // projection_projects (005, 039, 040) - `CREATE TABLE IF NOT EXISTS projection_projects ( + // projection_projects (005, 039, 040). repository_identity_json is hub-only: + // a hub has no checkout to run `git remote` in, so the identity is stored. + `CREATE TABLE projection_projects ( user_id text ${c} NOT NULL, project_id text ${c} NOT NULL, title text NOT NULL, @@ -128,15 +196,16 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ default_model_selection_json text, default_thread_env_mode text ${c}, favicon_path text, + repository_identity_json text, PRIMARY KEY (user_id, project_id) )`, - `CREATE INDEX IF NOT EXISTS idx_projection_projects_updated_at + `CREATE INDEX idx_projection_projects_updated_at ON projection_projects (user_id, updated_at)`, - `CREATE INDEX IF NOT EXISTS idx_projection_projects_workspace_root_deleted_at + `CREATE INDEX idx_projection_projects_workspace_root_deleted_at ON projection_projects (user_id, workspace_root, deleted_at)`, // projection_threads (005, 010, 012, 016, 017, 023, 033-036, 038, 042, 043) - `CREATE TABLE IF NOT EXISTS projection_threads ( + `CREATE TABLE projection_threads ( user_id text ${c} NOT NULL, thread_id text ${c} NOT NULL, project_id text ${c} NOT NULL, @@ -167,20 +236,20 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ unsettled_at text ${c}, PRIMARY KEY (user_id, thread_id) )`, - `CREATE INDEX IF NOT EXISTS idx_projection_threads_project_id + `CREATE INDEX idx_projection_threads_project_id ON projection_threads (user_id, project_id)`, - `CREATE INDEX IF NOT EXISTS idx_projection_threads_project_archived_at + `CREATE INDEX idx_projection_threads_project_archived_at ON projection_threads (user_id, project_id, archived_at)`, - `CREATE INDEX IF NOT EXISTS idx_projection_threads_project_deleted_created + `CREATE INDEX idx_projection_threads_project_deleted_created ON projection_threads (user_id, project_id, deleted_at, created_at)`, - `CREATE INDEX IF NOT EXISTS idx_projection_threads_shell_active + `CREATE INDEX idx_projection_threads_shell_active ON projection_threads (user_id, deleted_at, archived_at, project_id, created_at, thread_id)`, - `CREATE INDEX IF NOT EXISTS idx_projection_threads_shell_archived + `CREATE INDEX idx_projection_threads_shell_archived ON projection_threads (user_id, deleted_at, archived_at, project_id, thread_id)`, // projection_thread_messages (005, 007, 029). SQLite's (thread_id, created_at) - // and (thread_id, sequence) indexes are prefixes of the 029 indexes and are dropped. - `CREATE TABLE IF NOT EXISTS projection_thread_messages ( + // and (thread_id, sequence) indexes are prefixes of the 029 index. + `CREATE TABLE projection_thread_messages ( user_id text ${c} NOT NULL, message_id text ${c} NOT NULL, thread_id text ${c} NOT NULL, @@ -193,11 +262,11 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ attachments_json text, PRIMARY KEY (user_id, message_id) )`, - `CREATE INDEX IF NOT EXISTS idx_projection_thread_messages_thread_created_id + `CREATE INDEX idx_projection_thread_messages_thread_created_id ON projection_thread_messages (user_id, thread_id, created_at, message_id)`, // projection_thread_activities (005, 008, 029) - `CREATE TABLE IF NOT EXISTS projection_thread_activities ( + `CREATE TABLE projection_thread_activities ( user_id text ${c} NOT NULL, activity_id text ${c} NOT NULL, thread_id text ${c} NOT NULL, @@ -210,13 +279,13 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ sequence bigint, PRIMARY KEY (user_id, activity_id) )`, - `CREATE INDEX IF NOT EXISTS idx_projection_thread_activities_thread_created + `CREATE INDEX idx_projection_thread_activities_thread_created ON projection_thread_activities (user_id, thread_id, created_at)`, - `CREATE INDEX IF NOT EXISTS idx_projection_thread_activities_thread_sequence_created_id + `CREATE INDEX idx_projection_thread_activities_thread_sequence_created_id ON projection_thread_activities (user_id, thread_id, sequence NULLS FIRST, created_at, activity_id)`, // projection_thread_sessions (005, 006, 028) - `CREATE TABLE IF NOT EXISTS projection_thread_sessions ( + `CREATE TABLE projection_thread_sessions ( user_id text ${c} NOT NULL, thread_id text ${c} NOT NULL, status text ${c} NOT NULL, @@ -230,15 +299,15 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ provider_instance_id text ${c}, PRIMARY KEY (user_id, thread_id) )`, - `CREATE INDEX IF NOT EXISTS idx_projection_thread_sessions_provider_session + `CREATE INDEX idx_projection_thread_sessions_provider_session ON projection_thread_sessions (user_id, provider_session_id)`, - `CREATE INDEX IF NOT EXISTS idx_projection_thread_sessions_instance + `CREATE INDEX idx_projection_thread_sessions_instance ON projection_thread_sessions (user_id, provider_instance_id)`, - // projection_turns (005, 015, 037) — row_id keeps SQLite's surrogate key. - `CREATE TABLE IF NOT EXISTS projection_turns ( - row_id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + // projection_turns (005, 015, 037). row_id keeps SQLite's surrogate key. + `CREATE TABLE projection_turns ( user_id text ${c} NOT NULL, + row_id bigint GENERATED ALWAYS AS IDENTITY, thread_id text ${c} NOT NULL, turn_id text ${c}, pending_message_id text ${c}, @@ -253,18 +322,19 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ checkpoint_files_json text NOT NULL, source_proposed_plan_thread_id text ${c}, source_proposed_plan_id text ${c}, + PRIMARY KEY (user_id, row_id), UNIQUE (user_id, thread_id, turn_id), UNIQUE (user_id, thread_id, checkpoint_turn_count) )`, - `CREATE INDEX IF NOT EXISTS idx_projection_turns_thread_requested + `CREATE INDEX idx_projection_turns_thread_requested ON projection_turns (user_id, thread_id, requested_at)`, - `CREATE INDEX IF NOT EXISTS idx_projection_turns_thread_checkpoint_completed + `CREATE INDEX idx_projection_turns_thread_checkpoint_completed ON projection_turns (user_id, thread_id, checkpoint_turn_count, completed_at)`, - `CREATE INDEX IF NOT EXISTS idx_projection_turns_thread_keyset + `CREATE INDEX idx_projection_turns_thread_keyset ON projection_turns (user_id, thread_id, requested_at, turn_id NULLS FIRST)`, // projection_pending_approvals (005, 025) - `CREATE TABLE IF NOT EXISTS projection_pending_approvals ( + `CREATE TABLE projection_pending_approvals ( user_id text ${c} NOT NULL, request_id text ${c} NOT NULL, thread_id text ${c} NOT NULL, @@ -275,11 +345,11 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ resolved_at text ${c}, PRIMARY KEY (user_id, request_id) )`, - `CREATE INDEX IF NOT EXISTS idx_projection_pending_approvals_thread_status + `CREATE INDEX idx_projection_pending_approvals_thread_status ON projection_pending_approvals (user_id, thread_id, status)`, // projection_state (005) - `CREATE TABLE IF NOT EXISTS projection_state ( + `CREATE TABLE projection_state ( user_id text ${c} NOT NULL, projector text ${c} NOT NULL, last_applied_sequence bigint NOT NULL, @@ -288,7 +358,7 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ )`, // projection_thread_proposed_plans (013, 014) - `CREATE TABLE IF NOT EXISTS projection_thread_proposed_plans ( + `CREATE TABLE projection_thread_proposed_plans ( user_id text ${c} NOT NULL, plan_id text ${c} NOT NULL, thread_id text ${c} NOT NULL, @@ -300,12 +370,12 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ implementation_thread_id text ${c}, PRIMARY KEY (user_id, plan_id) )`, - `CREATE INDEX IF NOT EXISTS idx_projection_thread_proposed_plans_thread_created + `CREATE INDEX idx_projection_thread_proposed_plans_thread_created ON projection_thread_proposed_plans (user_id, thread_id, created_at)`, - // auth_pairing_links / auth_sessions (020-022, 031, 032, 041). The hub replaces - // pairing and bearer sessions with a signed gateway identity; kept for parity. - `CREATE TABLE IF NOT EXISTS auth_pairing_links ( + // auth_pairing_links / auth_sessions (020-022, 031, 032, 041): the hub keeps + // T3's pairing links, browser sessions, and bearer sessions. + `CREATE TABLE auth_pairing_links ( user_id text ${c} NOT NULL, id text ${c} NOT NULL, credential text ${c} NOT NULL, @@ -319,11 +389,11 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ revoked_at text ${c}, proof_key_thumbprint text ${c}, PRIMARY KEY (user_id, id), - UNIQUE (credential) + UNIQUE (user_id, credential) )`, - `CREATE INDEX IF NOT EXISTS idx_auth_pairing_links_active + `CREATE INDEX idx_auth_pairing_links_active ON auth_pairing_links (user_id, revoked_at, consumed_at, expires_at)`, - `CREATE TABLE IF NOT EXISTS auth_sessions ( + `CREATE TABLE auth_sessions ( user_id text ${c} NOT NULL, session_id text ${c} NOT NULL, subject text ${c} NOT NULL, @@ -343,40 +413,15 @@ export const HUB_BASELINE_STATEMENTS: ReadonlyArray = [ client_app_version text, PRIMARY KEY (user_id, session_id) )`, - `CREATE INDEX IF NOT EXISTS idx_auth_sessions_active + `CREATE INDEX idx_auth_sessions_active ON auth_sessions (user_id, revoked_at, expires_at, issued_at)`, -]; -/** Tables whose rows belong to one hub user; used by tests and the RLS backstop. */ -export const HUB_TENANT_TABLES = [ - "orchestration_events", - "orchestration_command_receipts", - "checkpoint_diff_blobs", - "provider_session_runtime", - "projection_projects", - "projection_threads", - "projection_thread_messages", - "projection_thread_activities", - "projection_thread_sessions", - "projection_turns", - "projection_pending_approvals", - "projection_state", - "projection_thread_proposed_plans", - "auth_pairing_links", - "auth_sessions", -] as const; + ...HUB_BASELINE_TENANT_TABLES.flatMap(hubTenantPolicyStatements), +]; -/** - * Optional row-level-security backstop. Policies compare `user_id` with the - * transaction-local `hub.user_id` setting, so a statement that runs without a - * tenant context sees no rows and cannot insert (fails closed). Table owners - * bypass RLS unless FORCE is set, so the backstop applies to a non-owner role. - */ -export const hubRowLevelSecurityStatements = (tables: ReadonlyArray = HUB_TENANT_TABLES) => - tables.flatMap((table) => [ - `ALTER TABLE ${table} ENABLE ROW LEVEL SECURITY`, - `DROP POLICY IF EXISTS hub_tenant_isolation ON ${table}`, - `CREATE POLICY hub_tenant_isolation ON ${table} - USING (user_id = current_setting('hub.user_id', true)) - WITH CHECK (user_id = current_setting('hub.user_id', true))`, - ]); +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + for (const statement of statements) { + yield* sql.unsafe(statement); + } +}); diff --git a/apps/server/src/persistence/Postgres/migrations/index.ts b/apps/server/src/persistence/Postgres/migrations/index.ts new file mode 100644 index 000000000000..d7397ba95136 --- /dev/null +++ b/apps/server/src/persistence/Postgres/migrations/index.ts @@ -0,0 +1,20 @@ +/** + * Hub migration registry. Add a numbered file under this directory and list it + * in the range that owns it; `runHubMigrations` orders entries by id, so the + * position in these lists does not matter. + */ +import { orderHubMigrations, type HubMigrationEntry } from "../HubMigrator.ts"; +import Migration001 from "./001_HubBaseline.ts"; + +/** 001–049: hub persistence (orchestration, auth, settings, secrets, attachments). */ +const hubPersistenceMigrations: ReadonlyArray = [ + { id: 1, name: "HubBaseline", migration: Migration001 }, +]; + +/** 050–099: thread-machine state (runner cursors, machine and VCS caches). */ +const threadMachineMigrations: ReadonlyArray = []; + +export const hubMigrations = orderHubMigrations([ + ...hubPersistenceMigrations, + ...threadMachineMigrations, +]); diff --git a/apps/server/src/persistence/ProviderSessionRuntime.ts b/apps/server/src/persistence/ProviderSessionRuntime.ts index 2ccdd862522f..635866e257ca 100644 --- a/apps/server/src/persistence/ProviderSessionRuntime.ts +++ b/apps/server/src/persistence/ProviderSessionRuntime.ts @@ -23,6 +23,8 @@ import { PersistenceSqlError, type ProviderSessionRuntimeRepositoryError, } from "./Errors.ts"; +import { localOrHub } from "./Postgres/HubDatabase.ts"; +import * as PgRepository from "./Postgres/ProviderSessionRuntime.ts"; /** * ProviderSessionRuntimeRepository - Repository interface for provider runtime sessions. @@ -330,4 +332,7 @@ export const make = Effect.gen(function* () { } satisfies ProviderSessionRuntimeRepository["Service"]; }); -export const layer = Layer.effect(ProviderSessionRuntimeRepository, make); +export const layer = localOrHub( + Layer.effect(ProviderSessionRuntimeRepository, make), + Layer.effect(ProviderSessionRuntimeRepository, PgRepository.make), +); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 19b4a480682e..ea996a58625d 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -34,6 +34,7 @@ import { pullRequestHttpApiLayer } from "./pullRequest/http.ts"; import * as PullRequestProviderRegistry from "./pullRequest/PullRequestProviderRegistry.ts"; import * as PullRequestService from "./pullRequest/PullRequestService.ts"; import { layerConfig as SqlitePersistenceLayerLive } from "./persistence/Layers/Sqlite.ts"; +import * as HubDatabase from "./persistence/Postgres/HubDatabase.ts"; import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; import { ProviderSessionDirectoryLive } from "./provider/Layers/ProviderSessionDirectory.ts"; @@ -777,6 +778,8 @@ export const makeServerLayer = Layer.unwrap( Layer.provide(activationLayer), Layer.provideMerge(serverRelayBrokerTracingLayer), Layer.provideMerge(HttpServerLive), + // Hub mode: the tenant's Postgres database, visible to every store below. + Layer.provideMerge(HubDatabase.layerConfig), Layer.provide(ApplicationObservabilityLive), Layer.provideMerge(FetchHttpClient.layer), Layer.provideMerge(VcsProcess.layer), From c094ad2b16e45bb4874e48d0dd5afdd6aca27223 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 22:33:12 +0000 Subject: [PATCH 07/58] feat(server): keep hub settings, keybindings, secrets, ids, and attachments in Postgres A hub's base directory is disposable, so the state a standalone server keeps as files moves to tenant rows in hub mode: - settings.json and keybindings.json go through a StateDocument that is a file (with the directory watcher) standalone and a hub_documents row in a hub, where the owning process's in-process notifications replace the watcher. The provider-history settings query is tenant-scoped. - ServerSecretStore values are AES-256-GCM encrypted with the hub secret key; the additional data binds each value to its tenant and name. - The environment id and anonymous telemetry id are created once per tenant. - Attachment bytes are persisted to hub_attachments when they become durable, restored into the local cache before id lookups, and deleted, pruned, and swept alongside the files. Standalone behavior is unchanged; every branch keys off HubDatabase. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/assets/AssetAccess.ts | 9 + apps/server/src/assets/AttachmentUpload.ts | 9 + apps/server/src/auth/ServerSecretStore.ts | 165 ++++++++- .../src/environment/ServerEnvironment.ts | 42 +++ apps/server/src/keybindings.ts | 42 +-- .../Layers/ProjectionPipeline.ts | 3 + apps/server/src/orchestration/Normalizer.ts | 30 ++ .../persistence/Postgres/HubAttachments.ts | 192 +++++++++++ .../src/persistence/Postgres/HubDocuments.ts | 64 ++++ .../persistence/Postgres/HubSecretCipher.ts | 94 ++++++ .../persistence/Postgres/HubStores.test.ts | 313 ++++++++++++++++++ .../persistence/Postgres/hubTestDatabase.ts | 34 ++ apps/server/src/persistence/StateDocument.ts | 106 ++++++ apps/server/src/serverSettings.ts | 100 +++--- apps/server/src/telemetry/Identify.ts | 37 +++ 15 files changed, 1153 insertions(+), 87 deletions(-) create mode 100644 apps/server/src/persistence/Postgres/HubAttachments.ts create mode 100644 apps/server/src/persistence/Postgres/HubDocuments.ts create mode 100644 apps/server/src/persistence/Postgres/HubSecretCipher.ts create mode 100644 apps/server/src/persistence/Postgres/HubStores.test.ts create mode 100644 apps/server/src/persistence/StateDocument.ts diff --git a/apps/server/src/assets/AssetAccess.ts b/apps/server/src/assets/AssetAccess.ts index 05801acca88f..583889336813 100644 --- a/apps/server/src/assets/AssetAccess.ts +++ b/apps/server/src/assets/AssetAccess.ts @@ -43,6 +43,7 @@ import * as ServerConfig from "../config.ts"; import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; import { openMediaFile, type OpenMediaFile } from "./MediaFile.ts"; +import { hydrateHubAttachment } from "../persistence/Postgres/HubAttachments.ts"; export const ASSET_ROUTE_PREFIX = "/api/assets"; @@ -351,6 +352,10 @@ export const issueAssetUrl = Effect.fn("AssetAccess.issueAssetUrl")(function* (i } case "attachment": { const config = yield* ServerConfig.ServerConfig; + yield* hydrateHubAttachment({ + attachmentsDir: config.attachmentsDir, + attachmentId: input.resource.attachmentId, + }); const attachmentPath = resolveAttachmentPathById({ attachmentsDir: config.attachmentsDir, attachmentId: input.resource.attachmentId, @@ -532,6 +537,10 @@ export const resolveAsset = Effect.fn("AssetAccess.resolveAsset")(function* ( if (claims.kind === "attachment") { const config = yield* ServerConfig.ServerConfig; + yield* hydrateHubAttachment({ + attachmentsDir: config.attachmentsDir, + attachmentId: claims.attachmentId, + }); const attachmentPath = resolveAttachmentPathById({ attachmentsDir: config.attachmentsDir, attachmentId: claims.attachmentId, diff --git a/apps/server/src/assets/AttachmentUpload.ts b/apps/server/src/assets/AttachmentUpload.ts index ba3539a3df40..7b2384b747b3 100644 --- a/apps/server/src/assets/AttachmentUpload.ts +++ b/apps/server/src/assets/AttachmentUpload.ts @@ -19,6 +19,7 @@ import { attachmentFileExtension, createPendingAttachmentId, parseThreadSegmentFromAttachmentId, + PENDING_ATTACHMENT_MAX_AGE_MS, PENDING_ATTACHMENT_THREAD_SEGMENT, resolveAttachmentPathById, sweepStalePendingAttachments, @@ -33,6 +34,11 @@ import { import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as ServerConfig from "../config.ts"; import { inferImageExtension } from "../imageMime.ts"; +import { + persistHubAttachment, + removeHubAttachmentById, + sweepHubPendingAttachments, +} from "../persistence/Postgres/HubAttachments.ts"; export const ATTACHMENT_UPLOAD_ROUTE_PREFIX = "/api/attachments/upload"; @@ -86,6 +92,7 @@ export const issueAttachmentUploadUrl = Effect.fn("AttachmentUpload.issueUrl")(f nowMs - previousSweep >= PENDING_ATTACHMENT_SWEEP_INTERVAL_MS ) { lastPendingSweepByDirectory.set(config.attachmentsDir, nowMs); + yield* sweepHubPendingAttachments(PENDING_ATTACHMENT_MAX_AGE_MS); const swept = sweepStalePendingAttachments({ attachmentsDir: config.attachmentsDir, nowMs, @@ -202,6 +209,7 @@ export const storeAttachmentUpload = Effect.fn("AttachmentUpload.store")(functio } satisfies StoreAttachmentUploadResult; } yield* fileSystem.rename(partPath, finalPath); + yield* persistHubAttachment({ attachmentsDir: config.attachmentsDir, relativePath }); return { ok: true } satisfies StoreAttachmentUploadResult; }).pipe( Effect.catch((cause) => @@ -230,6 +238,7 @@ export const deletePendingAttachment = Effect.fn("AttachmentUpload.deletePending } const config = yield* ServerConfig.ServerConfig; + yield* removeHubAttachmentById(attachmentId); const attachmentPath = resolveAttachmentPathById({ attachmentsDir: config.attachmentsDir, attachmentId, diff --git a/apps/server/src/auth/ServerSecretStore.ts b/apps/server/src/auth/ServerSecretStore.ts index 5e9890c1ea28..c499d72c3d94 100644 --- a/apps/server/src/auth/ServerSecretStore.ts +++ b/apps/server/src/auth/ServerSecretStore.ts @@ -10,6 +10,13 @@ import * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; import * as ServerConfig from "../config.ts"; +import { HubDatabase, type HubDatabaseShape } from "../persistence/Postgres/HubDatabase.ts"; +import { + decryptHubSecret, + encryptHubSecret, + HubSecretKeyError, + parseHubSecretKey, +} from "../persistence/Postgres/HubSecretCipher.ts"; const secretStoreErrorContext = { resource: Schema.String, @@ -310,4 +317,160 @@ export const make = Effect.gen(function* () { }); }); -export const layer = Layer.effect(ServerSecretStore, make); +const hubSecretAlreadyExists = (name: string) => + PlatformError.systemError({ + _tag: "AlreadyExists", + module: "ServerSecretStore", + method: "create", + pathOrDescriptor: name, + }); + +/** + * Hub mode keeps secrets as tenant rows encrypted with the hub secret key (see + * `HubSecretCipher`): the hub's state directory is disposable and shared + * machines must not hold them in plain files. + */ +const makeHub = (hubDatabase: HubDatabaseShape) => + Effect.gen(function* () { + const crypto = yield* Crypto.Crypto; + const serverConfig = yield* ServerConfig.ServerConfig; + const encodedKey = serverConfig.hub?.secretKey; + if (encodedKey === undefined) { + return yield* Effect.die(new HubSecretKeyError()); + } + // An invalid key is a configuration defect: fail startup (without echoing it). + const key = yield* Effect.sync(() => parseHubSecretKey(encodedKey)); + const { sql, tenantId } = hubDatabase; + + const get: ServerSecretStore["Service"]["get"] = (name) => + sql<{ readonly format: number; readonly nonce: Uint8Array; readonly ciphertext: Uint8Array }>` + SELECT format, nonce, ciphertext + FROM hub_secrets + WHERE user_id = ${tenantId} + AND name = ${name} + `.pipe( + Effect.mapError((cause) => new SecretStoreReadError({ resource: `secret ${name}`, cause })), + Effect.flatMap(([row]) => + row === undefined + ? Effect.succeed(Option.none()) + : Effect.try({ + try: () => Option.some(decryptHubSecret({ key, tenantId, name, stored: row })), + catch: (cause) => new SecretStoreDecodeError({ resource: `secret ${name}`, cause }), + }), + ), + Effect.withSpan("ServerSecretStore.get"), + ); + + const encrypt = (name: string, value: Uint8Array) => + Effect.try({ + try: () => encryptHubSecret({ key, tenantId, name, plaintext: value }), + catch: (cause) => new SecretStoreEncodeError({ resource: `secret ${name}`, cause }), + }); + + const set: ServerSecretStore["Service"]["set"] = (name, value) => + encrypt(name, value).pipe( + Effect.flatMap((stored) => + sql` + INSERT INTO hub_secrets (user_id, name, format, nonce, ciphertext, updated_at) + VALUES (${tenantId}, ${name}, ${stored.format}, ${stored.nonce}, ${stored.ciphertext}, now()) + ON CONFLICT (user_id, name) + DO UPDATE SET + format = excluded.format, + nonce = excluded.nonce, + ciphertext = excluded.ciphertext, + updated_at = excluded.updated_at + `.pipe( + Effect.mapError( + (cause) => new SecretStorePersistError({ resource: `secret ${name}`, cause }), + ), + ), + ), + Effect.asVoid, + Effect.withSpan("ServerSecretStore.set"), + ); + + const create: ServerSecretStore["Service"]["create"] = (name, value) => + encrypt(name, value).pipe( + Effect.flatMap((stored) => + sql<{ readonly name: string }>` + INSERT INTO hub_secrets (user_id, name, format, nonce, ciphertext) + VALUES (${tenantId}, ${name}, ${stored.format}, ${stored.nonce}, ${stored.ciphertext}) + ON CONFLICT (user_id, name) DO NOTHING + RETURNING name + `.pipe( + Effect.mapError( + (cause) => new SecretStorePersistError({ resource: `secret ${name}`, cause }), + ), + ), + ), + Effect.flatMap((rows) => + rows.length > 0 + ? Effect.void + : Effect.fail( + new SecretStorePersistError({ + resource: `secret ${name}`, + cause: hubSecretAlreadyExists(name), + }), + ), + ), + Effect.withSpan("ServerSecretStore.create"), + ); + + const getOrCreateRandom: ServerSecretStore["Service"]["getOrCreateRandom"] = (name, bytes) => + get(name).pipe( + Effect.flatMap( + Option.match({ + onSome: Effect.succeed, + onNone: () => + crypto.randomBytes(bytes).pipe( + Effect.mapError( + (cause) => + new SecretStoreRandomGenerationError({ resource: `secret ${name}`, cause }), + ), + Effect.flatMap((generated) => + create(name, generated).pipe( + Effect.as(Uint8Array.from(generated)), + Effect.catchIf(isSecretAlreadyExistsError, () => + get(name).pipe( + Effect.flatMap( + Option.match({ + onSome: Effect.succeed, + onNone: () => + Effect.fail( + new SecretStoreConcurrentReadError({ resource: `secret ${name}` }), + ), + }), + ), + ), + ), + ), + ), + ), + }), + ), + Effect.withSpan("ServerSecretStore.getOrCreateRandom"), + ); + + const remove: ServerSecretStore["Service"]["remove"] = (name) => + sql` + DELETE FROM hub_secrets + WHERE user_id = ${tenantId} + AND name = ${name} + `.pipe( + Effect.mapError( + (cause) => new SecretStoreRemoveError({ resource: `secret ${name}`, cause }), + ), + Effect.asVoid, + Effect.withSpan("ServerSecretStore.remove"), + ); + + return ServerSecretStore.of({ get, set, create, getOrCreateRandom, remove }); + }); + +export const layer = Layer.effect( + ServerSecretStore, + Effect.gen(function* () { + const hubDatabase = yield* HubDatabase; + return hubDatabase === undefined ? yield* make : yield* makeHub(hubDatabase); + }), +); diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index e240e8812f3a..337f745fd91e 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -17,6 +17,8 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { readAgentActivityPublishingActive } from "../cloud/config.ts"; import { resolveServerSelfUpdateCapability } from "../cloud/selfUpdate.ts"; import { resolveServiceLauncherMode } from "../cloud/serviceLauncherClient.ts"; +import { HubDatabase, type HubDatabaseShape } from "../persistence/Postgres/HubDatabase.ts"; +import { makeHubDocuments } from "../persistence/Postgres/HubDocuments.ts"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; import { resolveServerEnvironmentLabel } from "./ServerEnvironmentLabel.ts"; @@ -78,10 +80,50 @@ function platformArch( } } +/** + * A hub's state directory is disposable, so its environment id lives in the + * tenant's documents. Concurrent initializers agree on the first stored id, + * which is what the standalone link-then-recover dance guarantees for files. + */ +const makeHubIdentity = Effect.fn("ServerEnvironmentIdentity.makeHubIdentity")(function* ( + hubDatabase: HubDatabaseShape, + environmentIdPath: string, +) { + const crypto = yield* Crypto.Crypto; + const path = yield* Path.Path; + const generated = yield* crypto.randomUUIDv4; + const environmentId = yield* makeHubDocuments(hubDatabase) + .createIfAbsent(path.basename(environmentIdPath), `${generated}\n`) + .pipe( + Effect.map((value) => value.trim()), + Effect.mapError( + (cause) => + new ServerEnvironmentIdPersistenceError({ + operation: "write", + environmentIdPath, + cause, + }), + ), + ); + if (environmentId.length === 0) { + return yield* new ServerEnvironmentIdPersistenceError({ + operation: "initialize", + environmentIdPath, + }); + } + return ServerEnvironmentIdentity.of({ + getEnvironmentId: Effect.succeed(EnvironmentId.make(environmentId)), + }); +}); + const makeIdentity = Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; const serverConfig = yield* ServerConfig.ServerConfig; const crypto = yield* Crypto.Crypto; + const hubDatabase = yield* HubDatabase; + if (hubDatabase !== undefined) { + return yield* makeHubIdentity(hubDatabase, serverConfig.environmentIdPath); + } const readPersistedEnvironmentId = Effect.gen(function* () { const exists = yield* fileSystem.exists(serverConfig.environmentIdPath).pipe( diff --git a/apps/server/src/keybindings.ts b/apps/server/src/keybindings.ts index 1795808bc179..526404fd974a 100644 --- a/apps/server/src/keybindings.ts +++ b/apps/server/src/keybindings.ts @@ -23,11 +23,8 @@ import * as Array from "effect/Array"; import * as Cache from "effect/Cache"; import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; -import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; -import * as FileSystem from "effect/FileSystem"; -import * as Path from "effect/Path"; import * as Layer from "effect/Layer"; import * as Predicate from "effect/Predicate"; import * as PubSub from "effect/PubSub"; @@ -41,7 +38,7 @@ import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as Semaphore from "effect/Semaphore"; import * as ServerConfig from "./config.ts"; -import { writeFileStringAtomically } from "./atomicWrite.ts"; +import * as StateDocument from "./persistence/StateDocument.ts"; import { fromJsonStringPretty, fromLenientJson } from "@t3tools/shared/schemaJson"; import { DEFAULT_KEYBINDINGS, @@ -284,8 +281,7 @@ export class Keybindings extends Context.Service< const make = Effect.gen(function* () { const { keybindingsConfigPath } = yield* ServerConfig.ServerConfig; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; + const keybindingsDocument = yield* StateDocument.make(keybindingsConfigPath); const upsertSemaphore = yield* Semaphore.make(1); const resolvedConfigCacheKey = "resolved" as const; const changesPubSub = yield* PubSub.unbounded(); @@ -296,7 +292,7 @@ const make = Effect.gen(function* () { const emitChange = (configState: KeybindingsConfigState) => PubSub.publish(changesPubSub, configState).pipe(Effect.asVoid); - const readConfigExists = fs.exists(keybindingsConfigPath).pipe( + const readConfigExists = keybindingsDocument.exists.pipe( Effect.mapError( (cause) => new KeybindingsConfigError({ @@ -307,7 +303,7 @@ const make = Effect.gen(function* () { ), ); - const readRawConfig = fs.readFileString(keybindingsConfigPath).pipe( + const readRawConfig = keybindingsDocument.readString.pipe( Effect.mapError( (cause) => new KeybindingsConfigError({ @@ -421,15 +417,7 @@ const make = Effect.gen(function* () { const writeConfigAtomically = (rules: readonly KeybindingRule[]) => { return encodeKeybindingsConfigPrettyJson(rules).pipe( Effect.map((encoded) => `${encoded}\n`), - Effect.flatMap((encoded) => - writeFileStringAtomically({ - filePath: keybindingsConfigPath, - contents: encoded, - }).pipe( - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(Path.Path, path), - ), - ), + Effect.flatMap((encoded) => keybindingsDocument.writeStringAtomically(encoded)), Effect.mapError( (cause) => new KeybindingsConfigError({ @@ -565,11 +553,7 @@ const make = Effect.gen(function* () { ); const startWatcher = Effect.gen(function* () { - const keybindingsConfigDir = path.dirname(keybindingsConfigPath); - const keybindingsConfigFile = path.basename(keybindingsConfigPath); - const keybindingsConfigPathResolved = path.resolve(keybindingsConfigPath); - - yield* fs.makeDirectory(keybindingsConfigDir, { recursive: true }).pipe( + const debouncedKeybindingsEvents = yield* keybindingsDocument.watchExternalChanges.pipe( Effect.mapError( (cause) => new KeybindingsConfigError({ @@ -582,20 +566,6 @@ const make = Effect.gen(function* () { const revalidateAndEmitSafely = revalidateAndEmit.pipe(Effect.ignoreCause({ log: true })); - // Debounce watch events so the file is fully written before we read it. - // Editors emit multiple events per save (truncate, write, rename) and - // `fs.watch` can fire before the content has been flushed to disk. - const debouncedKeybindingsEvents = fs.watch(keybindingsConfigDir).pipe( - Stream.filter((event) => { - return ( - event.path === keybindingsConfigFile || - event.path === keybindingsConfigPath || - path.resolve(keybindingsConfigDir, event.path) === keybindingsConfigPathResolved - ); - }), - Stream.debounce(Duration.millis(100)), - ); - yield* Stream.runForEach(debouncedKeybindingsEvents, () => revalidateAndEmitSafely).pipe( Effect.ignoreCause({ log: true }), Effect.forkIn(watcherScope), diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts index 1a8afcac8520..09ef7733dbd9 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts @@ -54,6 +54,7 @@ import { parseThreadSegmentFromAttachmentId, toSafeThreadAttachmentSegment, } from "../../attachmentStore.ts"; +import { removeHubThreadAttachments } from "../../persistence/Postgres/HubAttachments.ts"; export const ORCHESTRATION_PROJECTOR_NAMES = { projects: "projection.projects", @@ -417,6 +418,7 @@ const runAttachmentSideEffects = Effect.fn("runAttachmentSideEffects")(function* return; } + yield* removeHubThreadAttachments(threadSegment); const entries = yield* readAttachmentRootEntries; yield* Effect.forEach( entries, @@ -470,6 +472,7 @@ const runAttachmentSideEffects = Effect.fn("runAttachmentSideEffects")(function* return; } + yield* removeHubThreadAttachments(threadSegment, keptThreadRelativePaths); const entries = yield* readAttachmentRootEntries; yield* Effect.forEach( entries, diff --git a/apps/server/src/orchestration/Normalizer.ts b/apps/server/src/orchestration/Normalizer.ts index 1226a6cd25d5..fa81b16afb48 100644 --- a/apps/server/src/orchestration/Normalizer.ts +++ b/apps/server/src/orchestration/Normalizer.ts @@ -18,6 +18,11 @@ import { resolveAttachmentPath, } from "../attachmentStore.ts"; import { ServerConfig } from "../config.ts"; +import { + hydrateHubAttachment, + persistHubAttachment, + removeHubAttachments, +} from "../persistence/Postgres/HubAttachments.ts"; import { parseBase64DataUrl } from "../imageMime.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; @@ -55,6 +60,11 @@ const removeClaimedAttachmentPaths = Effect.fn("Normalizer.removeClaimedAttachme return; } const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { attachmentsDir } = yield* ServerConfig; + yield* removeHubAttachments( + attachmentPaths.map((attachmentPath) => path.relative(attachmentsDir, attachmentPath)), + ); yield* Effect.forEach( attachmentPaths, (attachmentPath) => @@ -133,12 +143,30 @@ export const normalizeDispatchCommand = (command: ClientOrchestrationCommand) => return canonicalCommand as OrchestrationCommand; } + // Hub mode keeps attachment bytes in Postgres; a no-op otherwise. + const persistAttachment = (attachmentPath: string, name: string) => + persistHubAttachment({ + attachmentsDir: serverConfig.attachmentsDir, + relativePath: path.relative(serverConfig.attachmentsDir, attachmentPath), + }).pipe( + Effect.mapError( + (cause) => + new OrchestrationDispatchCommandError({ + message: `Failed to persist attachment '${name}'.`, + cause, + }), + ), + ); const claimedAttachmentPaths: string[] = []; const normalizedAttachments = yield* Effect.forEach( canonicalCommand.message.attachments, (attachment) => Effect.gen(function* () { if (!("dataUrl" in attachment)) { + yield* hydrateHubAttachment({ + attachmentsDir: serverConfig.attachmentsDir, + attachmentId: attachment.id, + }); const claim = planAttachmentClaim({ attachmentsDir: serverConfig.attachmentsDir, threadId: canonicalCommand.threadId, @@ -194,6 +222,7 @@ export const normalizeDispatchCommand = (command: ClientOrchestrationCommand) => ), ); claimedAttachmentPaths.push(claim.finalPath); + yield* persistAttachment(claim.finalPath, attachment.name); return normalizedAttachment; } @@ -253,6 +282,7 @@ export const normalizeDispatchCommand = (command: ClientOrchestrationCommand) => }), ), ); + yield* persistAttachment(attachmentPath, attachment.name); return persistedAttachment; }), diff --git a/apps/server/src/persistence/Postgres/HubAttachments.ts b/apps/server/src/persistence/Postgres/HubAttachments.ts new file mode 100644 index 000000000000..fa581428b5b4 --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubAttachments.ts @@ -0,0 +1,192 @@ +/** + * HubAttachments - durable attachment bytes for a hub. + * + * Attachment code works with files under `ServerConfig.attachmentsDir`. In a + * hub that directory is a disposable cache and `hub_attachments` holds the + * bytes: a file is persisted when it becomes durable (an upload completes, a + * pending upload is claimed by a turn, an inline image is written), restored + * into the cache before it is looked up by id, and deleted when the files are. + * Outside hub mode every helper here is a no-op. + * + * Bytes live in `bytea` within the existing upload limits for now; object + * storage can replace the table behind the same helpers. + * + * @module HubAttachments + */ +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import { + parseAttachmentIdFromRelativePath, + parseThreadSegmentFromAttachmentId, + PENDING_ATTACHMENT_THREAD_SEGMENT, + resolveAttachmentPathById, +} from "../../attachmentStore.ts"; +import { + normalizeAttachmentRelativePath, + resolveAttachmentRelativePath, +} from "../../attachmentPaths.ts"; +import { HubDatabase, type HubDatabaseShape } from "./HubDatabase.ts"; + +const withHubDatabase = ( + use: (database: HubDatabaseShape) => Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + const database = yield* HubDatabase; + if (database !== undefined) { + yield* use(database); + } + }); + +const logFailure = + (message: string, annotations: Record) => + (effect: Effect.Effect) => + effect.pipe( + Effect.asVoid, + Effect.catchCause((cause) => Effect.logWarning(message, { ...annotations, cause })), + ); + +/** + * Stores a finished attachment file (`relativePath` under `attachmentsDir`). + * Fails when the bytes cannot be made durable. + */ +export const persistHubAttachment = (input: { + readonly attachmentsDir: string; + readonly relativePath: string; +}) => + withHubDatabase(({ sql, tenantId }) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const relativePath = normalizeAttachmentRelativePath(input.relativePath); + const attachmentId = relativePath ? parseAttachmentIdFromRelativePath(relativePath) : null; + const threadSegment = attachmentId ? parseThreadSegmentFromAttachmentId(attachmentId) : null; + if (!relativePath || !attachmentId || !threadSegment) { + return yield* Effect.die(new Error("Refusing to persist an invalid attachment path.")); + } + const content = yield* fileSystem.readFile(path.join(input.attachmentsDir, relativePath)); + yield* sql` + INSERT INTO hub_attachments ( + user_id, + relative_path, + attachment_id, + thread_segment, + size_bytes, + content, + created_at + ) + VALUES ( + ${tenantId}, + ${relativePath}, + ${attachmentId}, + ${threadSegment}, + ${content.byteLength}, + ${content}, + now() + ) + ON CONFLICT (user_id, relative_path) + DO UPDATE SET + size_bytes = excluded.size_bytes, + content = excluded.content, + created_at = excluded.created_at + `; + }), + ); + +/** + * Restores an attachment into the local cache when it is not already there, + * so id-based lookups that follow find the file. Best effort: a failure is + * logged and the lookup then reports the attachment as missing. + */ +export const hydrateHubAttachment = (input: { + readonly attachmentsDir: string; + readonly attachmentId: string; +}) => + withHubDatabase(({ sql, tenantId }) => + Effect.gen(function* () { + if (resolveAttachmentPathById(input) !== null) { + return; + } + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const rows = yield* sql<{ readonly relativePath: string; readonly content: Uint8Array }>` + SELECT relative_path AS "relativePath", content + FROM hub_attachments + WHERE user_id = ${tenantId} + AND attachment_id = ${input.attachmentId} + `; + for (const row of rows) { + const filePath = resolveAttachmentRelativePath({ + attachmentsDir: input.attachmentsDir, + relativePath: row.relativePath, + }); + if (!filePath) { + continue; + } + yield* fileSystem.makeDirectory(path.dirname(filePath), { recursive: true }); + // Write beside the target and rename so a concurrent reader never sees + // a partial file. + const partPath = `${filePath}.${process.pid}.hydrate.part`; + yield* fileSystem.writeFile(partPath, row.content); + yield* fileSystem.rename(partPath, filePath); + } + }).pipe( + logFailure("Failed to restore a hub attachment.", { attachmentId: input.attachmentId }), + ), + ); + +/** Deletes stored attachments by relative path. Best effort, like file removal. */ +export const removeHubAttachments = (relativePaths: ReadonlyArray) => + withHubDatabase(({ sql, tenantId }) => { + const normalized = relativePaths.flatMap((relativePath) => { + const value = normalizeAttachmentRelativePath(relativePath); + return value ? [value] : []; + }); + return normalized.length === 0 + ? Effect.void + : sql` + DELETE FROM hub_attachments + WHERE user_id = ${tenantId} + AND relative_path IN ${sql.in(normalized)} + `.pipe(logFailure("Failed to delete hub attachments.", { count: normalized.length })); + }); + +/** Deletes a stored attachment by id (every extension it was stored under). */ +export const removeHubAttachmentById = (attachmentId: string) => + withHubDatabase(({ sql, tenantId }) => + sql` + DELETE FROM hub_attachments + WHERE user_id = ${tenantId} + AND attachment_id = ${attachmentId} + `.pipe(logFailure("Failed to delete a hub attachment.", { attachmentId })), + ); + +/** + * Deletes a thread's stored attachments, keeping `keptRelativePaths` when + * given (a revert prunes only what the remaining messages no longer use). + */ +export const removeHubThreadAttachments = ( + threadSegment: string, + keptRelativePaths?: ReadonlySet, +) => + withHubDatabase(({ sql, tenantId }) => { + const kept = [...(keptRelativePaths ?? [])]; + return sql` + DELETE FROM hub_attachments + WHERE user_id = ${tenantId} + AND thread_segment = ${threadSegment} + ${kept.length > 0 ? sql`AND relative_path NOT IN ${sql.in(kept)}` : sql``} + `.pipe(logFailure("Failed to delete a thread's hub attachments.", { threadSegment })); + }); + +/** Deletes pending uploads older than `maxAgeMs` that no turn claimed. */ +export const sweepHubPendingAttachments = (maxAgeMs: number) => + withHubDatabase(({ sql, tenantId }) => + sql` + DELETE FROM hub_attachments + WHERE user_id = ${tenantId} + AND thread_segment = ${PENDING_ATTACHMENT_THREAD_SEGMENT} + AND created_at < now() - make_interval(secs => ${maxAgeMs / 1000}) + `.pipe(logFailure("Failed to sweep pending hub attachments.", {})), + ); diff --git a/apps/server/src/persistence/Postgres/HubDocuments.ts b/apps/server/src/persistence/Postgres/HubDocuments.ts new file mode 100644 index 000000000000..c247ddb06a1d --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubDocuments.ts @@ -0,0 +1,64 @@ +/** + * HubDocuments - the small documents a standalone server keeps as files in its + * state directory (settings.json, keybindings.json, environment-id, + * anonymous-id), stored as tenant rows in `hub_documents`. + * + * Documents are keyed by the file's base name, so `t3 hub import` can copy a + * standalone state directory verbatim. + * + * @module HubDocuments + */ +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import type { HubDatabaseShape } from "./HubDatabase.ts"; + +export const makeHubDocuments = (database: HubDatabaseShape) => { + const { sql, tenantId } = database; + + const read = (name: string) => + sql<{ readonly contents: string }>` + SELECT contents + FROM hub_documents + WHERE user_id = ${tenantId} + AND name = ${name} + `.pipe(Effect.map((rows) => Option.fromNullishOr(rows[0]?.contents))); + + const write = (name: string, contents: string) => + sql` + INSERT INTO hub_documents (user_id, name, contents, updated_at) + VALUES (${tenantId}, ${name}, ${contents}, now()) + ON CONFLICT (user_id, name) + DO UPDATE SET contents = excluded.contents, updated_at = excluded.updated_at + `.pipe(Effect.asVoid); + + /** + * Stores `contents` unless the document exists and returns the stored value, + * so concurrent initializers agree on one winner. The read is a separate + * statement so it sees a winner that committed while the insert waited. + */ + const createIfAbsent = (name: string, contents: string) => + sql + .withTransaction( + Effect.gen(function* () { + yield* sql` + INSERT INTO hub_documents (user_id, name, contents) + VALUES (${tenantId}, ${name}, ${contents}) + ON CONFLICT (user_id, name) DO NOTHING + `; + return yield* read(name); + }), + ) + .pipe( + Effect.flatMap( + Option.match({ + onNone: () => Effect.die(new Error(`Hub document ${name} vanished during creation.`)), + onSome: Effect.succeed, + }), + ), + ); + + return { read, write, createIfAbsent }; +}; + +export type HubDocuments = ReturnType; diff --git a/apps/server/src/persistence/Postgres/HubSecretCipher.ts b/apps/server/src/persistence/Postgres/HubSecretCipher.ts new file mode 100644 index 000000000000..227f95143a1c --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubSecretCipher.ts @@ -0,0 +1,94 @@ +/** + * HubSecretCipher - encryption of `ServerSecretStore` values at rest in the hub. + * + * AES-256-GCM with a random 96-bit nonce per write. The additional + * authenticated data binds each ciphertext to its tenant and secret name, so a + * row copied to another tenant or renamed fails to decrypt. The key is + * `T3CODE_HUB_SECRET_KEY` (base64, 32 bytes) and never reaches the database. + * + * @module HubSecretCipher + */ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeCrypto from "node:crypto"; + +/** Stored in `hub_secrets.format`; bump when the scheme or key changes. */ +export const HUB_SECRET_FORMAT_AES_256_GCM_V1 = 1; +const NONCE_BYTES = 12; +const TAG_BYTES = 16; + +export class HubSecretKeyError extends Error { + constructor() { + super("T3CODE_HUB_SECRET_KEY must be a base64-encoded 32-byte key."); + this.name = "HubSecretKeyError"; + } +} + +/** Decodes the configured key; throws `HubSecretKeyError` without echoing it. */ +export const parseHubSecretKey = (encoded: string): Buffer => { + const trimmed = encoded.trim(); + if (!/^[A-Za-z0-9+/_-]+={0,2}$/.test(trimmed)) { + throw new HubSecretKeyError(); + } + const key = Buffer.from(trimmed.replaceAll("-", "+").replaceAll("_", "/"), "base64"); + if (key.length !== 32) { + throw new HubSecretKeyError(); + } + return key; +}; + +const additionalData = (tenantId: string, name: string) => + Buffer.from(`t3-hub-secret\u0000v1\u0000${tenantId}\u0000${name}`, "utf8"); + +export interface HubSecretCiphertext { + readonly format: number; + readonly nonce: Uint8Array; + /** Ciphertext followed by the 16-byte GCM tag. */ + readonly ciphertext: Uint8Array; +} + +export const encryptHubSecret = (input: { + readonly key: Buffer; + readonly tenantId: string; + readonly name: string; + readonly plaintext: Uint8Array; +}): HubSecretCiphertext => { + const nonce = NodeCrypto.randomBytes(NONCE_BYTES); + const cipher = NodeCrypto.createCipheriv("aes-256-gcm", input.key, nonce, { + authTagLength: TAG_BYTES, + }); + cipher.setAAD(additionalData(input.tenantId, input.name)); + const ciphertext = Buffer.concat([ + cipher.update(input.plaintext), + cipher.final(), + cipher.getAuthTag(), + ]); + return { format: HUB_SECRET_FORMAT_AES_256_GCM_V1, nonce, ciphertext }; +}; + +/** Throws when the format is unknown or authentication fails. */ +export const decryptHubSecret = (input: { + readonly key: Buffer; + readonly tenantId: string; + readonly name: string; + readonly stored: HubSecretCiphertext; +}): Uint8Array => { + const { format, nonce, ciphertext } = input.stored; + if (format !== HUB_SECRET_FORMAT_AES_256_GCM_V1) { + throw new Error(`Unsupported hub secret format ${format}.`); + } + if (nonce.byteLength !== NONCE_BYTES || ciphertext.byteLength < TAG_BYTES) { + throw new Error("Malformed hub secret ciphertext."); + } + const body = Buffer.from(ciphertext); + const decipher = NodeCrypto.createDecipheriv("aes-256-gcm", input.key, Buffer.from(nonce), { + authTagLength: TAG_BYTES, + }); + decipher.setAAD(additionalData(input.tenantId, input.name)); + decipher.setAuthTag(body.subarray(body.byteLength - TAG_BYTES)); + return Uint8Array.from( + Buffer.concat([ + decipher.update(body.subarray(0, body.byteLength - TAG_BYTES)), + decipher.final(), + ]), + ); +}; diff --git a/apps/server/src/persistence/Postgres/HubStores.test.ts b/apps/server/src/persistence/Postgres/HubStores.test.ts new file mode 100644 index 000000000000..637703c97e33 --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubStores.test.ts @@ -0,0 +1,313 @@ +/** + * Hub replacements for state-directory files: secrets, settings, keybindings, + * environment and anonymous ids, and attachments. Runs only with + * T3_HUB_TEST_DATABASE_URL (see `hubTestDatabase.ts`). + */ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts"; +import { assert, describe, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Stream from "effect/Stream"; +import * as Reactivity from "effect/unstable/reactivity/Reactivity"; + +import { createPendingAttachmentId } from "../../attachmentStore.ts"; +import * as ServerSecretStore from "../../auth/ServerSecretStore.ts"; +import * as ServerConfig from "../../config.ts"; +import * as ServerEnvironment from "../../environment/ServerEnvironment.ts"; +import * as Keybindings from "../../keybindings.ts"; +import * as ServerSettings from "../../serverSettings.ts"; +import { getTelemetryIdentifierForHome } from "../../telemetry/Identify.ts"; +import { layerConfig as ServerPersistenceLive } from "../Layers/Sqlite.ts"; +import { + hydrateHubAttachment, + persistHubAttachment, + removeHubAttachmentById, + removeHubThreadAttachments, + sweepHubPendingAttachments, +} from "./HubAttachments.ts"; +import { HubDatabase } from "./HubDatabase.ts"; +import { + HUB_TEST_SECRET_KEY, + hubTestDatabaseLayer, + hubTestDatabaseUrl, + hubTestServerConfigLayer, + makeHubTestSchema, + type HubTestSchema, +} from "./hubTestDatabase.ts"; + +const withSchema = (use: (schema: HubTestSchema) => Effect.Effect) => + Effect.scoped(Effect.flatMap(makeHubTestSchema(hubTestDatabaseUrl!), use)).pipe( + Effect.provide(Reactivity.layer), + ); + +/** One hub process for `tenantId`: its database and hub-mode config. */ +const hubProcess = ( + schema: HubTestSchema, + tenantId: string, + options: { readonly secretKey?: string } = {}, +) => + Layer.mergeAll( + hubTestDatabaseLayer(schema, tenantId), + hubTestServerConfigLayer(schema, tenantId, options), + ); + +const secretStoreLayer = (schema: HubTestSchema, tenantId: string, secretKey?: string) => + ServerSecretStore.layer.pipe( + Layer.provideMerge(hubProcess(schema, tenantId, secretKey ? { secretKey } : {})), + ); + +const stateFileExists = (select: (config: ServerConfig.ServerConfig["Service"]) => string) => + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fileSystem = yield* FileSystem.FileSystem; + return yield* fileSystem.exists(select(config)); + }); + +describe.skipIf(hubTestDatabaseUrl === undefined)("hub stores", () => { + it.effect("encrypts secrets per tenant and name", () => + withSchema((schema) => + Effect.gen(function* () { + const plaintext = new TextEncoder().encode("session-signing-key-material"); + + const stored = yield* Effect.gen(function* () { + const secrets = yield* ServerSecretStore.ServerSecretStore; + yield* secrets.set("session-signing-key", plaintext); + assert.deepStrictEqual( + Option.getOrThrow(yield* secrets.get("session-signing-key")), + plaintext, + ); + const duplicate = yield* secrets + .create("session-signing-key", plaintext) + .pipe(Effect.exit); + assert.isTrue( + Exit.isFailure(duplicate) && + duplicate.cause.reasons.some( + (reason) => + reason._tag === "Fail" && + ServerSecretStore.isSecretAlreadyExistsError(reason.error), + ), + ); + const racers = yield* Effect.all( + Array.from({ length: 5 }, () => secrets.getOrCreateRandom("asset-signing-key", 32)), + { concurrency: "unbounded" }, + ); + assert.strictEqual( + new Set(racers.map((value) => Buffer.from(value).toString("hex"))).size, + 1, + ); + assert.strictEqual(racers[0]?.byteLength, 32); + + const sql = (yield* HubDatabase)!.sql; + const rows = yield* sql<{ + readonly format: number; + readonly nonce: Uint8Array; + readonly ciphertext: Uint8Array; + }>`SELECT format, nonce, ciphertext FROM hub_secrets WHERE user_id = 'tenant-a' AND name = 'session-signing-key'`; + const row = rows[0]!; + assert.strictEqual(row.format, 1); + assert.strictEqual(row.nonce.byteLength, 12); + assert.isFalse(Buffer.from(row.ciphertext).includes(Buffer.from(plaintext))); + + yield* secrets.remove("asset-signing-key"); + assert.isTrue(Option.isNone(yield* secrets.get("asset-signing-key"))); + return row; + }).pipe(Effect.provide(secretStoreLayer(schema, "tenant-a"))); + + // A row moved to another tenant does not authenticate there. + yield* Effect.gen(function* () { + const sql = (yield* HubDatabase)!.sql; + yield* sql` + INSERT INTO hub_secrets (user_id, name, format, nonce, ciphertext) + VALUES ('tenant-b', 'session-signing-key', ${stored.format}, ${stored.nonce}, ${stored.ciphertext}) + `; + const secrets = yield* ServerSecretStore.ServerSecretStore; + const read = yield* secrets.get("session-signing-key").pipe(Effect.exit); + assert.isTrue(Exit.isFailure(read)); + }).pipe(Effect.provide(secretStoreLayer(schema, "tenant-b"))); + + // Nor does it decrypt with another key. + const otherKey = Buffer.alloc(32, 9).toString("base64"); + yield* Effect.gen(function* () { + const secrets = yield* ServerSecretStore.ServerSecretStore; + const read = yield* secrets.get("session-signing-key").pipe(Effect.exit); + assert.isTrue(Exit.isFailure(read)); + }).pipe(Effect.provide(secretStoreLayer(schema, "tenant-a", otherKey))); + assert.notStrictEqual(otherKey, HUB_TEST_SECRET_KEY); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); + + it.effect("keeps settings, keybindings, and ids in Postgres across restarts", () => + withSchema((schema) => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("codex_personal"); + const stores = Layer.mergeAll( + ServerSettings.layer, + Keybindings.layer, + ServerEnvironment.identityLayer, + ).pipe( + Layer.provideMerge(ServerSecretStore.layer), + Layer.provideMerge(ServerPersistenceLive), + Layer.provideMerge(hubProcess(schema, "tenant-a")), + ); + + const first = yield* Effect.gen(function* () { + const settings = yield* ServerSettings.ServerSettingsService; + const keybindings = yield* Keybindings.Keybindings; + const identity = yield* ServerEnvironment.ServerEnvironmentIdentity; + yield* settings.start; + yield* keybindings.start; + + const changes = yield* settings.subscribeChanges; + yield* settings.updateSettings({ + enableLegacyTokenStreaming: true, + providerInstances: { + [instanceId]: { + driver: ProviderDriverKind.make("codex"), + environment: [ + { name: "OPENROUTER_API_KEY", value: "sk-or-secret", sensitive: true }, + ], + config: {}, + }, + }, + }); + // One process owns the tenant, so in-process notification is the watcher. + const changed = yield* Stream.runHead(changes); + assert.isTrue(Option.isSome(changed) && changed.value.enableLegacyTokenStreaming); + yield* keybindings.upsertKeybindingRule({ + key: "mod+shift+r", + command: "script.run-tests.run", + }); + + assert.isFalse(yield* stateFileExists((config) => config.settingsPath)); + assert.isFalse(yield* stateFileExists((config) => config.keybindingsConfigPath)); + assert.isFalse(yield* stateFileExists((config) => config.environmentIdPath)); + assert.isFalse(yield* stateFileExists((config) => config.secretsDir)); + assert.isFalse(yield* stateFileExists((config) => config.dbPath)); + + const sql = (yield* HubDatabase)!.sql; + const documents = yield* sql<{ readonly name: string; readonly contents: string }>` + SELECT name, contents FROM hub_documents WHERE user_id = 'tenant-a' ORDER BY name + `; + assert.deepStrictEqual( + documents.map((document) => document.name), + ["environment-id", "keybindings.json", "settings.json"], + ); + assert.notInclude( + documents.map((document) => document.contents).join("\n"), + "sk-or-secret", + ); + return { environmentId: yield* identity.getEnvironmentId }; + }).pipe(Effect.provide(Layer.fresh(stores))); + + // A new process on a new, empty base directory sees the same state. + yield* Effect.gen(function* () { + const settings = yield* ServerSettings.ServerSettingsService; + const keybindings = yield* Keybindings.Keybindings; + const identity = yield* ServerEnvironment.ServerEnvironmentIdentity; + yield* settings.start; + yield* keybindings.start; + + const current = yield* settings.getSettings; + assert.isTrue(current.enableLegacyTokenStreaming); + assert.strictEqual( + current.providerInstances[instanceId]?.environment?.[0]?.value, + "sk-or-secret", + ); + const snapshot = yield* keybindings.getSnapshot; + assert.isTrue( + snapshot.keybindings.some((binding) => binding.command === "script.run-tests.run"), + ); + assert.strictEqual(yield* identity.getEnvironmentId, first.environmentId); + }).pipe(Effect.provide(Layer.fresh(stores))); + + // The anonymous telemetry id is a hub document too. + const home = yield* Effect.flatMap(FileSystem.FileSystem, (fileSystem) => + fileSystem.makeTempDirectoryScoped({ prefix: "t3-hub-home-" }), + ); + const identify = getTelemetryIdentifierForHome(home).pipe( + Effect.provide(hubProcess(schema, "tenant-a")), + ); + const firstId = yield* identify; + assert.isString(firstId); + assert.strictEqual(yield* identify, firstId); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); + + it.effect("stores attachment bytes durably behind the local cache", () => + withSchema((schema) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfig.ServerConfig; + const sql = (yield* HubDatabase)!.sql; + const bytes = new Uint8Array([0, 1, 2, 3, 250, 251, 252, 253]); + + const pendingId = createPendingAttachmentId("pdf"); + const pendingPath = `${pendingId}.pdf`; + yield* fileSystem.writeFile(path.join(config.attachmentsDir, pendingPath), bytes); + yield* persistHubAttachment({ + attachmentsDir: config.attachmentsDir, + relativePath: pendingPath, + }); + + // The cache is disposable: a lookup after losing it restores the bytes. + yield* fileSystem.remove(path.join(config.attachmentsDir, pendingPath)); + yield* hydrateHubAttachment({ + attachmentsDir: config.attachmentsDir, + attachmentId: pendingId, + }); + assert.deepStrictEqual( + new Uint8Array(yield* fileSystem.readFile(path.join(config.attachmentsDir, pendingPath))), + bytes, + ); + + const threadFile = (name: string) => `thread-1-${name}.png`; + for (const name of [ + "00000000-0000-4000-8000-000000000001", + "00000000-0000-4000-8000-000000000002", + ]) { + yield* fileSystem.writeFile(path.join(config.attachmentsDir, threadFile(name)), bytes); + yield* persistHubAttachment({ + attachmentsDir: config.attachmentsDir, + relativePath: threadFile(name), + }); + } + const listThread = sql<{ readonly relativePath: string }>` + SELECT relative_path AS "relativePath" FROM hub_attachments + WHERE user_id = 'tenant-a' AND thread_segment = 'thread-1' ORDER BY relative_path + `.pipe(Effect.map((rows) => rows.map((row) => row.relativePath))); + assert.strictEqual((yield* listThread).length, 2); + yield* removeHubThreadAttachments( + "thread-1", + new Set([threadFile("00000000-0000-4000-8000-000000000002")]), + ); + assert.deepStrictEqual(yield* listThread, [ + threadFile("00000000-0000-4000-8000-000000000002"), + ]); + yield* removeHubThreadAttachments("thread-1"); + assert.deepStrictEqual(yield* listThread, []); + + // Unclaimed pending uploads expire. + yield* sweepHubPendingAttachments(60_000); + const countPending = sql<{ readonly count: number }>` + SELECT count(*) AS count FROM hub_attachments WHERE thread_segment = 'pending' + `.pipe(Effect.map((rows) => rows[0]?.count)); + assert.strictEqual(yield* countPending, 1); + yield* sql`UPDATE hub_attachments SET created_at = now() - interval '2 days'`; + yield* sweepHubPendingAttachments(24 * 60 * 60 * 1000); + assert.strictEqual(yield* countPending, 0); + + yield* removeHubAttachmentById(pendingId); + }).pipe( + Effect.provide(hubProcess(schema, "tenant-a").pipe(Layer.provideMerge(NodeServices.layer))), + ), + ), + ); +}); diff --git a/apps/server/src/persistence/Postgres/hubTestDatabase.ts b/apps/server/src/persistence/Postgres/hubTestDatabase.ts index fbd4dfe7e26e..5d2af544de62 100644 --- a/apps/server/src/persistence/Postgres/hubTestDatabase.ts +++ b/apps/server/src/persistence/Postgres/hubTestDatabase.ts @@ -21,6 +21,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Reactivity from "effect/unstable/reactivity/Reactivity"; +import * as ServerConfig from "../../config.ts"; import { HubDatabase } from "./HubDatabase.ts"; import { makeHubDatabase } from "./HubDatabaseLive.ts"; import { makeHubPool } from "./HubClient.ts"; @@ -110,3 +111,36 @@ export const hubTestDatabaseLayer = (schema: HubTestSchema, tenantId: string) => tenantId, }), ); + +/** A fixed test-only key; never use outside tests. */ +export const HUB_TEST_SECRET_KEY = Buffer.alloc(32, 7).toString("base64"); + +/** + * A test `ServerConfig` in hub mode for `tenantId`, over a temporary base + * directory (which a hub treats as disposable). + */ +export const hubTestServerConfigLayer = ( + schema: HubTestSchema, + tenantId: string, + options: { readonly baseDir?: string; readonly secretKey?: string } = {}, +) => + Layer.effect( + ServerConfig.ServerConfig, + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + return { + ...config, + serverMode: "hub" as const, + hub: { + databaseUrl: schema.runtimeUrl, + databaseAdminUrl: schema.separateRuntimeRole ? schema.adminUrl : undefined, + tenantId, + secretKey: options.secretKey ?? HUB_TEST_SECRET_KEY, + }, + }; + }), + ).pipe( + Layer.provide( + ServerConfig.layerTest(process.cwd(), options.baseDir ?? { prefix: "t3-hub-test-" }), + ), + ); diff --git a/apps/server/src/persistence/StateDocument.ts b/apps/server/src/persistence/StateDocument.ts new file mode 100644 index 000000000000..acb18bff1202 --- /dev/null +++ b/apps/server/src/persistence/StateDocument.ts @@ -0,0 +1,106 @@ +/** + * StateDocument - a small durable text document of the server's state, such as + * settings.json or keybindings.json. + * + * A standalone server keeps it as a file and watches the directory for edits + * made outside the process. A hub keeps it as a Postgres row (see + * `HubDocuments`); one process owns a tenant, so every change already passes + * through the service that wrote it and there is nothing to watch. + * + * @module StateDocument + */ +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; +import * as Stream from "effect/Stream"; +import type { SqlError } from "effect/unstable/sql/SqlError"; + +import { writeFileStringAtomically } from "../atomicWrite.ts"; +import { HubDatabase } from "./Postgres/HubDatabase.ts"; +import { makeHubDocuments } from "./Postgres/HubDocuments.ts"; + +export type StateDocumentError = PlatformError.PlatformError | SqlError; + +export interface StateDocument { + readonly exists: Effect.Effect; + readonly readString: Effect.Effect; + readonly writeStringAtomically: (contents: string) => Effect.Effect; + /** + * Prepares change detection and returns a stream that emits (debounced) + * after another writer changes the document. + */ + readonly watchExternalChanges: Effect.Effect< + Stream.Stream, + StateDocumentError + >; +} + +const makeFileStateDocument = ( + fs: FileSystem.FileSystem, + path: Path.Path, + filePath: string, +): StateDocument => ({ + exists: fs.exists(filePath), + readString: fs.readFileString(filePath), + writeStringAtomically: (contents) => + writeFileStringAtomically({ filePath, contents }).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ), + watchExternalChanges: Effect.gen(function* () { + const directory = path.dirname(filePath); + const fileName = path.basename(filePath); + const resolvedPath = path.resolve(filePath); + yield* fs.makeDirectory(directory, { recursive: true }); + // Debounce watch events so the file is fully written before we read it. + // Editors emit multiple events per save (truncate, write, rename) and + // `fs.watch` can fire before the content has been flushed to disk. + return fs.watch(directory).pipe( + Stream.filter( + (event) => + event.path === fileName || + event.path === filePath || + path.resolve(directory, event.path) === resolvedPath, + ), + Stream.debounce(Duration.millis(100)), + Stream.map(() => undefined), + ); + }), +}); + +/** The document at `filePath`, or its hub row (keyed by the file name) in hub mode. */ +export const make = Effect.fn("StateDocument.make")(function* (filePath: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const hubDatabase = yield* HubDatabase; + if (hubDatabase === undefined) { + return makeFileStateDocument(fs, path, filePath); + } + + const documents = makeHubDocuments(hubDatabase); + const name = path.basename(filePath); + return { + exists: documents.read(name).pipe(Effect.map(Option.isSome)), + readString: documents.read(name).pipe( + Effect.flatMap( + Option.match({ + onSome: Effect.succeed, + onNone: () => + Effect.fail( + PlatformError.systemError({ + _tag: "NotFound", + module: "StateDocument", + method: "readString", + pathOrDescriptor: name, + }), + ), + }), + ), + ), + writeStringAtomically: (contents) => documents.write(name, contents), + watchExternalChanges: Effect.succeed(Stream.empty), + } satisfies StateDocument; +}); diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index 5a8650b7e405..6ed732ea7988 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -6,7 +6,7 @@ * text generation model selection). * * Follows the same pattern as `keybindings.ts`: JSON file + Cache + PubSub + - * Semaphore + FileSystem.watch for concurrency and external edit detection. + * Semaphore + StateDocument watch for concurrency and external edit detection. * * @module ServerSettings */ @@ -32,10 +32,8 @@ import * as Duration from "effect/Duration"; import * as Equal from "effect/Equal"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; -import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; @@ -43,7 +41,6 @@ import * as Semaphore from "effect/Semaphore"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as SqlClient from "effect/unstable/sql/SqlClient"; -import { writeFileStringAtomically } from "./atomicWrite.ts"; import * as ServerConfig from "./config.ts"; import { type DeepPartial, deepMerge } from "@t3tools/shared/Struct"; import { fromJsonStringPretty, fromLenientJson } from "@t3tools/shared/schemaJson"; @@ -52,6 +49,8 @@ import { isModelSelectionProviderEnabled, } from "@t3tools/shared/serverSettings"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; +import { HubDatabase, type HubDatabaseShape } from "./persistence/Postgres/HubDatabase.ts"; +import * as StateDocument from "./persistence/StateDocument.ts"; export { resolveSourceControlWriterModelSelection } from "@t3tools/shared/serverSettings"; @@ -371,10 +370,34 @@ function stripDefaultServerSettings(current: unknown, defaults: unknown): unknow return Object.is(current, defaults) ? undefined : current; } +interface ProviderHistoryRow { + readonly providerName: string; + readonly providerInstanceId: string | null; +} + +// The standalone query reads every row of both tables; a hub database holds +// every tenant, so it filters on the tenant it serves. +const readHubProviderHistory = (hubDatabase: HubDatabaseShape) => + hubDatabase.sql` + SELECT DISTINCT + provider_name AS "providerName", + provider_instance_id AS "providerInstanceId" + FROM projection_thread_sessions + WHERE user_id = ${hubDatabase.tenantId} + AND provider_name IN ('cursor', 'grok', 'opencode') + UNION + SELECT DISTINCT + provider_name AS "providerName", + provider_instance_id AS "providerInstanceId" + FROM provider_session_runtime + WHERE user_id = ${hubDatabase.tenantId} + AND provider_name IN ('cursor', 'grok', 'opencode') + `; + const make = Effect.gen(function* () { const { settingsPath } = yield* ServerConfig.ServerConfig; - const fs = yield* FileSystem.FileSystem; - const pathService = yield* Path.Path; + const settingsDocument = yield* StateDocument.make(settingsPath); + const hubDatabase = yield* HubDatabase; const secretStore = yield* ServerSecretStore.ServerSecretStore; const sql = yield* SqlClient.SqlClient; const writeSemaphore = yield* Semaphore.make(1); @@ -388,7 +411,7 @@ const make = Effect.gen(function* () { const emitChange = (settings: ServerSettings) => PubSub.publish(changesPubSub, settings).pipe(Effect.asVoid); - const readConfigExists = fs.exists(settingsPath).pipe( + const readConfigExists = settingsDocument.exists.pipe( Effect.mapError( (cause) => new ServerSettingsError({ @@ -399,7 +422,7 @@ const make = Effect.gen(function* () { ), ); - const readRawConfig = fs.readFileString(settingsPath).pipe( + const readRawConfig = settingsDocument.readString.pipe( Effect.mapError( (cause) => new ServerSettingsError({ @@ -435,22 +458,23 @@ const make = Effect.gen(function* () { } } - const providerHistory = yield* sql<{ - readonly providerName: string; - readonly providerInstanceId: string | null; - }>` - SELECT DISTINCT - provider_name AS "providerName", - provider_instance_id AS "providerInstanceId" - FROM projection_thread_sessions - WHERE provider_name IN ('cursor', 'grok', 'opencode') - UNION - SELECT DISTINCT - provider_name AS "providerName", - provider_instance_id AS "providerInstanceId" - FROM provider_session_runtime - WHERE provider_name IN ('cursor', 'grok', 'opencode') - `.pipe( + const providerHistory = yield* ( + hubDatabase === undefined + ? sql` + SELECT DISTINCT + provider_name AS "providerName", + provider_instance_id AS "providerInstanceId" + FROM projection_thread_sessions + WHERE provider_name IN ('cursor', 'grok', 'opencode') + UNION + SELECT DISTINCT + provider_name AS "providerName", + provider_instance_id AS "providerInstanceId" + FROM provider_session_runtime + WHERE provider_name IN ('cursor', 'grok', 'opencode') + ` + : readHubProviderHistory(hubDatabase) + ).pipe( Effect.mapError( (cause) => new ServerSettingsError({ @@ -642,13 +666,7 @@ const make = Effect.gen(function* () { stripDefaultServerSettings(settings, PERSISTED_SERVER_SETTINGS_DEFAULTS) ?? {}, ); - return yield* writeFileStringAtomically({ - filePath: settingsPath, - contents: `${sparseSettingsJson}\n`, - }).pipe( - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(Path.Path, pathService), - ); + return yield* settingsDocument.writeStringAtomically(`${sparseSettingsJson}\n`); }, Effect.mapError( (cause) => @@ -669,11 +687,7 @@ const make = Effect.gen(function* () { ); const startWatcher = Effect.gen(function* () { - const settingsDir = pathService.dirname(settingsPath); - const settingsFile = pathService.basename(settingsPath); - const settingsPathResolved = pathService.resolve(settingsPath); - - yield* fs.makeDirectory(settingsDir, { recursive: true }).pipe( + const debouncedSettingsEvents = yield* settingsDocument.watchExternalChanges.pipe( Effect.mapError( (cause) => new ServerSettingsError({ @@ -686,20 +700,6 @@ const make = Effect.gen(function* () { const revalidateAndEmitSafely = revalidateAndEmit.pipe(Effect.ignoreCause({ log: true })); - // Debounce watch events so the file is fully written before we read it. - // Editors emit multiple events per save (truncate, write, rename) and - // `fs.watch` can fire before the content has been flushed to disk. - const debouncedSettingsEvents = fs.watch(settingsDir).pipe( - Stream.filter((event) => { - return ( - event.path === settingsFile || - event.path === settingsPath || - pathService.resolve(settingsDir, event.path) === settingsPathResolved - ); - }), - Stream.debounce(Duration.millis(100)), - ); - yield* Stream.runForEach(debouncedSettingsEvents, () => revalidateAndEmitSafely).pipe( Effect.ignoreCause({ log: true }), Effect.forkIn(watcherScope), diff --git a/apps/server/src/telemetry/Identify.ts b/apps/server/src/telemetry/Identify.ts index b6c3d0066dff..5c88754a8b5b 100644 --- a/apps/server/src/telemetry/Identify.ts +++ b/apps/server/src/telemetry/Identify.ts @@ -8,6 +8,8 @@ import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; +import { HubDatabase, type HubDatabaseShape } from "../persistence/Postgres/HubDatabase.ts"; +import { makeHubDocuments } from "../persistence/Postgres/HubDocuments.ts"; import * as ServerConfig from "../config.ts"; const CodexAuthJsonSchema = Schema.Struct({ @@ -211,9 +213,44 @@ const getClaudeUserId = Effect.fn("TelemetryIdentity.getClaudeUserId")(function* return Option.some(claudeJson.userID); }); +// A hub's state directory is disposable; the id lives in the tenant's documents. +const upsertHubAnonymousId = Effect.fn("TelemetryIdentity.upsertHubAnonymousId")(function* ( + hubDatabase: HubDatabaseShape, + anonymousIdPath: string, +) { + const path = yield* Path.Path; + const generated = yield* Crypto.Crypto.pipe( + Effect.flatMap((crypto) => crypto.randomUUIDv4), + Effect.mapError( + (cause) => + new TelemetryAnonymousIdGenerationError({ + source: "anonymous", + filePath: anonymousIdPath, + cause, + }), + ), + ); + return yield* makeHubDocuments(hubDatabase) + .createIfAbsent(path.basename(anonymousIdPath), generated) + .pipe( + Effect.mapError( + (cause) => + new TelemetryAnonymousIdPersistenceError({ + source: "anonymous", + filePath: anonymousIdPath, + cause, + }), + ), + ); +}); + const upsertAnonymousId = Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; const { anonymousIdPath } = yield* ServerConfig.ServerConfig; + const hubDatabase = yield* HubDatabase; + if (hubDatabase !== undefined) { + return yield* upsertHubAnonymousId(hubDatabase, anonymousIdPath); + } const existing = yield* readIdentityFile(fileSystem, "anonymous", anonymousIdPath); if (Option.isSome(existing)) { From b4d212668870ed0433bcea74d5d4b40aad38b127 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 22:45:11 +0000 Subject: [PATCH 08/58] feat(server): record repository identity on hub projects A hub has no checkout to resolve a project's repository from git remotes, so project.create and project.meta.update accept an optional recorded repositoryIdentity (with an optional defaultBranch), carried through the events into the projection. The hub stores it with the project, serves it from the same snapshot read, and resolves it by workspace root or thread checkout path. Standalone keeps resolving from the checkout and ignores the field. Add a hub boot test: the real server composition in hub mode against Postgres creates a project and a thread, serves snapshots over HTTP with a bearer session, and after a restart on an empty base directory still serves them, accepts the session, keeps its environment id, and continues the event sequence, with no state files left in either base directory. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Layers/ProjectionPipeline.ts | 6 + apps/server/src/orchestration/decider.ts | 6 + apps/server/src/orchestration/projector.ts | 6 + .../Postgres/HubRepositoryIdentityResolver.ts | 62 ++++ .../Postgres/ProjectionProjects.ts | 8 +- .../Postgres/ProjectionSnapshotQuery.ts | 55 ++-- .../Services/ProjectionProjects.ts | 4 + apps/server/src/server.hub.test.ts | 269 ++++++++++++++++++ apps/server/src/server.ts | 3 +- packages/contracts/src/environment.ts | 3 + packages/contracts/src/orchestration.ts | 5 + 11 files changed, 390 insertions(+), 37 deletions(-) create mode 100644 apps/server/src/persistence/Postgres/HubRepositoryIdentityResolver.ts create mode 100644 apps/server/src/server.hub.test.ts diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts index 09ef7733dbd9..13f7c9dd04d9 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts @@ -520,6 +520,9 @@ export const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationP projectId: event.payload.projectId, title: event.payload.title, workspaceRoot: event.payload.workspaceRoot, + ...(event.payload.repositoryIdentity !== undefined + ? { repositoryIdentity: event.payload.repositoryIdentity } + : {}), defaultModelSelection: event.payload.defaultModelSelection, defaultThreadEnvMode: null, faviconPath: event.payload.faviconPath ?? null, @@ -553,6 +556,9 @@ export const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationP ? { faviconPath: event.payload.faviconPath } : {}), ...(event.payload.scripts !== undefined ? { scripts: event.payload.scripts } : {}), + ...(event.payload.repositoryIdentity !== undefined + ? { repositoryIdentity: event.payload.repositoryIdentity } + : {}), updatedAt: event.payload.updatedAt, }); return; diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index ae9100a3375d..c973585d49b5 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -221,6 +221,9 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" projectId: command.projectId, title: command.title, workspaceRoot: command.workspaceRoot, + ...(command.repositoryIdentity !== undefined + ? { repositoryIdentity: command.repositoryIdentity } + : {}), defaultModelSelection: command.defaultModelSelection ?? null, faviconPath: null, scripts: [], @@ -265,6 +268,9 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" : {}), ...(command.faviconPath !== undefined ? { faviconPath: command.faviconPath } : {}), ...(command.scripts !== undefined ? { scripts: command.scripts } : {}), + ...(command.repositoryIdentity !== undefined + ? { repositoryIdentity: command.repositoryIdentity } + : {}), updatedAt: occurredAt, }, }; diff --git a/apps/server/src/orchestration/projector.ts b/apps/server/src/orchestration/projector.ts index 28995dc2f19d..3a6b240b3bc5 100644 --- a/apps/server/src/orchestration/projector.ts +++ b/apps/server/src/orchestration/projector.ts @@ -213,6 +213,9 @@ export function projectEvent( id: payload.projectId, title: payload.title, workspaceRoot: payload.workspaceRoot, + ...(payload.repositoryIdentity !== undefined + ? { repositoryIdentity: payload.repositoryIdentity } + : {}), defaultModelSelection: payload.defaultModelSelection, defaultThreadEnvMode: null, faviconPath: payload.faviconPath ?? null, @@ -255,6 +258,9 @@ export function projectEvent( ? { faviconPath: payload.faviconPath } : {}), ...(payload.scripts !== undefined ? { scripts: payload.scripts } : {}), + ...(payload.repositoryIdentity !== undefined + ? { repositoryIdentity: payload.repositoryIdentity } + : {}), updatedAt: payload.updatedAt, } : project, diff --git a/apps/server/src/persistence/Postgres/HubRepositoryIdentityResolver.ts b/apps/server/src/persistence/Postgres/HubRepositoryIdentityResolver.ts new file mode 100644 index 000000000000..27f0752c7ae0 --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubRepositoryIdentityResolver.ts @@ -0,0 +1,62 @@ +/** + * Repository identity for a hub, which has no checkout to run `git remote` in. + * + * Projects record their identity at creation (`project.create` / + * `project.meta.update` `repositoryIdentity`), and the hub answers from that + * record: for a project's workspace root, or for a thread checkout path + * through the thread's project. + * + * @module HubRepositoryIdentityResolver + */ +import { RepositoryIdentity } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import { RepositoryIdentityResolver } from "../../project/RepositoryIdentityResolver.ts"; +import { HubDatabase } from "./HubDatabase.ts"; + +const decodeIdentity = Schema.decodeUnknownEffect(Schema.fromJsonString(RepositoryIdentity)); + +export const layer = Layer.effect( + RepositoryIdentityResolver, + Effect.gen(function* () { + const database = yield* HubDatabase; + if (database === undefined) { + return yield* Effect.die( + new Error("The hub repository identity resolver needs HubDatabase."), + ); + } + const { sql, tenantId } = database; + + return RepositoryIdentityResolver.of({ + resolve: (cwd) => + sql<{ readonly identity: string | null }>` + SELECT repository_identity_json AS identity + FROM projection_projects + WHERE user_id = ${tenantId} + AND workspace_root = ${cwd} + AND deleted_at IS NULL + UNION ALL + SELECT projects.repository_identity_json AS identity + FROM projection_threads threads + JOIN projection_projects projects + ON projects.user_id = threads.user_id + AND projects.project_id = threads.project_id + WHERE threads.user_id = ${tenantId} + AND threads.worktree_path = ${cwd} + AND threads.deleted_at IS NULL + LIMIT 1 + `.pipe( + Effect.flatMap(([row]) => + row?.identity ? decodeIdentity(row.identity) : Effect.succeed(null), + ), + Effect.catchCause((cause) => + Effect.logWarning("Failed to read a recorded repository identity.", { cause }).pipe( + Effect.as(null), + ), + ), + ), + }); + }), +); diff --git a/apps/server/src/persistence/Postgres/ProjectionProjects.ts b/apps/server/src/persistence/Postgres/ProjectionProjects.ts index 75c7073d7165..77b42ea4845f 100644 --- a/apps/server/src/persistence/Postgres/ProjectionProjects.ts +++ b/apps/server/src/persistence/Postgres/ProjectionProjects.ts @@ -5,7 +5,7 @@ import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; import * as Struct from "effect/Struct"; -import { ModelSelection, ProjectScript } from "@t3tools/contracts"; +import { ModelSelection, ProjectScript, RepositoryIdentity } from "@t3tools/contracts"; import { toPersistenceSqlError } from "../Errors.ts"; import { HubTenant } from "./HubTenant.ts"; import { @@ -19,6 +19,7 @@ import { const ProjectionProjectDbRow = ProjectionProject.mapFields( Struct.assign({ defaultModelSelection: Schema.NullOr(Schema.fromJsonString(ModelSelection)), + repositoryIdentity: Schema.optional(Schema.NullOr(Schema.fromJsonString(RepositoryIdentity))), scripts: Schema.fromJsonString(Schema.Array(ProjectScript)), }), ); @@ -40,6 +41,7 @@ const makeProjectionProjectRepository = Effect.gen(function* () { default_model_selection_json, default_thread_env_mode, favicon_path, + repository_identity_json, scripts_json, created_at, updated_at, @@ -53,6 +55,7 @@ const makeProjectionProjectRepository = Effect.gen(function* () { ${row.defaultModelSelection !== null ? JSON.stringify(row.defaultModelSelection) : null}, ${row.defaultThreadEnvMode}, ${row.faviconPath ?? null}, + ${row.repositoryIdentity ? JSON.stringify(row.repositoryIdentity) : null}, ${JSON.stringify(row.scripts)}, ${row.createdAt}, ${row.updatedAt}, @@ -65,6 +68,7 @@ const makeProjectionProjectRepository = Effect.gen(function* () { default_model_selection_json = excluded.default_model_selection_json, default_thread_env_mode = excluded.default_thread_env_mode, favicon_path = excluded.favicon_path, + repository_identity_json = excluded.repository_identity_json, scripts_json = excluded.scripts_json, created_at = excluded.created_at, updated_at = excluded.updated_at, @@ -84,6 +88,7 @@ const makeProjectionProjectRepository = Effect.gen(function* () { default_model_selection_json AS "defaultModelSelection", default_thread_env_mode AS "defaultThreadEnvMode", favicon_path AS "faviconPath", + repository_identity_json AS "repositoryIdentity", scripts_json AS "scripts", created_at AS "createdAt", updated_at AS "updatedAt", @@ -106,6 +111,7 @@ const makeProjectionProjectRepository = Effect.gen(function* () { default_model_selection_json AS "defaultModelSelection", default_thread_env_mode AS "defaultThreadEnvMode", favicon_path AS "faviconPath", + repository_identity_json AS "repositoryIdentity", scripts_json AS "scripts", created_at AS "createdAt", updated_at AS "updatedAt", diff --git a/apps/server/src/persistence/Postgres/ProjectionSnapshotQuery.ts b/apps/server/src/persistence/Postgres/ProjectionSnapshotQuery.ts index 25230e6464c0..1fc964abd8b4 100644 --- a/apps/server/src/persistence/Postgres/ProjectionSnapshotQuery.ts +++ b/apps/server/src/persistence/Postgres/ProjectionSnapshotQuery.ts @@ -12,6 +12,7 @@ import { OrchestrationThread, OrchestrationThreadDetailSnapshot, ProjectScript, + RepositoryIdentity, TurnId, type OrchestrationCheckpointSummary, type OrchestrationLatestTurn, @@ -58,7 +59,6 @@ import { encodeThreadDetailPageCursor, } from "../../orchestration/threadDetailCursor.ts"; import { projectActivityPayload } from "../../orchestration/ActivityPayloadProjection.ts"; -import * as RepositoryIdentityResolver from "../../project/RepositoryIdentityResolver.ts"; import { HubTenant } from "./HubTenant.ts"; import { ORCHESTRATION_PROJECTOR_NAMES } from "../../orchestration/Layers/ProjectionPipeline.ts"; import { @@ -83,6 +83,7 @@ const THREAD_DETAIL_ACTIVITY_PAYLOAD_BATCH_SIZE = 25; const ProjectionProjectDbRowSchema = ProjectionProject.mapFields( Struct.assign({ defaultModelSelection: Schema.NullOr(Schema.fromJsonString(ModelSelection)), + repositoryIdentity: Schema.optional(Schema.NullOr(Schema.fromJsonString(RepositoryIdentity))), scripts: Schema.fromJsonString(Schema.Array(ProjectScript)), }), ); @@ -387,40 +388,23 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { const threadPlanProgress = yield* ThreadPlanProgressService; const sql = yield* SqlClient.SqlClient; const { userId } = yield* HubTenant; - const repositoryIdentityResolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; - const repositoryIdentityResolutionConcurrency = 4; - const resolveRepositoryIdentitiesForProjects = Effect.fn( - "ProjectionSnapshotQuery.resolveRepositoryIdentitiesForProjects", - )(function* ( + // A hub has no checkout to run `git remote` in: projects carry the identity + // recorded at creation (repository_identity_json), read in the same snapshot. + const resolveRepositoryIdentitiesForProjects = ( projectRows: ReadonlyArray>, options?: { readonly includeDeleted?: boolean; }, - ) { - const filteredProjectRows = - options?.includeDeleted === true - ? projectRows - : projectRows.filter((row) => row.deletedAt === null); - const uniqueWorkspaceRoots = [...new Set(filteredProjectRows.map((row) => row.workspaceRoot))]; - const repositoryIdentityByWorkspaceRoot = new Map( - yield* Effect.forEach( - uniqueWorkspaceRoots, - (workspaceRoot) => - repositoryIdentityResolver - .resolve(workspaceRoot) - .pipe(Effect.map((identity) => [workspaceRoot, identity] as const)), - { concurrency: repositoryIdentityResolutionConcurrency }, + ) => + Effect.succeed( + new Map( + (options?.includeDeleted === true + ? projectRows + : projectRows.filter((row) => row.deletedAt === null) + ).map((row) => [row.projectId, row.repositoryIdentity ?? null] as const), ), ); - return new Map( - filteredProjectRows.map((row) => [ - row.projectId, - repositoryIdentityByWorkspaceRoot.get(row.workspaceRoot) ?? null, - ]), - ); - }); - // A SQLite read transaction reads one snapshot. Under Postgres READ COMMITTED // each statement would read its own, so a snapshot could pair rows with a newer // snapshotSequence and a resuming client would skip events. REPEATABLE READ @@ -442,6 +426,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { default_model_selection_json AS "defaultModelSelection", default_thread_env_mode AS "defaultThreadEnvMode", favicon_path AS "faviconPath", + repository_identity_json AS "repositoryIdentity", scripts_json AS "scripts", created_at AS "createdAt", updated_at AS "updatedAt", @@ -927,6 +912,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { default_model_selection_json AS "defaultModelSelection", default_thread_env_mode AS "defaultThreadEnvMode", favicon_path AS "faviconPath", + repository_identity_json AS "repositoryIdentity", scripts_json AS "scripts", created_at AS "createdAt", updated_at AS "updatedAt", @@ -952,6 +938,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { default_model_selection_json AS "defaultModelSelection", default_thread_env_mode AS "defaultThreadEnvMode", favicon_path AS "faviconPath", + repository_identity_json AS "repositoryIdentity", scripts_json AS "scripts", created_at AS "createdAt", updated_at AS "updatedAt", @@ -2555,7 +2542,7 @@ pending_approval_requests AS ( Effect.flatMap((option) => Option.isNone(option) ? Effect.succeed(Option.none()) - : repositoryIdentityResolver.resolve(option.value.workspaceRoot).pipe( + : Effect.succeed(option.value.repositoryIdentity ?? null).pipe( Effect.map((repositoryIdentity) => Option.some({ id: option.value.projectId, @@ -2586,13 +2573,11 @@ pending_approval_requests AS ( Effect.flatMap((option) => Option.isNone(option) ? Effect.succeed(Option.none()) - : repositoryIdentityResolver - .resolve(option.value.workspaceRoot) - .pipe( - Effect.map((repositoryIdentity) => - Option.some(mapProjectShellRow(option.value, repositoryIdentity)), - ), + : Effect.succeed(option.value.repositoryIdentity ?? null).pipe( + Effect.map((repositoryIdentity) => + Option.some(mapProjectShellRow(option.value, repositoryIdentity)), ), + ), ), ); diff --git a/apps/server/src/persistence/Services/ProjectionProjects.ts b/apps/server/src/persistence/Services/ProjectionProjects.ts index 339439fdfcb8..bceb26ecc7d7 100644 --- a/apps/server/src/persistence/Services/ProjectionProjects.ts +++ b/apps/server/src/persistence/Services/ProjectionProjects.ts @@ -11,6 +11,7 @@ import { ModelSelection, ProjectId, ProjectScript, + RepositoryIdentity, ThreadEnvMode, } from "@t3tools/contracts"; import * as Option from "effect/Option"; @@ -27,6 +28,9 @@ export const ProjectionProject = Schema.Struct({ defaultModelSelection: Schema.NullOr(ModelSelection), defaultThreadEnvMode: Schema.NullOr(ThreadEnvMode), faviconPath: Schema.optional(Schema.NullOr(Schema.String)), + // Hub projects only: the recorded identity (standalone resolves it from the + // checkout and does not store it). + repositoryIdentity: Schema.optional(Schema.NullOr(RepositoryIdentity)), scripts: Schema.Array(ProjectScript), createdAt: IsoDateTime, updatedAt: IsoDateTime, diff --git a/apps/server/src/server.hub.test.ts b/apps/server/src/server.hub.test.ts new file mode 100644 index 000000000000..9e227c7de2c7 --- /dev/null +++ b/apps/server/src/server.hub.test.ts @@ -0,0 +1,269 @@ +/** + * Boots the real server composition in hub mode against Postgres, creates a + * project and a thread through the engine and the HTTP dispatch route, reads + * the shell and thread snapshots, restarts on a fresh base directory, and + * checks that everything (including the bearer session) survived while the + * disposable base directory kept no durable state. + * + * Runs only with T3_HUB_TEST_DATABASE_URL (see persistence/Postgres/hubTestDatabase.ts). + * No runner is configured, so the hub's runner client is inert; nothing here + * needs a thread machine. + */ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { + CommandId, + DEFAULT_PROVIDER_INTERACTION_MODE, + EnvironmentHttpApi, + ProjectId, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import * as NetService from "@t3tools/shared/Net"; +import { assert, describe, it } from "@effect/vitest"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; +import * as Reactivity from "effect/unstable/reactivity/Reactivity"; +import { FetchHttpClient, HttpServer } from "effect/unstable/http"; +import * as HttpApiClient from "effect/unstable/httpapi/HttpApiClient"; + +import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; +import * as ServerConfig from "./config.ts"; +import { ServerEnvironment } from "./environment/ServerEnvironment.ts"; +import { OrchestrationEngineService } from "./orchestration/Services/OrchestrationEngine.ts"; +import { AuthAdministrativeScopes } from "@t3tools/contracts"; +import { HubDatabase } from "./persistence/Postgres/HubDatabase.ts"; +import { makeHubDocuments } from "./persistence/Postgres/HubDocuments.ts"; +import { + hubTestDatabaseLayer, + hubTestDatabaseUrl, + hubTestServerConfigLayer, + makeHubTestSchema, + type HubTestSchema, +} from "./persistence/Postgres/hubTestDatabase.ts"; +import { makeServerLayer } from "./server.ts"; + +const TENANT = "user_hub_boot"; +const projectId = ProjectId.make("project-hub-boot"); +const threadId = ThreadId.make("thread-hub-boot"); +const modelSelection = { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5-codex" }; +const repositoryIdentity = { + canonicalKey: "github.com/aldo/app", + locator: { + source: "git-remote" as const, + remoteName: "origin", + remoteUrl: "https://github.com/aldo/app.git", + }, + provider: "github", + owner: "aldo", + name: "app", + defaultBranch: "main", +}; + +const hubConfig = (schema: HubTestSchema, baseDir: string) => + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + return { + ...config, + host: "127.0.0.1", + port: 0, + logLevel: (process.env.T3_HUB_TEST_LOG_LEVEL ?? + "Warn") as ServerConfig.ServerConfig["Service"]["logLevel"], + noBrowser: true, + startupPresentation: "headless", + } satisfies ServerConfig.ServerConfig["Service"]; + }).pipe(Effect.provide(hubTestServerConfigLayer(schema, TENANT, { baseDir }))); + +/** + * Starts one hub process the way the CLI does (the server layer held open by + * a fiber); `stop` interrupts it like a shutdown signal. + */ +const startHub = (schema: HubTestSchema, baseDir: string) => + Effect.gen(function* () { + const config = yield* hubConfig(schema, baseDir); + const ready = yield* Deferred.make>(); + const fiber = yield* Effect.forkChild( + Effect.scoped( + Effect.gen(function* () { + const context = yield* Layer.build(makeServerLayer); + yield* Deferred.succeed(ready, context as Context.Context); + return yield* Effect.never; + }), + ).pipe(Effect.provideService(ServerConfig.ServerConfig, config)), + ); + const context = yield* Effect.raceFirst(Deferred.await(ready), Fiber.join(fiber)); + const server = Context.getUnsafe(context, HttpServer.HttpServer); + const address = server.address; + if (typeof address === "string" || !("port" in address)) { + return yield* Effect.die(new Error("hub server has no TCP address")); + } + return { + config, + origin: `http://127.0.0.1:${address.port}`, + engine: Context.getUnsafe(context, OrchestrationEngineService), + auth: Context.getUnsafe(context, EnvironmentAuth.EnvironmentAuth), + environment: Context.getUnsafe(context, ServerEnvironment), + stop: Fiber.interrupt(fiber), + }; + }); + +const durableStateFiles = (config: ServerConfig.ServerConfig["Service"]) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const candidates = [ + config.dbPath, + config.settingsPath, + config.keybindingsConfigPath, + config.environmentIdPath, + config.anonymousIdPath, + config.secretsDir, + ]; + const present: Array = []; + for (const candidate of candidates) { + if (yield* fileSystem.exists(candidate)) present.push(candidate); + } + return present; + }); + +describe.skipIf(hubTestDatabaseUrl === undefined)("hub server", () => { + it.effect( + "serves projects and threads from Postgres across restarts with no durable local state", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const schema = yield* makeHubTestSchema(hubTestDatabaseUrl!); + + // Provider update checks would reach the network; the hub reads its + // settings from Postgres, so seed them there. + yield* Effect.gen(function* () { + const database = yield* HubDatabase; + yield* makeHubDocuments(database!).write( + "settings.json", + '{ "enableProviderUpdateChecks": false }\n', + ); + }).pipe(Effect.provide(hubTestDatabaseLayer(schema, TENANT))); + + const firstBaseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-hub-boot-a-", + }); + const first = yield* startHub(schema, firstBaseDir); + const client = yield* HttpApiClient.make(EnvironmentHttpApi, { baseUrl: first.origin }); + + const issued = yield* first.auth.issueSession({ + scopes: AuthAdministrativeScopes, + label: "hub boot test", + }); + const headers = { authorization: `Bearer ${issued.token}` }; + + // The project CLI and managed platform dispatch through the engine; a + // hub has no checkout to normalize a workspace root against. + yield* first.engine.dispatch({ + type: "project.create", + commandId: CommandId.make("cmd-hub-boot-project"), + projectId, + title: "Hub project", + workspaceRoot: `/workspace/p/${projectId}`, + defaultModelSelection: modelSelection, + repositoryIdentity, + createdAt: "2026-09-25T00:00:00.000Z", + }); + yield* client.orchestration.dispatch({ + headers, + payload: { + type: "thread.create", + commandId: CommandId.make("cmd-hub-boot-thread"), + threadId, + projectId, + title: "Hub thread", + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: `/workspace/t/${threadId}`, + createdAt: "2026-09-25T00:00:01.000Z", + }, + }); + + const shell = yield* client.orchestration.shellSnapshot({ headers }); + assert.deepStrictEqual( + shell.projects.map((project) => [project.id, project.repositoryIdentity?.defaultBranch]), + [[projectId, "main"]], + ); + assert.deepStrictEqual( + shell.threads.map((thread) => thread.id), + [threadId], + ); + const detail = yield* client.orchestration.threadSnapshot({ + headers, + params: { threadId }, + payload: {}, + }); + assert.strictEqual(detail.thread.title, "Hub thread"); + const environmentId = yield* first.environment.getEnvironmentId; + + assert.deepStrictEqual(yield* durableStateFiles(first.config), []); + yield* first.stop; + + // A new process on an empty base directory: nothing came from disk. + const secondBaseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-hub-boot-b-", + }); + assert.deepStrictEqual(yield* fileSystem.readDirectory(secondBaseDir), []); + const second = yield* startHub(schema, secondBaseDir); + const secondClient = yield* HttpApiClient.make(EnvironmentHttpApi, { + baseUrl: second.origin, + }); + + // The bearer session issued by the first process still authenticates. + const restartedShell = yield* secondClient.orchestration.shellSnapshot({ headers }); + assert.deepStrictEqual( + restartedShell.projects.map((project) => [ + project.id, + project.title, + project.repositoryIdentity?.canonicalKey, + ]), + [[projectId, "Hub project", "github.com/aldo/app"]], + ); + assert.deepStrictEqual( + restartedShell.threads.map((thread) => [thread.id, thread.title]), + [[threadId, "Hub thread"]], + ); + assert.isAtLeast(restartedShell.snapshotSequence, shell.snapshotSequence); + assert.strictEqual(yield* second.environment.getEnvironmentId, environmentId); + + // Commands keep sequencing after the restart. + yield* secondClient.orchestration.dispatch({ + headers, + payload: { + type: "thread.meta.update", + commandId: CommandId.make("cmd-hub-boot-rename"), + threadId, + title: "Hub thread, renamed", + }, + }); + const renamed = yield* secondClient.orchestration.threadSnapshot({ + headers, + params: { threadId }, + payload: {}, + }); + assert.strictEqual(renamed.thread.title, "Hub thread, renamed"); + assert.isAbove(renamed.snapshotSequence, restartedShell.snapshotSequence); + + assert.deepStrictEqual(yield* durableStateFiles(second.config), []); + yield* second.stop; + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NetService.layer, + FetchHttpClient.layer, + Reactivity.layer, + ), + ), + ), + ); +}); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index ea996a58625d..4a5faf1ff6c2 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -35,6 +35,7 @@ import * as PullRequestProviderRegistry from "./pullRequest/PullRequestProviderR import * as PullRequestService from "./pullRequest/PullRequestService.ts"; import { layerConfig as SqlitePersistenceLayerLive } from "./persistence/Layers/Sqlite.ts"; import * as HubDatabase from "./persistence/Postgres/HubDatabase.ts"; +import * as HubRepositoryIdentityResolver from "./persistence/Postgres/HubRepositoryIdentityResolver.ts"; import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; import { ProviderSessionDirectoryLive } from "./provider/Layers/ProviderSessionDirectory.ts"; @@ -494,7 +495,7 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( Layer.provideMerge(WorkspaceLayerLive), Layer.provideMerge(ProjectFaviconResolverLayerLive), Layer.provideMerge( - byServerMode(RepositoryIdentityResolver.layer, HubLayers.hubRepositoryIdentityResolverLayer), + byServerMode(RepositoryIdentityResolver.layer, HubRepositoryIdentityResolver.layer), ), Layer.provideMerge(ServerEnvironmentLayerLive), Layer.provideMerge(AuthLayerLive), diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index 724665397b10..34a864748f26 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -136,6 +136,9 @@ export const RepositoryIdentity = Schema.Struct({ provider: Schema.optionalKey(TrimmedNonEmptyString), owner: Schema.optionalKey(TrimmedNonEmptyString), name: Schema.optionalKey(TrimmedNonEmptyString), + // Set when the identity is recorded rather than read from a checkout (hub + // projects): the branch a new thread machine checks out. + defaultBranch: Schema.optionalKey(TrimmedNonEmptyString), }); export type RepositoryIdentity = typeof RepositoryIdentity.Type; diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts index 60027a91dbe2..1e9c92e2ec3b 100644 --- a/packages/contracts/src/orchestration.ts +++ b/packages/contracts/src/orchestration.ts @@ -713,6 +713,9 @@ export const ProjectCreateCommand = Schema.Struct({ workspaceRoot: TrimmedNonEmptyString, createWorkspaceRootIfMissing: Schema.optional(Schema.Boolean), defaultModelSelection: Schema.optional(Schema.NullOr(ModelSelection)), + // Recorded identity for projects without a local checkout (hub mode), which + // cannot resolve it from `git remote`. Absent = resolve from the checkout. + repositoryIdentity: Schema.optional(Schema.NullOr(RepositoryIdentity)), createdAt: IsoDateTime, }); @@ -727,6 +730,8 @@ const ProjectMetaUpdateCommand = Schema.Struct({ defaultThreadEnvMode: Schema.optional(Schema.NullOr(ThreadEnvMode)), faviconPath: Schema.optional(Schema.NullOr(ProjectFaviconPath)), scripts: Schema.optional(Schema.Array(ProjectScript)), + // Absent = leave unchanged; null = clear the recorded identity. + repositoryIdentity: Schema.optional(Schema.NullOr(RepositoryIdentity)), }); const ProjectDeleteCommand = Schema.Struct({ From 68083f9088e1beaad878f9d83c6012852fc16339 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 22:49:51 +0000 Subject: [PATCH 09/58] test(server): prove hub repositories match SQLite and isolate tenants Run the same orchestration commands through the standalone SQLite layers and the hub's Postgres layers and require identical results from every snapshot query method, including the ones the prototype never exercised, plus the event stream. A second tenant reusing every id sees only its own rows and its own sequence, and a rebuilt engine reads the same model back. Auth sessions, pairing links, provider session runtime, and checkpoint repositories get the same parity and isolation checks. The tenant client also replaces NUL characters in string parameters, which Postgres text cannot store but SQLite accepted. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/persistence/Postgres/HubClient.ts | 30 +- .../Postgres/HubRepositories.test.ts | 527 ++++++++++++++++++ 2 files changed, 555 insertions(+), 2 deletions(-) create mode 100644 apps/server/src/persistence/Postgres/HubRepositories.test.ts diff --git a/apps/server/src/persistence/Postgres/HubClient.ts b/apps/server/src/persistence/Postgres/HubClient.ts index fe33794a6dee..af02d34516aa 100644 --- a/apps/server/src/persistence/Postgres/HubClient.ts +++ b/apps/server/src/persistence/Postgres/HubClient.ts @@ -112,6 +112,30 @@ const withTenantTransaction = ( }), ); +// Postgres `text` cannot hold U+0000, which SQLite stores happily (provider +// output occasionally contains it). Replace it in string parameters instead of +// failing the write; JSON text keeps its escaped `\u0000` form and is handled +// where it is parsed. +const withoutNul = (params: ReadonlyArray): ReadonlyArray => + params.some((param) => typeof param === "string" && param.includes("\u0000")) + ? params.map((param) => + typeof param === "string" ? param.replaceAll("\u0000", "\uFFFD") : param, + ) + : params; + +const sanitizingParameters = (connection: SqlConnection.Connection): SqlConnection.Connection => ({ + execute: (sql, params, transformRows) => + connection.execute(sql, withoutNul(params), transformRows), + executeRaw: (sql, params) => connection.executeRaw(sql, withoutNul(params)), + executeValues: (sql, params) => connection.executeValues(sql, withoutNul(params)), + executeValuesUnprepared: (sql, params) => + connection.executeValuesUnprepared(sql, withoutNul(params)), + executeUnprepared: (sql, params, transformRows) => + connection.executeUnprepared(sql, withoutNul(params), transformRows), + executeStream: (sql, params, transformRows) => + connection.executeStream(sql, withoutNul(params), transformRows), +}); + /** * Wraps a pooled client so every statement and transaction carries the tenant. * @@ -152,8 +176,10 @@ export const makeTenantSqlClient = (base: PgClient.PgClient, tenantId: string) = return yield* SqlClient.make({ // `reserve` pins one pooled connection for the statement, so the tenant // transaction and the statement share it. - acquirer: Effect.map(base.reserve, autocommit), - transactionAcquirer: base.reserve, + acquirer: Effect.map(base.reserve, (connection) => + autocommit(sanitizingParameters(connection)), + ), + transactionAcquirer: Effect.map(base.reserve, sanitizingParameters), compiler: PgClient.makeCompiler(), spanAttributes: [ ["db.system.name", "postgresql"], diff --git a/apps/server/src/persistence/Postgres/HubRepositories.test.ts b/apps/server/src/persistence/Postgres/HubRepositories.test.ts new file mode 100644 index 000000000000..77bb4781b6e3 --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubRepositories.test.ts @@ -0,0 +1,527 @@ +/** + * The hub's Postgres repositories against the standalone SQLite ones: the same + * commands or calls must produce the same reads, and one tenant never sees + * another's rows even when they reuse the same ids. Runs only with + * T3_HUB_TEST_DATABASE_URL (see `hubTestDatabase.ts`). + */ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { + AuthStandardClientScopes, + AuthSessionId, + CheckpointRef, + CommandId, + DEFAULT_PROVIDER_INTERACTION_MODE, + EventId, + MessageId, + ProjectId, + ProviderInstanceId, + ThreadId, + TurnId, + type OrchestrationCommand, +} from "@t3tools/contracts"; +import { assert, describe, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; +import * as Reactivity from "effect/unstable/reactivity/Reactivity"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import * as ServerConfig from "../../config.ts"; +import { OrchestrationEngineService } from "../../orchestration/Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../../orchestration/Services/ProjectionSnapshotQuery.ts"; +import { OrchestrationLayerLive } from "../../orchestration/runtimeLayer.ts"; +import { RepositoryIdentityResolver } from "../../project/RepositoryIdentityResolver.ts"; +import * as AuthPairingLinks from "../AuthPairingLinks.ts"; +import * as AuthSessions from "../AuthSessions.ts"; +import { ProjectionCheckpointRepositoryLive } from "../Layers/ProjectionCheckpoints.ts"; +import { layerConfig as ServerPersistenceLive, SqlitePersistenceMemory } from "../Layers/Sqlite.ts"; +import * as ProviderSessionRuntime from "../ProviderSessionRuntime.ts"; +import { ProjectionCheckpointRepository } from "../Services/ProjectionCheckpoints.ts"; +import { + hubTestDatabaseLayer, + hubTestDatabaseUrl, + makeHubTestSchema, + type HubTestSchema, +} from "./hubTestDatabase.ts"; + +const withSchema = (use: (schema: HubTestSchema) => Effect.Effect) => + Effect.scoped(Effect.flatMap(makeHubTestSchema(hubTestDatabaseUrl!), use)).pipe( + Effect.provide(Reactivity.layer), + ); + +const platform = Layer.mergeAll( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-repositories-" }), + Layer.succeed(RepositoryIdentityResolver, { resolve: () => Effect.succeed(null) }), +).pipe(Layer.provideMerge(NodeServices.layer)); + +/** Standalone persistence: a fresh migrated in-memory SQLite database. */ +const sqlite = () => Layer.fresh(SqlitePersistenceMemory); + +/** Hub persistence for `tenantId`: the same live layers, selected by HubDatabase. */ +const hub = (schema: HubTestSchema, tenantId: string) => + ServerPersistenceLive.pipe( + Layer.provideMerge(hubTestDatabaseLayer(schema, tenantId)), + Layer.provide(platform), + ); + +const at = (second: number) => `2026-09-25T00:00:${String(second).padStart(2, "0")}.000Z`; +const modelSelection = { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5-codex" }; +const projectId = ProjectId.make("project-shared"); +const threadId = ThreadId.make("thread-shared"); +const otherThreadId = ThreadId.make("thread-archived"); +const turnId = TurnId.make("turn-1"); + +// Both tenants reuse the same ids: isolation must come from the tenant key. +const commandsFor = (label: string): ReadonlyArray => [ + { + type: "project.create", + commandId: CommandId.make("cmd-project-create"), + projectId, + title: `Project of ${label}`, + workspaceRoot: `/workspace/p/${projectId}`, + defaultModelSelection: modelSelection, + createdAt: at(1), + }, + { + type: "thread.create", + commandId: CommandId.make("cmd-thread-create"), + threadId, + projectId, + title: `Thread of ${label}`, + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: `/workspace/t/${threadId}`, + createdAt: at(2), + }, + { + type: "thread.turn.start", + commandId: CommandId.make("cmd-turn-start"), + threadId, + message: { + messageId: MessageId.make("msg-user-1"), + role: "user", + text: `hello from ${label}`, + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: at(3), + }, + { + type: "thread.session.set", + commandId: CommandId.make("cmd-session-set"), + threadId, + session: { + threadId, + status: "running", + providerName: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "approval-required", + activeTurnId: turnId, + lastError: null, + updatedAt: at(4), + }, + createdAt: at(4), + }, + { + type: "thread.activity.append", + commandId: CommandId.make("cmd-activity-1"), + threadId, + activity: { + id: EventId.make("activity-1"), + tone: "tool", + kind: "tool.completed", + summary: `tool ran for ${label}`, + // JSON payloads may carry escaped NULs; Postgres keeps them as text. + payload: { owner: label, output: "binary\u0000tail" }, + turnId, + createdAt: at(5), + }, + createdAt: at(5), + }, + { + type: "thread.message.assistant.delta", + commandId: CommandId.make("cmd-assistant-delta-1"), + threadId, + messageId: MessageId.make("msg-assistant-1"), + delta: `reply to ${label}`, + turnId, + createdAt: at(6), + }, + { + type: "thread.message.assistant.complete", + commandId: CommandId.make("cmd-assistant-complete-1"), + threadId, + messageId: MessageId.make("msg-assistant-1"), + turnId, + createdAt: at(7), + }, + { + type: "thread.proposed-plan.upsert", + commandId: CommandId.make("cmd-plan-1"), + threadId, + proposedPlan: { + id: "plan-1", + turnId, + planMarkdown: `# Plan for ${label}`, + implementedAt: null, + implementationThreadId: null, + createdAt: at(8), + updatedAt: at(8), + }, + createdAt: at(8), + }, + { + type: "thread.turn.diff.complete", + commandId: CommandId.make("cmd-diff-1"), + threadId, + turnId, + completedAt: at(9), + checkpointRef: CheckpointRef.make("refs/t3/checkpoints/thread-shared/turn/1"), + status: "ready", + files: [{ path: "notes.txt", kind: "modified", additions: 1, deletions: 0 }], + assistantMessageId: MessageId.make("msg-assistant-1"), + checkpointTurnCount: 1, + createdAt: at(9), + }, + { + type: "thread.meta.update", + commandId: CommandId.make("cmd-thread-rename"), + threadId, + title: `Renamed thread of ${label}`, + }, + { + type: "thread.create", + commandId: CommandId.make("cmd-thread-archived-create"), + threadId: otherThreadId, + projectId, + title: `Archived thread of ${label}`, + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt: at(10), + }, + { + type: "thread.archive", + commandId: CommandId.make("cmd-thread-archive"), + threadId: otherThreadId, + }, +]; + +/** Dispatches the scenario and reads back through every snapshot query method. */ +const orchestrationScenario = (label: string) => + Effect.gen(function* () { + const engine = yield* OrchestrationEngineService; + const query = yield* ProjectionSnapshotQuery; + for (const command of commandsFor(label)) { + yield* engine.dispatch(command); + } + const events = yield* Stream.runCollect(engine.readEvents(0)); + return { + events: Array.from(events, (event) => ({ + sequence: event.sequence, + type: event.type, + aggregateId: event.aggregateId, + payload: event.payload, + })), + commandReadModel: yield* query.getCommandReadModel(), + snapshot: yield* query.getSnapshot(), + shell: yield* query.getShellSnapshot(), + archivedShell: yield* query.getArchivedShellSnapshot(), + counts: yield* query.getCounts(), + snapshotSequence: yield* query.getSnapshotSequence(), + projectByRoot: yield* query.getActiveProjectByWorkspaceRoot(`/workspace/p/${projectId}`), + projectShell: yield* query.getProjectShellById(projectId), + firstThread: yield* query.getFirstActiveThreadIdByProjectId(projectId), + checkpointContext: yield* query.getThreadCheckpointContext(threadId), + fullDiffContext: yield* query.getFullThreadDiffContext(threadId, 1), + threadShell: yield* query.getThreadShellById(threadId), + threadDetail: yield* query.getThreadDetailById(threadId), + detailSnapshot: yield* query.getThreadDetailSnapshot(threadId), + windowedDetail: yield* query.getThreadDetailSnapshot(threadId, { turnLimit: 1 }), + search: yield* query.searchThreads({ query: "RENAMED" }), + }; + }); + +const runOrchestration = (persistence: Layer.Layer, label: string) => + orchestrationScenario(label).pipe( + Effect.provide( + OrchestrationLayerLive.pipe(Layer.provideMerge(persistence), Layer.provideMerge(platform)), + ), + ); + +describe.skipIf(hubTestDatabaseUrl === undefined)("hub repositories", () => { + it.effect("match SQLite for every snapshot read and isolate tenants that reuse ids", () => + withSchema((schema) => + Effect.gen(function* () { + const standalone = yield* runOrchestration(sqlite(), "alice"); + const alice = yield* runOrchestration(hub(schema, "tenant-alice"), "alice"); + assert.deepStrictEqual(alice, standalone); + + const bob = yield* runOrchestration(hub(schema, "tenant-bob"), "bob"); + assert.deepStrictEqual( + bob.events.map((event) => event.sequence), + alice.events.map((event) => event.sequence), + ); + assert.strictEqual(bob.events[0]?.sequence, 1); + assert.deepStrictEqual( + bob.shell.threads.map((thread) => thread.title), + ["Renamed thread of bob"], + ); + // @effect-diagnostics-next-line preferSchemaOverJson:off + assert.notInclude(JSON.stringify(bob), "alice"); + + // A new engine for alice rebuilds the same read model from Postgres. + const reread = yield* Effect.gen(function* () { + const query = yield* ProjectionSnapshotQuery; + return { + commandReadModel: yield* query.getCommandReadModel(), + shell: yield* query.getShellSnapshot(), + }; + }).pipe( + Effect.provide( + OrchestrationLayerLive.pipe( + Layer.provideMerge(hub(schema, "tenant-alice")), + Layer.provideMerge(platform), + ), + ), + ); + assert.deepStrictEqual(reread.commandReadModel, alice.commandReadModel); + assert.deepStrictEqual(reread.shell, alice.shell); + }), + ), + ); + + it.effect("keep auth sessions and pairing links per tenant", () => + withSchema((schema) => + Effect.gen(function* () { + const t0 = DateTime.makeUnsafe("2026-09-25T00:00:00.000Z"); + const later = (minutes: number) => DateTime.add(t0, { minutes }); + const client = { + label: "browser", + ipAddress: "127.0.0.1", + userAgent: "test", + deviceType: "desktop" as const, + os: null, + browser: null, + }; + const sessionA = AuthSessionId.make("session-a"); + const sessionB = AuthSessionId.make("session-b"); + + const scenario = Effect.gen(function* () { + const sessions = yield* AuthSessions.AuthSessionRepository; + const links = yield* AuthPairingLinks.AuthPairingLinkRepository; + for (const [sessionId, minute] of [ + [sessionA, 1], + [sessionB, 2], + ] as const) { + yield* sessions.create({ + sessionId, + subject: "owner", + scopes: AuthStandardClientScopes, + method: "browser-session-cookie", + client, + issuedAt: later(minute), + expiresAt: later(60), + }); + } + yield* sessions.setLastConnectedAt({ sessionId: sessionA, lastConnectedAt: later(3) }); + yield* sessions.setClientConnection({ + sessionId: sessionA, + surface: "web", + appVersion: "1.2.3", + }); + const activeBefore = yield* sessions.listActive({ now: later(4) }); + const revoked = yield* sessions.revoke({ sessionId: sessionA, revokedAt: later(5) }); + const revokedAgain = yield* sessions.revoke({ sessionId: sessionA, revokedAt: later(5) }); + const others = yield* sessions.revokeAllExcept({ + currentSessionId: sessionB, + revokedAt: later(6), + }); + + for (const [id, credential, proofKeyThumbprint] of [ + ["link-1", "credential-1", null], + ["link-2", "credential-2", "thumbprint"], + ] as const) { + yield* links.create({ + id, + credential, + method: "one-time-token", + scopes: AuthStandardClientScopes, + subject: "owner", + label: null, + proofKeyThumbprint, + createdAt: later(1), + expiresAt: later(30), + }); + } + const linksBefore = yield* links.listActive({ now: later(2) }); + const consumed = yield* links.consumeAvailable({ + credential: "credential-1", + proofKeyThumbprint: null, + consumedAt: later(3), + now: later(3), + }); + const consumedTwice = yield* links.consumeAvailable({ + credential: "credential-1", + proofKeyThumbprint: null, + consumedAt: later(3), + now: later(3), + }); + const wrongProof = yield* links.consumeAvailable({ + credential: "credential-2", + proofKeyThumbprint: "other", + consumedAt: later(4), + now: later(4), + }); + const revokedLink = yield* links.revoke({ id: "link-2", revokedAt: later(5) }); + return { + session: yield* sessions.getById({ sessionId: sessionA }), + activeBefore, + activeAfter: yield* sessions.listActive({ now: later(7) }), + revoked, + revokedAgain, + others, + linksBefore, + consumed, + consumedTwice, + wrongProof, + revokedLink, + byCredential: yield* links.getByCredential({ credential: "credential-1" }), + linksAfter: yield* links.listActive({ now: later(6) }), + }; + }); + const repositories = (persistence: Layer.Layer) => + Layer.mergeAll(AuthSessions.layer, AuthPairingLinks.layer).pipe( + Layer.provideMerge(persistence), + ); + + const standalone = yield* scenario.pipe(Effect.provide(repositories(sqlite()))); + const tenantA = yield* scenario.pipe(Effect.provide(repositories(hub(schema, "tenant-a")))); + assert.deepStrictEqual(tenantA, standalone); + assert.isTrue(Option.isSome(tenantA.session)); + assert.strictEqual(tenantA.activeBefore.length, 2); + + // Another tenant sees none of it, even for the same ids and credentials. + yield* Effect.gen(function* () { + const sessions = yield* AuthSessions.AuthSessionRepository; + const links = yield* AuthPairingLinks.AuthPairingLinkRepository; + assert.isTrue(Option.isNone(yield* sessions.getById({ sessionId: sessionB }))); + assert.deepStrictEqual(yield* sessions.listActive({ now: later(4) }), []); + assert.deepStrictEqual( + yield* sessions.revokeAllExcept({ currentSessionId: sessionA, revokedAt: later(8) }), + [], + ); + assert.isTrue( + Option.isNone(yield* links.getByCredential({ credential: "credential-1" })), + ); + assert.isTrue( + Option.isNone( + yield* links.consumeAvailable({ + credential: "credential-2", + proofKeyThumbprint: "thumbprint", + consumedAt: later(2), + now: later(2), + }), + ), + ); + }).pipe(Effect.provide(repositories(hub(schema, "tenant-b")))); + + // A later process for tenant-a still finds its rows. + const persisted = yield* Effect.flatMap(AuthSessions.AuthSessionRepository, (sessions) => + sessions.getById({ sessionId: sessionB }), + ).pipe(Effect.provide(repositories(hub(schema, "tenant-a")))); + assert.isTrue(Option.isSome(persisted)); + }), + ), + ); + + it.effect("keep provider runtime and checkpoint rows per tenant", () => + withSchema((schema) => + Effect.gen(function* () { + const scenario = Effect.gen(function* () { + const runtime = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const checkpoints = yield* ProjectionCheckpointRepository; + for (const [thread, seen] of [ + [ThreadId.make("thread-b"), at(2)], + [ThreadId.make("thread-a"), at(1)], + ] as const) { + yield* runtime.upsert({ + threadId: thread, + providerName: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + adapterKey: "codex", + runtimeMode: "full-access", + status: "running", + lastSeenAt: seen, + resumeCursor: { threadId: `${thread}-provider`, cursor: 7 }, + runtimePayload: null, + }); + } + yield* runtime.upsert({ + threadId: ThreadId.make("thread-a"), + providerName: "codex", + providerInstanceId: null, + adapterKey: "codex", + runtimeMode: "approval-required", + status: "stopped", + lastSeenAt: at(3), + resumeCursor: null, + runtimePayload: { reason: "stopped" }, + }); + yield* runtime.deleteByThreadId({ threadId: ThreadId.make("thread-b") }); + + const checkpoint = { + threadId, + turnId, + checkpointTurnCount: 1, + checkpointRef: CheckpointRef.make("refs/t3/checkpoints/thread-shared/turn/1"), + status: "ready" as const, + files: [{ path: "a.ts", kind: "modified", additions: 2, deletions: 1 }], + assistantMessageId: null, + completedAt: at(4), + }; + yield* checkpoints.upsert(checkpoint); + return { + runtime: yield* runtime.list(), + runtimeA: yield* runtime.getByThreadId({ threadId: ThreadId.make("thread-a") }), + checkpoints: yield* checkpoints.listByThreadId({ threadId }), + checkpoint: yield* checkpoints.getByThreadAndTurnCount({ + threadId, + checkpointTurnCount: 1, + }), + }; + }); + const repositories = (persistence: Layer.Layer) => + Layer.mergeAll(ProviderSessionRuntime.layer, ProjectionCheckpointRepositoryLive).pipe( + Layer.provideMerge(persistence), + ); + + const standalone = yield* scenario.pipe(Effect.provide(repositories(sqlite()))); + const tenantA = yield* scenario.pipe(Effect.provide(repositories(hub(schema, "tenant-a")))); + assert.deepStrictEqual(tenantA, standalone); + assert.strictEqual(tenantA.runtime.length, 1); + + yield* Effect.gen(function* () { + const runtime = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const checkpoints = yield* ProjectionCheckpointRepository; + assert.deepStrictEqual(yield* runtime.list(), []); + assert.deepStrictEqual(yield* checkpoints.listByThreadId({ threadId }), []); + yield* checkpoints.deleteByThreadId({ threadId }); + }).pipe(Effect.provide(repositories(hub(schema, "tenant-b")))); + + // tenant-b's delete did not reach tenant-a. + const stillThere = yield* Effect.flatMap(ProjectionCheckpointRepository, (checkpoints) => + checkpoints.listByThreadId({ threadId }), + ).pipe(Effect.provide(repositories(hub(schema, "tenant-a")))); + assert.strictEqual(stillThere.length, 1); + }), + ), + ); +}); From b1eb9f9f8e980a89cf8f830075fd0eddded302fc Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 22:55:28 +0000 Subject: [PATCH 10/58] feat(contracts): define the versioned hub-runner protocol Move the runner protocol out of the server prototype into @t3tools/contracts/runner as RunnerRpcGroup (protocol version 1) with a version-range handshake that also binds a runner to one thread. It carries provider sessions, text generation, the durable event outbox, checkout preparation, checkpoints, workspace files, VCS and git actions, review, and terminals, reusing the client contract schemas. Also defines the machine directory schemas, thread checkout path helpers, and typed errors for sleeping or unavailable machines and unsupported hub operations. Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/contracts/package.json | 4 + packages/contracts/src/runner.test.ts | 91 +++ packages/contracts/src/runner.ts | 964 ++++++++++++++++++++++++++ 3 files changed, 1059 insertions(+) create mode 100644 packages/contracts/src/runner.test.ts create mode 100644 packages/contracts/src/runner.ts diff --git a/packages/contracts/package.json b/packages/contracts/package.json index dbd8ee3744e8..3681ecb2dd0d 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -18,6 +18,10 @@ "./relay": { "types": "./src/relay.ts", "import": "./src/relay.ts" + }, + "./runner": { + "types": "./src/runner.ts", + "import": "./src/runner.ts" } }, "scripts": { diff --git a/packages/contracts/src/runner.test.ts b/packages/contracts/src/runner.test.ts new file mode 100644 index 000000000000..35bb5d12d17d --- /dev/null +++ b/packages/contracts/src/runner.test.ts @@ -0,0 +1,91 @@ +import * as Schema from "effect/Schema"; +import { describe, expect, it } from "vite-plus/test"; + +import { + parseThreadCheckoutPath, + projectVirtualRoot, + runnerProtocolsOverlap, + ThreadMachineStatus, + threadCheckoutPath, +} from "./runner.ts"; + +describe("thread checkout paths", () => { + it("round-trips a thread id through its checkout path", () => { + expect(threadCheckoutPath("thread-1")).toBe("/workspace/t/thread-1"); + expect(parseThreadCheckoutPath("/workspace/t/thread-1")).toBe("thread-1"); + }); + + it("routes paths inside a checkout to its thread", () => { + expect(parseThreadCheckoutPath("/workspace/t/thread-1/src/index.ts")).toBe("thread-1"); + }); + + it("encodes ids that are not a single safe path segment", () => { + const path = threadCheckoutPath("handoff/a b"); + expect(path).toBe("/workspace/t/handoff%2Fa%20b"); + expect(parseThreadCheckoutPath(path)).toBe("handoff/a b"); + expect(threadCheckoutPath("..")).toBe("/workspace/t/%2E%2E"); + expect(parseThreadCheckoutPath(threadCheckoutPath(".."))).toBe(".."); + }); + + it("rejects paths that are not a thread checkout", () => { + expect(parseThreadCheckoutPath("/workspace/t")).toBeNull(); + expect(parseThreadCheckoutPath("/workspace/t/")).toBeNull(); + expect(parseThreadCheckoutPath("/workspace/tt/thread-1")).toBeNull(); + expect(parseThreadCheckoutPath("/workspace/p/project-1")).toBeNull(); + expect(parseThreadCheckoutPath("/workspace/t/thread-1/../thread-2")).toBeNull(); + expect(parseThreadCheckoutPath("/workspace/t/%E0%A4%A")).toBeNull(); + }); + + it("honors a custom checkout root", () => { + const path = threadCheckoutPath("thread-1", "/tmp/hub/t/"); + expect(path).toBe("/tmp/hub/t/thread-1"); + expect(parseThreadCheckoutPath(path, "/tmp/hub/t")).toBe("thread-1"); + expect(parseThreadCheckoutPath(path)).toBeNull(); + }); + + it("names virtual project roots under /workspace/p", () => { + expect(projectVirtualRoot("project-1")).toBe("/workspace/p/project-1"); + }); +}); + +describe("runner protocol negotiation", () => { + it("accepts overlapping version ranges and refuses disjoint ones", () => { + const base = { hubProtocolVersion: 2, hubMinProtocolVersion: 1 }; + expect( + runnerProtocolsOverlap({ ...base, runnerProtocolVersion: 1, runnerMinProtocolVersion: 1 }), + ).toBe(true); + expect( + runnerProtocolsOverlap({ ...base, runnerProtocolVersion: 3, runnerMinProtocolVersion: 2 }), + ).toBe(true); + expect( + runnerProtocolsOverlap({ ...base, runnerProtocolVersion: 4, runnerMinProtocolVersion: 3 }), + ).toBe(false); + expect( + runnerProtocolsOverlap({ + hubProtocolVersion: 5, + hubMinProtocolVersion: 4, + runnerProtocolVersion: 3, + runnerMinProtocolVersion: 1, + }), + ).toBe(false); + }); +}); + +describe("machine directory status", () => { + it("decodes the documented response shapes", () => { + const decode = Schema.decodeUnknownSync(ThreadMachineStatus); + expect(decode({ state: "paused", bootId: "boot-1", detail: null })).toEqual({ + state: "paused", + bootId: "boot-1", + detail: null, + }); + expect( + decode({ + state: "running", + runner: { url: "wss://runner", token: "t", expiresAt: "2026-09-25T00:00:00.000Z" }, + bootId: "boot-2", + }).runner?.url, + ).toBe("wss://runner"); + expect(() => decode({ state: "sleeping" })).toThrow(); + }); +}); diff --git a/packages/contracts/src/runner.ts b/packages/contracts/src/runner.ts new file mode 100644 index 000000000000..788338454328 --- /dev/null +++ b/packages/contracts/src/runner.ts @@ -0,0 +1,964 @@ +/** + * Runner protocol: the contract between a hub and a thread machine's runner. + * + * A hub (`T3CODE_SERVER_MODE=hub`) owns orchestration, projections and the + * provider session directory but no checkout. Each thread's work runs on its + * own machine, where `t3 runner` serves this RPC group for exactly one thread + * over one authenticated WebSocket at `RUNNER_WS_PATH`. + * + * The protocol is versioned independently of the client protocol. The hub + * sends its supported range in `runner.hello`; a runner outside that range, or + * one bound to a different thread, refuses the handshake with a typed error. + * + * Payloads and results reuse the client contract schemas, so a routed call has + * exactly the shape the in-process service already uses. Errors that already + * have a contract schema (VCS, git, terminal, review, text generation) cross + * the wire unchanged. Server-internal error unions (provider adapter and + * workspace errors) travel inside `RunnerRemoteError`, encoded with the + * server's own schema; hub and runner run the same build for a protocol + * version, so the hub decodes them back to the original class. + * + * See docs/internals/thread-machines.md. + * + * @module runner + */ +import * as Schema from "effect/Schema"; +import * as Rpc from "effect/unstable/rpc/Rpc"; +import * as RpcGroup from "effect/unstable/rpc/RpcGroup"; + +import { + ApprovalRequestId, + CheckpointRef, + EnvironmentId, + IsoDateTime, + NonNegativeInt, + ProjectId, + ThreadId, + TrimmedNonEmptyString, + TurnId, +} from "./baseSchemas.ts"; +import { + GitActionProgressEvent, + GitCommandError, + GitManagerServiceError, + GitPreparePullRequestThreadInput, + GitPreparePullRequestThreadResult, + GitPullRequestRefInput, + GitResolvePullRequestResult, + GitRunStackedActionInput, + GitRunStackedActionResult, + TextGenerationError, + VcsCreateRefInput, + VcsCreateRefResult, + VcsCreateWorktreeInput, + VcsCreateWorktreeResult, + VcsInitInput, + VcsListRefsInput, + VcsListRefsResult, + VcsPullResult, + VcsRemoveWorktreeInput, + VcsStatusInput, + VcsStatusLocalResult, + VcsStatusRemoteResult, + VcsStatusResult, + VcsStatusStreamEvent, + VcsSwitchRefInput, + VcsSwitchRefResult, +} from "./git.ts"; +import { + ChatAttachment, + ModelSelection, + ProviderApprovalDecision, + ProviderUserInputAnswers, +} from "./orchestration.ts"; +import { + ProjectListEntriesInput, + ProjectListEntriesResult, + ProjectReadFileInput, + ProjectReadFileResult, + ProjectSearchContentsInput, + ProjectSearchContentsResult, + ProjectSearchEntriesInput, + ProjectSearchEntriesResult, + ProjectWriteFileInput, + ProjectWriteFileResult, +} from "./project.ts"; +import { + ProviderSendTurnInput, + ProviderSession, + ProviderSessionStartInput, + ProviderTurnStartResult, +} from "./provider.ts"; +import { ProviderInstanceId } from "./providerInstance.ts"; +import { ProviderRuntimeEvent } from "./providerRuntime.ts"; +import { + ReviewDiffFileContentsInput, + ReviewDiffFileContentsResult, + ReviewDiffPreviewError, + ReviewDiffPreviewInput, + ReviewDiffPreviewResult, +} from "./review.ts"; +import { ServerProvider } from "./server.ts"; +import { + TerminalAttachInput, + TerminalAttachStreamEvent, + TerminalClearInput, + TerminalCloseInput, + TerminalError, + TerminalEvent, + TerminalMetadataStreamEvent, + TerminalOpenInput, + TerminalResizeInput, + TerminalRestartInput, + TerminalSessionSnapshot, + TerminalWriteInput, +} from "./terminal.ts"; +import { VcsError } from "./vcs.ts"; + +/** Current runner protocol version. Bump when an RPC changes incompatibly. */ +export const RUNNER_PROTOCOL_VERSION = 1; +/** Oldest protocol version this build still speaks. */ +export const RUNNER_MIN_PROTOCOL_VERSION = 1; +export const RUNNER_WS_PATH = "/runner/ws"; + +// ── Checkout paths ───────────────────────────────────────────────────── + +/** Default root of per-thread checkouts on thread machines. */ +export const THREAD_CHECKOUT_ROOT = "/workspace/t"; +/** Root of the virtual project roots a hub assigns; nothing reads them. */ +export const PROJECT_VIRTUAL_ROOT = "/workspace/p"; + +const trimTrailingSlashes = (value: string) => value.replace(/\/+$/, ""); + +/** Thread ids are opaque; encode them so each is exactly one safe path segment. */ +function encodePathSegment(value: string): string { + const encoded = encodeURIComponent(value); + return encoded === "." || encoded === ".." ? encoded.replaceAll(".", "%2E") : encoded; +} + +function decodePathSegment(segment: string): string | null { + try { + return decodeURIComponent(segment); + } catch { + return null; + } +} + +/** The checkout path of a thread: `/`. */ +export function threadCheckoutPath(threadId: string, root: string = THREAD_CHECKOUT_ROOT): string { + return `${trimTrailingSlashes(root)}/${encodePathSegment(threadId)}`; +} + +/** The virtual workspace root of a project in hub mode. */ +export function projectVirtualRoot(projectId: string): string { + return `${PROJECT_VIRTUAL_ROOT}/${encodePathSegment(projectId)}`; +} + +/** + * The thread whose checkout contains `path`, or null when the path is not + * inside `/`. Paths inside the checkout (`//src`) + * resolve to the same thread; `.` and `..` segments are rejected rather than + * normalized, so a path can never escape into another thread's checkout. + */ +export function parseThreadCheckoutPath( + path: string, + root: string = THREAD_CHECKOUT_ROOT, +): ThreadId | null { + const prefix = `${trimTrailingSlashes(root)}/`; + if (!path.startsWith(prefix)) return null; + const segments = path.slice(prefix.length).split("/"); + if (segments.some((segment) => segment === "." || segment === "..")) return null; + const [threadSegment = ""] = segments; + if (threadSegment.length === 0) return null; + const threadId = decodePathSegment(threadSegment); + if (threadId === null || threadId.trim().length === 0 || threadId !== threadId.trim()) { + return null; + } + return ThreadId.make(threadId); +} + +// ── Thread machine directory ─────────────────────────────────────────── + +/** Lifecycle state reported by the machine directory. */ +export const ThreadMachineState = Schema.Literals([ + "none", + "preparing", + "starting", + "running", + "paused", + "saved", + "failed", +]); +export type ThreadMachineState = typeof ThreadMachineState.Type; + +/** Repository a new thread machine checks out, from the project's identity. */ +export const ThreadMachineRepository = Schema.Struct({ + url: TrimmedNonEmptyString, + ref: Schema.NullOr(TrimmedNonEmptyString), +}); +export type ThreadMachineRepository = typeof ThreadMachineRepository.Type; + +/** `POST /threads/{threadId}/machine` body. */ +export const ThreadMachineEnsureRequest = Schema.Struct({ + projectId: Schema.NullOr(ProjectId), + repository: Schema.NullOr(ThreadMachineRepository), + branch: Schema.NullOr(TrimmedNonEmptyString), + checkout: TrimmedNonEmptyString, + wake: Schema.Boolean, +}); +export type ThreadMachineEnsureRequest = typeof ThreadMachineEnsureRequest.Type; + +export const ThreadMachineRunnerEndpoint = Schema.Struct({ + url: TrimmedNonEmptyString, + token: TrimmedNonEmptyString, + expiresAt: Schema.optional(Schema.NullOr(IsoDateTime)), +}); +export type ThreadMachineRunnerEndpoint = typeof ThreadMachineRunnerEndpoint.Type; + +/** Machine directory response for `POST` and `GET`. */ +export const ThreadMachineStatus = Schema.Struct({ + state: ThreadMachineState, + runner: Schema.optional(Schema.NullOr(ThreadMachineRunnerEndpoint)), + bootId: Schema.optional(Schema.NullOr(TrimmedNonEmptyString)), + detail: Schema.optional(Schema.NullOr(Schema.String)), +}); +export type ThreadMachineStatus = typeof ThreadMachineStatus.Type; + +export const ThreadMachineUnavailableReason = Schema.Literals([ + /** The machine is paused, saved, or not created, and the call must not wake it. */ + "asleep", + /** Waking did not reach `running` in time. */ + "wake-timeout", + /** The directory reports the machine failed. */ + "failed", + /** The machine was released (thread archived or deleted). */ + "released", + /** The machine is running but its runner could not be reached. */ + "unreachable", + /** The runner speaks an incompatible protocol or serves another thread. */ + "incompatible", + /** The machine directory itself failed or is not configured. */ + "directory", + /** The path does not name a thread checkout, so no machine owns it. */ + "not-a-thread-checkout", +]); +export type ThreadMachineUnavailableReason = typeof ThreadMachineUnavailableReason.Type; + +/** + * A thread's machine cannot serve a call right now. `asleep` is the typed + * "machine asleep" result for reads that never wake a machine; clients can + * render it as a resumable state rather than as a failure. + */ +export class ThreadMachineUnavailableError extends Schema.TaggedErrorClass()( + "ThreadMachineUnavailableError", + { + threadId: Schema.NullOr(ThreadId), + reason: ThreadMachineUnavailableReason, + state: Schema.optional(ThreadMachineState), + operation: Schema.String, + detail: Schema.String, + }, +) { + override get message(): string { + const subject = + this.threadId === null ? "No thread machine" : `Thread machine ${this.threadId}`; + return `${subject} is unavailable for ${this.operation} (${this.reason}): ${this.detail}`; + } +} + +const isThreadMachineUnavailableError = Schema.is(ThreadMachineUnavailableError); + +export const isThreadMachineAsleep = (error: unknown): boolean => + isThreadMachineUnavailableError(error) && error.reason === "asleep"; + +/** A feature that has no hub-mode implementation. Never falls back to the hub's disk. */ +export class HubModeUnsupportedError extends Schema.TaggedErrorClass()( + "HubModeUnsupportedError", + { + operation: Schema.String, + detail: Schema.String, + }, +) { + override get message(): string { + return `${this.operation} is not available on a hub: ${this.detail}`; + } +} + +// ── Handshake ────────────────────────────────────────────────────────── + +export const RunnerHelloInput = Schema.Struct({ + protocolVersion: NonNegativeInt, + minProtocolVersion: NonNegativeInt, + threadId: ThreadId, +}); +export type RunnerHelloInput = typeof RunnerHelloInput.Type; + +export const RunnerHello = Schema.Struct({ + protocolVersion: NonNegativeInt, + /** Stable for the runner's state disk; its outbox sequences belong to it. */ + runnerId: TrimmedNonEmptyString, + /** Changes on every runner process start. */ + bootId: TrimmedNonEmptyString, + threadId: ThreadId, + checkout: TrimmedNonEmptyString, + /** Highest sequence ever appended to the outbox (0 when empty). */ + headSequence: NonNegativeInt, + /** Highest sequence the hub has acknowledged. */ + ackedSequence: NonNegativeInt, + /** Lowest retained sequence; events below it past `ackedSequence` were dropped. */ + firstRetainedSequence: NonNegativeInt, + instances: Schema.Array(ProviderInstanceId), +}); +export type RunnerHello = typeof RunnerHello.Type; + +export class RunnerProtocolMismatchError extends Schema.TaggedErrorClass()( + "RunnerProtocolMismatchError", + { + runnerProtocolVersion: NonNegativeInt, + runnerMinProtocolVersion: NonNegativeInt, + hubProtocolVersion: NonNegativeInt, + hubMinProtocolVersion: NonNegativeInt, + }, +) { + override get message(): string { + return `Runner protocol ${this.runnerMinProtocolVersion}-${this.runnerProtocolVersion} does not overlap hub protocol ${this.hubMinProtocolVersion}-${this.hubProtocolVersion}.`; + } +} + +export class RunnerThreadMismatchError extends Schema.TaggedErrorClass()( + "RunnerThreadMismatchError", + { + requestedThreadId: Schema.String, + runnerThreadId: Schema.String, + }, +) { + override get message(): string { + return `Runner serves thread ${this.runnerThreadId}, not ${this.requestedThreadId}.`; + } +} + +/** A cwd outside the runner's checkout; the runner never serves other paths. */ +export class RunnerCwdOutsideCheckoutError extends Schema.TaggedErrorClass()( + "RunnerCwdOutsideCheckoutError", + { + cwd: Schema.String, + checkout: Schema.String, + operation: Schema.String, + }, +) { + override get message(): string { + return `${this.operation}: ${this.cwd} is outside the runner checkout ${this.checkout}.`; + } +} + +/** + * A server-internal error forwarded from the runner, encoded with the + * server's own schema for that error union. `errorTag` and `message` stay + * readable when the hub cannot decode `encoded`. + */ +export class RunnerRemoteError extends Schema.TaggedErrorClass()( + "RunnerRemoteError", + { + errorTag: Schema.String, + detail: Schema.String, + encoded: Schema.Unknown, + }, +) { + override get message(): string { + return `Runner error ${this.errorTag}: ${this.detail}`; + } +} + +export class RunnerCheckoutError extends Schema.TaggedErrorClass()( + "RunnerCheckoutError", + { + operation: Schema.String, + checkout: Schema.String, + detail: Schema.String, + }, +) { + override get message(): string { + return `Preparing checkout ${this.checkout} failed in ${this.operation}: ${this.detail}`; + } +} + +// ── Events ───────────────────────────────────────────────────────────── + +export const RunnerEventEnvelope = Schema.Struct({ + sequence: NonNegativeInt, + /** Boot of the runner process that appended the event. */ + bootId: TrimmedNonEmptyString, + event: ProviderRuntimeEvent, +}); +export type RunnerEventEnvelope = typeof RunnerEventEnvelope.Type; + +// ── Provider sessions ────────────────────────────────────────────────── + +/** The hub-minted MCP credential and the hub's public MCP endpoint for a session. */ +export const RunnerMcpSession = Schema.Struct({ + environmentId: EnvironmentId, + threadId: ThreadId, + providerSessionId: TrimmedNonEmptyString, + providerInstanceId: ProviderInstanceId, + endpoint: TrimmedNonEmptyString, + authorizationHeader: TrimmedNonEmptyString, +}); +export type RunnerMcpSession = typeof RunnerMcpSession.Type; + +/** + * Attachment bytes shipped with a turn. The runner writes them into its own + * attachment store and replaces `hubPath` in the prompt with its local path. + */ +export const RunnerAttachmentFile = Schema.Struct({ + attachment: ChatAttachment, + hubPath: TrimmedNonEmptyString, + bytesBase64: Schema.String, +}); +export type RunnerAttachmentFile = typeof RunnerAttachmentFile.Type; + +export const RunnerThreadSnapshot = Schema.Struct({ + threadId: ThreadId, + turns: Schema.Array(Schema.Struct({ id: TurnId, items: Schema.Array(Schema.Unknown) })), +}); +export type RunnerThreadSnapshot = typeof RunnerThreadSnapshot.Type; + +const InstanceScoped = { instanceId: ProviderInstanceId } as const; +const CwdScoped = { cwd: TrimmedNonEmptyString } as const; + +export const RunnerHelloRpc = Rpc.make("runner.hello", { + payload: RunnerHelloInput, + success: RunnerHello, + error: Schema.Union([RunnerProtocolMismatchError, RunnerThreadMismatchError]), +}); + +export const RunnerStartSessionRpc = Rpc.make("runner.provider.startSession", { + payload: Schema.Struct({ + ...InstanceScoped, + input: ProviderSessionStartInput, + mcp: Schema.NullOr(RunnerMcpSession), + }), + success: ProviderSession, + error: RunnerRemoteError, +}); + +export const RunnerSendTurnRpc = Rpc.make("runner.provider.sendTurn", { + payload: Schema.Struct({ + ...InstanceScoped, + input: ProviderSendTurnInput, + attachments: Schema.Array(RunnerAttachmentFile), + }), + success: ProviderTurnStartResult, + error: RunnerRemoteError, +}); + +export const RunnerInterruptTurnRpc = Rpc.make("runner.provider.interruptTurn", { + payload: Schema.Struct({ + ...InstanceScoped, + threadId: ThreadId, + turnId: Schema.optional(TurnId), + }), + error: RunnerRemoteError, +}); + +export const RunnerRespondToRequestRpc = Rpc.make("runner.provider.respondToRequest", { + payload: Schema.Struct({ + ...InstanceScoped, + threadId: ThreadId, + requestId: ApprovalRequestId, + decision: ProviderApprovalDecision, + }), + error: RunnerRemoteError, +}); + +export const RunnerRespondToUserInputRpc = Rpc.make("runner.provider.respondToUserInput", { + payload: Schema.Struct({ + ...InstanceScoped, + threadId: ThreadId, + requestId: ApprovalRequestId, + answers: ProviderUserInputAnswers, + }), + error: RunnerRemoteError, +}); + +export const RunnerStopSessionRpc = Rpc.make("runner.provider.stopSession", { + payload: Schema.Struct({ ...InstanceScoped, threadId: ThreadId }), + error: RunnerRemoteError, +}); + +export const RunnerListSessionsRpc = Rpc.make("runner.provider.listSessions", { + payload: Schema.Struct(InstanceScoped), + success: Schema.Array(ProviderSession), + error: RunnerRemoteError, +}); + +export const RunnerReadThreadRpc = Rpc.make("runner.provider.readThread", { + payload: Schema.Struct({ ...InstanceScoped, threadId: ThreadId }), + success: RunnerThreadSnapshot, + error: RunnerRemoteError, +}); + +export const RunnerRollbackThreadRpc = Rpc.make("runner.provider.rollbackThread", { + payload: Schema.Struct({ ...InstanceScoped, threadId: ThreadId, numTurns: NonNegativeInt }), + success: RunnerThreadSnapshot, + error: RunnerRemoteError, +}); + +/** Provider status and models as the runner's real driver sees them. */ +export const RunnerGetCapabilitiesRpc = Rpc.make("runner.provider.getCapabilities", { + payload: Schema.Struct({ ...InstanceScoped, refresh: Schema.optional(Schema.Boolean) }), + success: Schema.Struct({ + snapshot: ServerProvider, + sessionModelSwitch: Schema.Literals(["in-session", "unsupported"]), + }), + error: RunnerRemoteError, +}); + +// ── Text generation ──────────────────────────────────────────────────── + +const TextGenerationPayload = { + ...InstanceScoped, + ...CwdScoped, + message: Schema.String, + attachments: Schema.Array(RunnerAttachmentFile), + modelSelection: ModelSelection, +} as const; + +export const RunnerGenerateThreadTitleRpc = Rpc.make("runner.text.generateThreadTitle", { + payload: Schema.Struct({ + ...TextGenerationPayload, + previousTitle: Schema.optional(Schema.String), + }), + success: Schema.Struct({ title: Schema.String }), + error: TextGenerationError, +}); + +export const RunnerGenerateBranchNameRpc = Rpc.make("runner.text.generateBranchName", { + payload: Schema.Struct(TextGenerationPayload), + success: Schema.Struct({ branch: Schema.String }), + error: TextGenerationError, +}); + +// ── Event outbox ─────────────────────────────────────────────────────── + +/** + * Replays every retained event with `sequence > afterSequence`, then tails + * live events. The hub may resubscribe from any point at or after its ack. + */ +export const RunnerSubscribeEventsRpc = Rpc.make("runner.events.subscribe", { + payload: Schema.Struct({ afterSequence: NonNegativeInt }), + success: RunnerEventEnvelope, + stream: true, +}); + +/** Marks every event with `sequence <= throughSequence` durable on the hub. */ +export const RunnerAckEventsRpc = Rpc.make("runner.events.ack", { + payload: Schema.Struct({ throughSequence: NonNegativeInt }), + success: Schema.Struct({ ackedSequence: NonNegativeInt, retained: NonNegativeInt }), +}); + +// ── Checkout ─────────────────────────────────────────────────────────── + +export const RunnerPrepareCheckoutInput = Schema.Struct({ + threadId: ThreadId, + checkout: TrimmedNonEmptyString, + repository: Schema.NullOr(ThreadMachineRepository), + /** Branch the thread works on; created from `baseRef` when missing. */ + branch: Schema.NullOr(TrimmedNonEmptyString), + baseRef: Schema.NullOr(TrimmedNonEmptyString), +}); +export type RunnerPrepareCheckoutInput = typeof RunnerPrepareCheckoutInput.Type; + +export const RunnerPrepareCheckoutResult = Schema.Struct({ + checkout: TrimmedNonEmptyString, + isRepository: Schema.Boolean, + branch: Schema.NullOr(TrimmedNonEmptyString), + headCommit: Schema.NullOr(TrimmedNonEmptyString), + /** True when this call cloned or initialized the checkout. */ + created: Schema.Boolean, +}); +export type RunnerPrepareCheckoutResult = typeof RunnerPrepareCheckoutResult.Type; + +/** Idempotent: clones or fetches with the machine's git credentials and checks out `branch`. */ +export const RunnerPrepareCheckoutRpc = Rpc.make("runner.checkout.prepare", { + payload: RunnerPrepareCheckoutInput, + success: RunnerPrepareCheckoutResult, + error: RunnerCheckoutError, +}); + +// ── Checkpoints ──────────────────────────────────────────────────────── + +export const RunnerIsGitRepositoryRpc = Rpc.make("runner.checkpoint.isGitRepository", { + payload: Schema.Struct(CwdScoped), + success: Schema.Boolean, + error: VcsError, +}); + +export const RunnerCaptureCheckpointRpc = Rpc.make("runner.checkpoint.capture", { + payload: Schema.Struct({ ...CwdScoped, checkpointRef: CheckpointRef }), + error: VcsError, +}); + +export const RunnerHasCheckpointRefRpc = Rpc.make("runner.checkpoint.hasRef", { + payload: Schema.Struct({ ...CwdScoped, checkpointRef: CheckpointRef }), + success: Schema.Boolean, + error: VcsError, +}); + +export const RunnerRestoreCheckpointRpc = Rpc.make("runner.checkpoint.restore", { + payload: Schema.Struct({ + ...CwdScoped, + checkpointRef: CheckpointRef, + fallbackToHead: Schema.optional(Schema.Boolean), + }), + success: Schema.Boolean, + error: VcsError, +}); + +export const RunnerDiffCheckpointsRpc = Rpc.make("runner.checkpoint.diff", { + payload: Schema.Struct({ + ...CwdScoped, + fromCheckpointRef: CheckpointRef, + toCheckpointRef: CheckpointRef, + fallbackFromToHead: Schema.optional(Schema.Boolean), + ignoreWhitespace: Schema.Boolean, + }), + success: Schema.String, + error: VcsError, +}); + +export const RunnerDeleteCheckpointRefsRpc = Rpc.make("runner.checkpoint.deleteRefs", { + payload: Schema.Struct({ ...CwdScoped, checkpointRefs: Schema.Array(CheckpointRef) }), + error: VcsError, +}); + +// ── Workspace files ──────────────────────────────────────────────────── + +export const RunnerListEntriesRpc = Rpc.make("runner.workspace.listEntries", { + payload: ProjectListEntriesInput, + success: ProjectListEntriesResult, + error: RunnerRemoteError, +}); + +export const RunnerSearchEntriesRpc = Rpc.make("runner.workspace.searchEntries", { + payload: ProjectSearchEntriesInput, + success: ProjectSearchEntriesResult, + error: RunnerRemoteError, +}); + +export const RunnerSearchContentsRpc = Rpc.make("runner.workspace.searchContents", { + payload: ProjectSearchContentsInput, + success: ProjectSearchContentsResult, + error: RunnerRemoteError, +}); + +export const RunnerReadFileRpc = Rpc.make("runner.workspace.readFile", { + payload: ProjectReadFileInput, + success: ProjectReadFileResult, + error: RunnerRemoteError, +}); + +export const RunnerWriteFileRpc = Rpc.make("runner.workspace.writeFile", { + payload: ProjectWriteFileInput, + success: ProjectWriteFileResult, + error: RunnerRemoteError, +}); + +export const RunnerRefreshWorkspaceIndexRpc = Rpc.make("runner.workspace.refreshIndex", { + payload: Schema.Struct(CwdScoped), +}); + +// ── VCS and git ──────────────────────────────────────────────────────── + +export const RunnerVcsStatusRpc = Rpc.make("runner.vcs.status", { + payload: VcsStatusInput, + success: VcsStatusResult, + error: GitManagerServiceError, +}); + +export const RunnerVcsLocalStatusRpc = Rpc.make("runner.vcs.localStatus", { + payload: VcsStatusInput, + success: VcsStatusLocalResult, + error: GitManagerServiceError, +}); + +export const RunnerVcsRemoteStatusRpc = Rpc.make("runner.vcs.remoteStatus", { + payload: VcsStatusInput, + success: Schema.NullOr(VcsStatusRemoteResult), + error: GitManagerServiceError, +}); + +export const RunnerVcsRefreshStatusRpc = Rpc.make("runner.vcs.refreshStatus", { + payload: Schema.Struct(CwdScoped), + success: VcsStatusResult, + error: GitManagerServiceError, +}); + +export const RunnerVcsRefreshLocalStatusRpc = Rpc.make("runner.vcs.refreshLocalStatus", { + payload: Schema.Struct(CwdScoped), + success: VcsStatusLocalResult, + error: GitManagerServiceError, +}); + +/** Pushes the checkout's status to the hub cache while a connection is open. */ +export const RunnerVcsStreamStatusRpc = Rpc.make("runner.vcs.streamStatus", { + payload: VcsStatusInput, + success: VcsStatusStreamEvent, + error: GitManagerServiceError, + stream: true, +}); + +export const RunnerVcsInitRpc = Rpc.make("runner.vcs.init", { + payload: VcsInitInput, + error: VcsError, +}); + +export const RunnerGitPullRpc = Rpc.make("runner.git.pull", { + payload: Schema.Struct(CwdScoped), + success: VcsPullResult, + error: GitCommandError, +}); + +export const RunnerGitStackedActionEvent = Schema.Union([ + Schema.TaggedStruct("progress", { event: GitActionProgressEvent }), + Schema.TaggedStruct("result", { result: GitRunStackedActionResult }), +]); +export type RunnerGitStackedActionEvent = typeof RunnerGitStackedActionEvent.Type; + +/** Commit, push and PR creation run on the runner; progress streams back, then the result. */ +export const RunnerGitRunStackedActionRpc = Rpc.make("runner.git.runStackedAction", { + payload: GitRunStackedActionInput, + success: RunnerGitStackedActionEvent, + error: GitManagerServiceError, + stream: true, +}); + +export const RunnerGitResolvePullRequestRpc = Rpc.make("runner.git.resolvePullRequest", { + payload: GitPullRequestRefInput, + success: GitResolvePullRequestResult, + error: GitManagerServiceError, +}); + +export const RunnerGitPreparePullRequestThreadRpc = Rpc.make( + "runner.git.preparePullRequestThread", + { + payload: GitPreparePullRequestThreadInput, + success: GitPreparePullRequestThreadResult, + error: GitManagerServiceError, + }, +); + +export const RunnerGitListRefsRpc = Rpc.make("runner.git.listRefs", { + payload: VcsListRefsInput, + success: VcsListRefsResult, + error: GitCommandError, +}); + +export const RunnerGitCreateWorktreeRpc = Rpc.make("runner.git.createWorktree", { + payload: VcsCreateWorktreeInput, + success: VcsCreateWorktreeResult, + error: GitCommandError, +}); + +export const RunnerGitRemoveWorktreeRpc = Rpc.make("runner.git.removeWorktree", { + payload: VcsRemoveWorktreeInput, + error: GitCommandError, +}); + +export const RunnerGitPruneWorktreesRpc = Rpc.make("runner.git.pruneWorktrees", { + payload: Schema.Struct(CwdScoped), + error: GitCommandError, +}); + +export const RunnerGitCreateRefRpc = Rpc.make("runner.git.createRef", { + payload: VcsCreateRefInput, + success: VcsCreateRefResult, + error: GitCommandError, +}); + +export const RunnerGitSwitchRefRpc = Rpc.make("runner.git.switchRef", { + payload: VcsSwitchRefInput, + success: VcsSwitchRefResult, + error: GitCommandError, +}); + +export const RunnerGitRenameBranchRpc = Rpc.make("runner.git.renameBranch", { + payload: Schema.Struct({ ...CwdScoped, oldBranch: Schema.String, newBranch: Schema.String }), + success: Schema.Struct({ branch: Schema.String }), + error: GitManagerServiceError, +}); + +export const RunnerGitFetchRemoteRpc = Rpc.make("runner.git.fetchRemote", { + payload: Schema.Struct({ ...CwdScoped, remoteName: Schema.String }), + error: GitCommandError, +}); + +export const RunnerGitRemoteExistsRpc = Rpc.make("runner.git.remoteExists", { + payload: Schema.Struct({ ...CwdScoped, remoteName: Schema.String }), + success: Schema.Boolean, + error: GitCommandError, +}); + +export const RunnerGitResolveRemoteTrackingCommitRpc = Rpc.make( + "runner.git.resolveRemoteTrackingCommit", + { + payload: Schema.Struct({ + ...CwdScoped, + refName: Schema.String, + fallbackRemoteName: Schema.String, + }), + success: Schema.Struct({ commitSha: Schema.String, remoteRefName: Schema.String }), + error: GitCommandError, + }, +); + +// ── Review ───────────────────────────────────────────────────────────── + +export const RunnerReviewDiffPreviewRpc = Rpc.make("runner.review.getDiffPreview", { + payload: ReviewDiffPreviewInput, + success: ReviewDiffPreviewResult, + error: ReviewDiffPreviewError, +}); + +export const RunnerReviewDiffFileContentsRpc = Rpc.make("runner.review.getDiffFileContents", { + payload: ReviewDiffFileContentsInput, + success: ReviewDiffFileContentsResult, + error: ReviewDiffPreviewError, +}); + +// ── Terminals ────────────────────────────────────────────────────────── + +export const RunnerTerminalOpenRpc = Rpc.make("runner.terminal.open", { + payload: TerminalOpenInput, + success: TerminalSessionSnapshot, + error: TerminalError, +}); + +export const RunnerTerminalAttachRpc = Rpc.make("runner.terminal.attach", { + payload: TerminalAttachInput, + success: TerminalAttachStreamEvent, + error: TerminalError, + stream: true, +}); + +export const RunnerTerminalWriteRpc = Rpc.make("runner.terminal.write", { + payload: TerminalWriteInput, + error: TerminalError, +}); + +export const RunnerTerminalResizeRpc = Rpc.make("runner.terminal.resize", { + payload: TerminalResizeInput, + error: TerminalError, +}); + +export const RunnerTerminalClearRpc = Rpc.make("runner.terminal.clear", { + payload: TerminalClearInput, + error: TerminalError, +}); + +export const RunnerTerminalRestartRpc = Rpc.make("runner.terminal.restart", { + payload: TerminalRestartInput, + success: TerminalSessionSnapshot, + error: TerminalError, +}); + +export const RunnerTerminalCloseRpc = Rpc.make("runner.terminal.close", { + payload: TerminalCloseInput, + error: TerminalError, +}); + +export const RunnerTerminalEventsRpc = Rpc.make("runner.terminal.events", { + payload: Schema.Struct({}), + success: TerminalEvent, + stream: true, +}); + +/** Starts with a full snapshot of the runner's terminals, then upserts and removes. */ +export const RunnerTerminalMetadataRpc = Rpc.make("runner.terminal.metadata", { + payload: Schema.Struct({}), + success: TerminalMetadataStreamEvent, + stream: true, +}); + +export const RunnerRpcGroup = RpcGroup.make( + RunnerHelloRpc, + RunnerStartSessionRpc, + RunnerSendTurnRpc, + RunnerInterruptTurnRpc, + RunnerRespondToRequestRpc, + RunnerRespondToUserInputRpc, + RunnerStopSessionRpc, + RunnerListSessionsRpc, + RunnerReadThreadRpc, + RunnerRollbackThreadRpc, + RunnerGetCapabilitiesRpc, + RunnerGenerateThreadTitleRpc, + RunnerGenerateBranchNameRpc, + RunnerSubscribeEventsRpc, + RunnerAckEventsRpc, + RunnerPrepareCheckoutRpc, + RunnerIsGitRepositoryRpc, + RunnerCaptureCheckpointRpc, + RunnerHasCheckpointRefRpc, + RunnerRestoreCheckpointRpc, + RunnerDiffCheckpointsRpc, + RunnerDeleteCheckpointRefsRpc, + RunnerListEntriesRpc, + RunnerSearchEntriesRpc, + RunnerSearchContentsRpc, + RunnerReadFileRpc, + RunnerWriteFileRpc, + RunnerRefreshWorkspaceIndexRpc, + RunnerVcsStatusRpc, + RunnerVcsLocalStatusRpc, + RunnerVcsRemoteStatusRpc, + RunnerVcsRefreshStatusRpc, + RunnerVcsRefreshLocalStatusRpc, + RunnerVcsStreamStatusRpc, + RunnerVcsInitRpc, + RunnerGitPullRpc, + RunnerGitRunStackedActionRpc, + RunnerGitResolvePullRequestRpc, + RunnerGitPreparePullRequestThreadRpc, + RunnerGitListRefsRpc, + RunnerGitCreateWorktreeRpc, + RunnerGitRemoveWorktreeRpc, + RunnerGitPruneWorktreesRpc, + RunnerGitCreateRefRpc, + RunnerGitSwitchRefRpc, + RunnerGitRenameBranchRpc, + RunnerGitFetchRemoteRpc, + RunnerGitRemoteExistsRpc, + RunnerGitResolveRemoteTrackingCommitRpc, + RunnerReviewDiffPreviewRpc, + RunnerReviewDiffFileContentsRpc, + RunnerTerminalOpenRpc, + RunnerTerminalAttachRpc, + RunnerTerminalWriteRpc, + RunnerTerminalResizeRpc, + RunnerTerminalClearRpc, + RunnerTerminalRestartRpc, + RunnerTerminalCloseRpc, + RunnerTerminalEventsRpc, + RunnerTerminalMetadataRpc, +); + +/** Whether a runner speaking `[runnerMin, runnerMax]` can serve a hub speaking `[hubMin, hubMax]`. */ +export function runnerProtocolsOverlap(input: { + readonly runnerProtocolVersion: number; + readonly runnerMinProtocolVersion: number; + readonly hubProtocolVersion: number; + readonly hubMinProtocolVersion: number; +}): boolean { + return ( + input.runnerMinProtocolVersion <= input.hubProtocolVersion && + input.hubMinProtocolVersion <= input.runnerProtocolVersion + ); +} + +/** The version both sides speak: the highest version in the overlap. */ +export function negotiatedRunnerProtocolVersion(input: { + readonly runnerProtocolVersion: number; + readonly hubProtocolVersion: number; +}): number { + return Math.min(input.runnerProtocolVersion, input.hubProtocolVersion); +} From 98a41a3399c2825f3bca6858f0986adcaa50d5d4 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 22:55:31 +0000 Subject: [PATCH 11/58] feat(server): persist hub thread-machine state Add the hub repositories that let a hub serve history without waking machines and resume runner delivery exactly: RunnerCursorStore (outbox identity, last reconciled boot, durable sequence per thread), CheckpointTurnDiffStore (patches between checkpoints, both whitespace modes), and ThreadVcsStatusStore (last reported git status per thread). Postgres implementations are tenant-scoped and ship as hub migration 050 with the row-level-security backstop; the SQLite implementations back hub mode without a database URL and create their tables on first use, so standalone databases never gain hub-only tables. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Layers/HubThreadMachineState.ts | 222 ++++++++++++++++++ .../Postgres/HubThreadMachineState.ts | 194 +++++++++++++++ .../migrations/050_HubThreadMachineState.ts | 85 +++++++ .../Services/HubThreadMachineState.ts | 116 +++++++++ 4 files changed, 617 insertions(+) create mode 100644 apps/server/src/persistence/Layers/HubThreadMachineState.ts create mode 100644 apps/server/src/persistence/Postgres/HubThreadMachineState.ts create mode 100644 apps/server/src/persistence/Postgres/migrations/050_HubThreadMachineState.ts create mode 100644 apps/server/src/persistence/Services/HubThreadMachineState.ts diff --git a/apps/server/src/persistence/Layers/HubThreadMachineState.ts b/apps/server/src/persistence/Layers/HubThreadMachineState.ts new file mode 100644 index 000000000000..644b4fddddee --- /dev/null +++ b/apps/server/src/persistence/Layers/HubThreadMachineState.ts @@ -0,0 +1,222 @@ +/** + * SQLite implementations of the hub thread-machine repositories. + * + * Used only in hub mode without `T3CODE_HUB_DATABASE_URL` (tests and local + * development). The tables are created on first use instead of through a + * numbered migration so standalone databases never gain hub-only tables; the + * production schema is hub Postgres migration 050. + * + * @module HubThreadMachineStateSqlite + */ +import { VcsStatusLocalResult, VcsStatusRemoteResult, type ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; + +import { toPersistenceDecodeError, toPersistenceSqlError } from "../Errors.ts"; +import { + CheckpointTurnDiff, + CheckpointTurnDiffKey, + CheckpointTurnDiffStore, + RunnerCursor, + RunnerCursorStore, + ThreadVcsStatus, + ThreadVcsStatusStore, +} from "../Services/HubThreadMachineState.ts"; + +const HUB_SQLITE_TABLES = [ + `CREATE TABLE IF NOT EXISTS hub_runner_cursors ( + thread_id TEXT PRIMARY KEY, + outbox_id TEXT NOT NULL, + boot_id TEXT NOT NULL, + acked_sequence INTEGER NOT NULL, + updated_at TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS hub_checkpoint_turn_diffs ( + thread_id TEXT NOT NULL, + from_turn_count INTEGER NOT NULL, + to_turn_count INTEGER NOT NULL, + ignore_whitespace INTEGER NOT NULL, + diff TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (thread_id, from_turn_count, to_turn_count, ignore_whitespace) + )`, + `CREATE TABLE IF NOT EXISTS hub_thread_vcs_status ( + thread_id TEXT PRIMARY KEY, + local_json TEXT NOT NULL, + remote_json TEXT, + updated_at TEXT NOT NULL + )`, +] as const; + +const ensureTables = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + for (const statement of HUB_SQLITE_TABLES) { + yield* sql.unsafe(statement); + } +}).pipe(Effect.mapError(toPersistenceSqlError("HubThreadMachineState.ensureTables"))); + +const sqlOrDecode = (operation: string) => (cause: unknown) => + Schema.isSchemaError(cause) + ? toPersistenceDecodeError(operation)(cause) + : toPersistenceSqlError(operation)(cause); + +const RunnerCursorRow = RunnerCursor; + +export const makeRunnerCursorStore = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* ensureTables; + + const selectOne = SqlSchema.findOneOption({ + Request: Schema.String, + Result: RunnerCursorRow, + execute: (threadId) => sql` + SELECT thread_id AS "threadId", outbox_id AS "outboxId", boot_id AS "bootId", + acked_sequence AS "ackedSequence", updated_at AS "updatedAt" + FROM hub_runner_cursors WHERE thread_id = ${threadId} + `, + }); + const selectAll = SqlSchema.findAll({ + Request: Schema.Void, + Result: RunnerCursorRow, + execute: () => sql` + SELECT thread_id AS "threadId", outbox_id AS "outboxId", boot_id AS "bootId", + acked_sequence AS "ackedSequence", updated_at AS "updatedAt" + FROM hub_runner_cursors ORDER BY thread_id + `, + }); + const upsert = SqlSchema.void({ + Request: RunnerCursorRow, + execute: (row) => sql` + INSERT INTO hub_runner_cursors (thread_id, outbox_id, boot_id, acked_sequence, updated_at) + VALUES (${row.threadId}, ${row.outboxId}, ${row.bootId}, ${row.ackedSequence}, ${row.updatedAt}) + ON CONFLICT (thread_id) DO UPDATE SET + outbox_id = excluded.outbox_id, + boot_id = excluded.boot_id, + acked_sequence = excluded.acked_sequence, + updated_at = excluded.updated_at + `, + }); + + return RunnerCursorStore.of({ + get: (threadId) => + selectOne(threadId).pipe(Effect.mapError(sqlOrDecode("RunnerCursorStore.get"))), + list: () => selectAll(undefined).pipe(Effect.mapError(sqlOrDecode("RunnerCursorStore.list"))), + saveAll: (cursors) => + sql + .withTransaction(Effect.forEach(cursors, (cursor) => upsert(cursor), { discard: true })) + .pipe(Effect.mapError(sqlOrDecode("RunnerCursorStore.saveAll"))), + remove: (threadId) => + sql`DELETE FROM hub_runner_cursors WHERE thread_id = ${threadId}`.pipe( + Effect.asVoid, + Effect.mapError(toPersistenceSqlError("RunnerCursorStore.remove")), + ), + }); +}); + +export const makeCheckpointTurnDiffStore = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* ensureTables; + + const selectDiff = SqlSchema.findOneOption({ + Request: CheckpointTurnDiffKey, + Result: Schema.Struct({ diff: Schema.String }), + execute: (key) => sql` + SELECT diff FROM hub_checkpoint_turn_diffs + WHERE thread_id = ${key.threadId} + AND from_turn_count = ${key.fromTurnCount} + AND to_turn_count = ${key.toTurnCount} + AND ignore_whitespace = ${key.ignoreWhitespace ? 1 : 0} + `, + }); + const upsert = SqlSchema.void({ + Request: CheckpointTurnDiff, + execute: (row) => sql` + INSERT INTO hub_checkpoint_turn_diffs ( + thread_id, from_turn_count, to_turn_count, ignore_whitespace, diff, created_at + ) + VALUES ( + ${row.threadId}, ${row.fromTurnCount}, ${row.toTurnCount}, + ${row.ignoreWhitespace ? 1 : 0}, ${row.diff}, ${row.createdAt} + ) + ON CONFLICT (thread_id, from_turn_count, to_turn_count, ignore_whitespace) + DO UPDATE SET diff = excluded.diff, created_at = excluded.created_at + `, + }); + + return CheckpointTurnDiffStore.of({ + get: (key) => + selectDiff(key).pipe( + Effect.map(Option.map((row) => row.diff)), + Effect.mapError(sqlOrDecode("CheckpointTurnDiffStore.get")), + ), + put: (row) => upsert(row).pipe(Effect.mapError(sqlOrDecode("CheckpointTurnDiffStore.put"))), + invalidateFromTurn: ({ threadId, turnCount }) => + sql` + DELETE FROM hub_checkpoint_turn_diffs + WHERE thread_id = ${threadId} + AND (to_turn_count >= ${turnCount} OR from_turn_count >= ${turnCount}) + `.pipe( + Effect.asVoid, + Effect.mapError(toPersistenceSqlError("CheckpointTurnDiffStore.invalidateFromTurn")), + ), + removeThread: (threadId: ThreadId) => + sql`DELETE FROM hub_checkpoint_turn_diffs WHERE thread_id = ${threadId}`.pipe( + Effect.asVoid, + Effect.mapError(toPersistenceSqlError("CheckpointTurnDiffStore.removeThread")), + ), + }); +}); + +const ThreadVcsStatusRow = ThreadVcsStatus.mapFields((fields) => ({ + ...fields, + local: Schema.fromJsonString(VcsStatusLocalResult), + remote: Schema.NullOr(Schema.fromJsonString(VcsStatusRemoteResult)), +})); + +export const makeThreadVcsStatusStore = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* ensureTables; + + const selectAll = SqlSchema.findAll({ + Request: Schema.Void, + Result: ThreadVcsStatusRow, + execute: () => sql` + SELECT thread_id AS "threadId", local_json AS "local", remote_json AS "remote", + updated_at AS "updatedAt" + FROM hub_thread_vcs_status + `, + }); + const upsert = SqlSchema.void({ + Request: ThreadVcsStatusRow, + execute: (row) => sql` + INSERT INTO hub_thread_vcs_status (thread_id, local_json, remote_json, updated_at) + VALUES (${row.threadId}, ${row.local}, ${row.remote}, ${row.updatedAt}) + ON CONFLICT (thread_id) DO UPDATE SET + local_json = excluded.local_json, + remote_json = excluded.remote_json, + updated_at = excluded.updated_at + `, + }); + + return ThreadVcsStatusStore.of({ + list: () => + selectAll(undefined).pipe(Effect.mapError(sqlOrDecode("ThreadVcsStatusStore.list"))), + put: (row) => upsert(row).pipe(Effect.mapError(sqlOrDecode("ThreadVcsStatusStore.put"))), + remove: (threadId) => + sql`DELETE FROM hub_thread_vcs_status WHERE thread_id = ${threadId}`.pipe( + Effect.asVoid, + Effect.mapError(toPersistenceSqlError("ThreadVcsStatusStore.remove")), + ), + }); +}); + +/** All three repositories over the configured SQLite client. */ +export const HubThreadMachineStateSqliteLive = Layer.mergeAll( + Layer.effect(RunnerCursorStore, makeRunnerCursorStore), + Layer.effect(CheckpointTurnDiffStore, makeCheckpointTurnDiffStore), + Layer.effect(ThreadVcsStatusStore, makeThreadVcsStatusStore), +); diff --git a/apps/server/src/persistence/Postgres/HubThreadMachineState.ts b/apps/server/src/persistence/Postgres/HubThreadMachineState.ts new file mode 100644 index 000000000000..97b5ca669c5c --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubThreadMachineState.ts @@ -0,0 +1,194 @@ +/** + * Postgres implementations of the hub thread-machine repositories + * (tables from hub migration 050). Every statement is scoped to the + * `HubTenant` user, like the rest of the hub schema. + * + * @module HubThreadMachineStatePostgres + */ +import { VcsStatusLocalResult, VcsStatusRemoteResult, type ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as SqlSchema from "effect/unstable/sql/SqlSchema"; + +import { toPersistenceDecodeError, toPersistenceSqlError } from "../Errors.ts"; +import { + CheckpointTurnDiff, + CheckpointTurnDiffKey, + CheckpointTurnDiffStore, + RunnerCursor, + RunnerCursorStore, + ThreadVcsStatus, + ThreadVcsStatusStore, +} from "../Services/HubThreadMachineState.ts"; +import { HubTenant } from "./HubTenant.ts"; + +const sqlOrDecode = (operation: string) => (cause: unknown) => + Schema.isSchemaError(cause) + ? toPersistenceDecodeError(operation)(cause) + : toPersistenceSqlError(operation)(cause); + +export const makeRunnerCursorStore = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const selectOne = SqlSchema.findOneOption({ + Request: Schema.String, + Result: RunnerCursor, + execute: (threadId) => sql` + SELECT thread_id AS "threadId", outbox_id AS "outboxId", boot_id AS "bootId", + acked_sequence AS "ackedSequence", updated_at AS "updatedAt" + FROM hub_runner_cursors WHERE user_id = ${userId} AND thread_id = ${threadId} + `, + }); + const selectAll = SqlSchema.findAll({ + Request: Schema.Void, + Result: RunnerCursor, + execute: () => sql` + SELECT thread_id AS "threadId", outbox_id AS "outboxId", boot_id AS "bootId", + acked_sequence AS "ackedSequence", updated_at AS "updatedAt" + FROM hub_runner_cursors WHERE user_id = ${userId} ORDER BY thread_id + `, + }); + const upsert = SqlSchema.void({ + Request: RunnerCursor, + execute: (row) => sql` + INSERT INTO hub_runner_cursors ( + user_id, thread_id, outbox_id, boot_id, acked_sequence, updated_at + ) + VALUES ( + ${userId}, ${row.threadId}, ${row.outboxId}, ${row.bootId}, ${row.ackedSequence}, + ${row.updatedAt} + ) + ON CONFLICT (user_id, thread_id) DO UPDATE SET + outbox_id = excluded.outbox_id, + boot_id = excluded.boot_id, + acked_sequence = excluded.acked_sequence, + updated_at = excluded.updated_at + `, + }); + + return RunnerCursorStore.of({ + get: (threadId) => + selectOne(threadId).pipe(Effect.mapError(sqlOrDecode("RunnerCursorStore.get"))), + list: () => selectAll(undefined).pipe(Effect.mapError(sqlOrDecode("RunnerCursorStore.list"))), + saveAll: (cursors) => + sql + .withTransaction(Effect.forEach(cursors, (cursor) => upsert(cursor), { discard: true })) + .pipe(Effect.mapError(sqlOrDecode("RunnerCursorStore.saveAll"))), + remove: (threadId) => + sql`DELETE FROM hub_runner_cursors WHERE user_id = ${userId} AND thread_id = ${threadId}`.pipe( + Effect.asVoid, + Effect.mapError(toPersistenceSqlError("RunnerCursorStore.remove")), + ), + }); +}); + +export const makeCheckpointTurnDiffStore = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const selectDiff = SqlSchema.findOneOption({ + Request: CheckpointTurnDiffKey, + Result: Schema.Struct({ diff: Schema.String }), + execute: (key) => sql` + SELECT diff FROM hub_checkpoint_turn_diffs + WHERE user_id = ${userId} + AND thread_id = ${key.threadId} + AND from_turn_count = ${key.fromTurnCount} + AND to_turn_count = ${key.toTurnCount} + AND ignore_whitespace = ${key.ignoreWhitespace} + `, + }); + const upsert = SqlSchema.void({ + Request: CheckpointTurnDiff, + execute: (row) => sql` + INSERT INTO hub_checkpoint_turn_diffs ( + user_id, thread_id, from_turn_count, to_turn_count, ignore_whitespace, diff, created_at + ) + VALUES ( + ${userId}, ${row.threadId}, ${row.fromTurnCount}, ${row.toTurnCount}, + ${row.ignoreWhitespace}, ${row.diff}, ${row.createdAt} + ) + ON CONFLICT (user_id, thread_id, from_turn_count, to_turn_count, ignore_whitespace) + DO UPDATE SET diff = excluded.diff, created_at = excluded.created_at + `, + }); + + return CheckpointTurnDiffStore.of({ + get: (key) => + selectDiff(key).pipe( + Effect.map(Option.map((row) => row.diff)), + Effect.mapError(sqlOrDecode("CheckpointTurnDiffStore.get")), + ), + put: (row) => upsert(row).pipe(Effect.mapError(sqlOrDecode("CheckpointTurnDiffStore.put"))), + invalidateFromTurn: ({ threadId, turnCount }) => + sql` + DELETE FROM hub_checkpoint_turn_diffs + WHERE user_id = ${userId} + AND thread_id = ${threadId} + AND (to_turn_count >= ${turnCount} OR from_turn_count >= ${turnCount}) + `.pipe( + Effect.asVoid, + Effect.mapError(toPersistenceSqlError("CheckpointTurnDiffStore.invalidateFromTurn")), + ), + removeThread: (threadId: ThreadId) => + sql` + DELETE FROM hub_checkpoint_turn_diffs WHERE user_id = ${userId} AND thread_id = ${threadId} + `.pipe( + Effect.asVoid, + Effect.mapError(toPersistenceSqlError("CheckpointTurnDiffStore.removeThread")), + ), + }); +}); + +const ThreadVcsStatusRow = ThreadVcsStatus.mapFields((fields) => ({ + ...fields, + local: Schema.fromJsonString(VcsStatusLocalResult), + remote: Schema.NullOr(Schema.fromJsonString(VcsStatusRemoteResult)), +})); + +export const makeThreadVcsStatusStore = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const selectAll = SqlSchema.findAll({ + Request: Schema.Void, + Result: ThreadVcsStatusRow, + execute: () => sql` + SELECT thread_id AS "threadId", local_json AS "local", remote_json AS "remote", + updated_at AS "updatedAt" + FROM hub_thread_vcs_status WHERE user_id = ${userId} + `, + }); + const upsert = SqlSchema.void({ + Request: ThreadVcsStatusRow, + execute: (row) => sql` + INSERT INTO hub_thread_vcs_status (user_id, thread_id, local_json, remote_json, updated_at) + VALUES (${userId}, ${row.threadId}, ${row.local}, ${row.remote}, ${row.updatedAt}) + ON CONFLICT (user_id, thread_id) DO UPDATE SET + local_json = excluded.local_json, + remote_json = excluded.remote_json, + updated_at = excluded.updated_at + `, + }); + + return ThreadVcsStatusStore.of({ + list: () => + selectAll(undefined).pipe(Effect.mapError(sqlOrDecode("ThreadVcsStatusStore.list"))), + put: (row) => upsert(row).pipe(Effect.mapError(sqlOrDecode("ThreadVcsStatusStore.put"))), + remove: (threadId) => + sql` + DELETE FROM hub_thread_vcs_status WHERE user_id = ${userId} AND thread_id = ${threadId} + `.pipe(Effect.asVoid, Effect.mapError(toPersistenceSqlError("ThreadVcsStatusStore.remove"))), + }); +}); + +/** All three repositories; requires the hub Postgres `SqlClient` and `HubTenant`. */ +export const HubThreadMachineStatePostgresLive = Layer.mergeAll( + Layer.effect(RunnerCursorStore, makeRunnerCursorStore), + Layer.effect(CheckpointTurnDiffStore, makeCheckpointTurnDiffStore), + Layer.effect(ThreadVcsStatusStore, makeThreadVcsStatusStore), +); diff --git a/apps/server/src/persistence/Postgres/migrations/050_HubThreadMachineState.ts b/apps/server/src/persistence/Postgres/migrations/050_HubThreadMachineState.ts new file mode 100644 index 000000000000..824e5b548e91 --- /dev/null +++ b/apps/server/src/persistence/Postgres/migrations/050_HubThreadMachineState.ts @@ -0,0 +1,85 @@ +/** + * Hub Postgres migration 050: thread-machine state. + * + * - `hub_runner_cursors`: per thread, the runner outbox identity, the last + * runner boot the hub reconciled, and the highest runner event sequence whose + * effects are durable in the hub. + * - `hub_checkpoint_turn_diffs`: patches between two checkpoints of a thread, + * captured while the machine was awake, so diffs never wake a machine. + * - `hub_thread_vcs_status`: the last git status a runner reported per thread. + * + * Every table leads with `user_id` and carries the same row-level-security + * backstop as the hub baseline (`hub.user_id` transaction setting). + * + * Registration: hub migrations 001-049 belong to the hub persistence + * framework (feature/tm-hub). TODO(tm-hub integration): add this module to + * that framework's migration list as id 50, name "HubThreadMachineState". + * Until then `HUB_MIGRATION_050` can be applied directly; every statement is + * idempotent. + * + * @module 050_HubThreadMachineState + */ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +export const HUB_MIGRATION_050_ID = 50; +export const HUB_MIGRATION_050_NAME = "HubThreadMachineState"; + +const c = `COLLATE "C"`; + +export const HUB_THREAD_MACHINE_STATE_TABLES = [ + "hub_runner_cursors", + "hub_checkpoint_turn_diffs", + "hub_thread_vcs_status", +] as const; + +export const HUB_MIGRATION_050_STATEMENTS: ReadonlyArray = [ + `CREATE TABLE IF NOT EXISTS hub_runner_cursors ( + user_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + outbox_id text ${c} NOT NULL, + boot_id text ${c} NOT NULL, + acked_sequence bigint NOT NULL, + updated_at text ${c} NOT NULL, + PRIMARY KEY (user_id, thread_id) + )`, + `CREATE TABLE IF NOT EXISTS hub_checkpoint_turn_diffs ( + user_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + from_turn_count integer NOT NULL, + to_turn_count integer NOT NULL, + ignore_whitespace boolean NOT NULL, + diff text NOT NULL, + created_at text ${c} NOT NULL, + PRIMARY KEY (user_id, thread_id, from_turn_count, to_turn_count, ignore_whitespace) + )`, + `CREATE INDEX IF NOT EXISTS idx_hub_checkpoint_turn_diffs_thread_to_turn + ON hub_checkpoint_turn_diffs (user_id, thread_id, to_turn_count)`, + `CREATE TABLE IF NOT EXISTS hub_thread_vcs_status ( + user_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + local_json text NOT NULL, + remote_json text, + updated_at text ${c} NOT NULL, + PRIMARY KEY (user_id, thread_id) + )`, + ...HUB_THREAD_MACHINE_STATE_TABLES.flatMap((table) => [ + `ALTER TABLE ${table} ENABLE ROW LEVEL SECURITY`, + `DROP POLICY IF EXISTS hub_tenant_isolation ON ${table}`, + `CREATE POLICY hub_tenant_isolation ON ${table} + USING (user_id = current_setting('hub.user_id', true)) + WITH CHECK (user_id = current_setting('hub.user_id', true))`, + ]), +]; + +/** Applies migration 050 idempotently in one transaction. */ +export const HUB_MIGRATION_050 = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql.withTransaction( + Effect.forEach(HUB_MIGRATION_050_STATEMENTS, (statement) => sql.unsafe(statement), { + discard: true, + }), + ); +}); + +export default HUB_MIGRATION_050; diff --git a/apps/server/src/persistence/Services/HubThreadMachineState.ts b/apps/server/src/persistence/Services/HubThreadMachineState.ts new file mode 100644 index 000000000000..d44d3f58cebc --- /dev/null +++ b/apps/server/src/persistence/Services/HubThreadMachineState.ts @@ -0,0 +1,116 @@ +/** + * Hub-only persistence for thread machines. + * + * A hub serves history, diffs and git status without waking machines, and + * resumes runner event delivery exactly where it stopped. These repositories + * hold that state: + * + * - `RunnerCursorStore`: per thread, the runner outbox identity, the last boot + * the hub saw, and the highest event sequence whose effects are durable. + * - `CheckpointTurnDiffStore`: the patch between two checkpoints of a thread, + * captured while the machine was awake. + * - `ThreadVcsStatusStore`: the last git status a runner reported per thread. + * + * Postgres implementations live in `persistence/Postgres/HubThreadMachineState.ts` + * (hub migration 050); the SQLite implementations back hub mode without a + * database URL (tests and local development) and are never built in + * standalone mode. + * + * @module HubThreadMachineState + */ +import { + IsoDateTime, + NonNegativeInt, + ThreadId, + TrimmedNonEmptyString, + VcsStatusLocalResult, + VcsStatusRemoteResult, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import type * as Effect from "effect/Effect"; +import type * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; + +import type { ProjectionRepositoryError } from "../Errors.ts"; + +export const RunnerCursor = Schema.Struct({ + threadId: ThreadId, + /** `runnerId` of the outbox the sequence belongs to; a new outbox restarts at 0. */ + outboxId: TrimmedNonEmptyString, + /** Last runner boot the hub reconciled against. */ + bootId: TrimmedNonEmptyString, + ackedSequence: NonNegativeInt, + updatedAt: IsoDateTime, +}); +export type RunnerCursor = typeof RunnerCursor.Type; + +export interface RunnerCursorStoreShape { + readonly get: ( + threadId: ThreadId, + ) => Effect.Effect, ProjectionRepositoryError>; + readonly list: () => Effect.Effect, ProjectionRepositoryError>; + /** Upserts every cursor in one transaction. */ + readonly saveAll: ( + cursors: ReadonlyArray, + ) => Effect.Effect; + readonly remove: (threadId: ThreadId) => Effect.Effect; +} + +export class RunnerCursorStore extends Context.Service()( + "t3/persistence/Services/HubThreadMachineState/RunnerCursorStore", +) {} + +export const CheckpointTurnDiffKey = Schema.Struct({ + threadId: ThreadId, + fromTurnCount: NonNegativeInt, + toTurnCount: NonNegativeInt, + ignoreWhitespace: Schema.Boolean, +}); +export type CheckpointTurnDiffKey = typeof CheckpointTurnDiffKey.Type; + +export const CheckpointTurnDiff = Schema.Struct({ + ...CheckpointTurnDiffKey.fields, + diff: Schema.String, + createdAt: IsoDateTime, +}); +export type CheckpointTurnDiff = typeof CheckpointTurnDiff.Type; + +export interface CheckpointTurnDiffStoreShape { + readonly get: ( + key: CheckpointTurnDiffKey, + ) => Effect.Effect, ProjectionRepositoryError>; + readonly put: (row: CheckpointTurnDiff) => Effect.Effect; + /** + * Drops every diff that reads the checkpoint at `turnCount` or later. Called + * when that checkpoint is recaptured, restored past, or deleted. + */ + readonly invalidateFromTurn: (input: { + readonly threadId: ThreadId; + readonly turnCount: number; + }) => Effect.Effect; + readonly removeThread: (threadId: ThreadId) => Effect.Effect; +} + +export class CheckpointTurnDiffStore extends Context.Service< + CheckpointTurnDiffStore, + CheckpointTurnDiffStoreShape +>()("t3/persistence/Services/HubThreadMachineState/CheckpointTurnDiffStore") {} + +export const ThreadVcsStatus = Schema.Struct({ + threadId: ThreadId, + local: VcsStatusLocalResult, + remote: Schema.NullOr(VcsStatusRemoteResult), + updatedAt: IsoDateTime, +}); +export type ThreadVcsStatus = typeof ThreadVcsStatus.Type; + +export interface ThreadVcsStatusStoreShape { + readonly list: () => Effect.Effect, ProjectionRepositoryError>; + readonly put: (row: ThreadVcsStatus) => Effect.Effect; + readonly remove: (threadId: ThreadId) => Effect.Effect; +} + +export class ThreadVcsStatusStore extends Context.Service< + ThreadVcsStatusStore, + ThreadVcsStatusStoreShape +>()("t3/persistence/Services/HubThreadMachineState/ThreadVcsStatusStore") {} From 7fa9472e01b2931b39765042195e49815959b306 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 22:55:39 +0000 Subject: [PATCH 12/58] feat(server): add narrow hub-mode hooks to shared server code Hub mode changes shared code only through Context.Reference hooks whose defaults keep standalone behavior: - DeterministicIngestionCommandIds: ingestion derives command ids from the runtime event id, so runner events replayed after a hub restart become receipt replays instead of duplicates. - HubThreadCheckouts: client commands get virtual project roots and per-thread checkout paths, bootstrap prepares the checkout on the thread's machine instead of a local worktree, and turn start ensures the checkout instead of stat'ing a path. - ProviderAdapterCapabilities.sessionsOutliveServer: stopping a hub leaves sessions hosted on other machines running and their bindings live. Workspace asset URLs fail with a typed error on a hub. Hub configuration gains T3CODE_HUB_PUBLIC_URL and T3CODE_HUB_CHECKOUT_ROOT, and a hub without T3CODE_HUB_DATABASE_URL persists to SQLite. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/assets/AssetAccess.ts | 14 ++++ apps/server/src/cli/config.ts | 37 +++++++--- apps/server/src/config.ts | 27 +++++-- .../src/orchestration/CommandDispatcher.ts | 28 +++++++- .../Layers/ProviderCommandReactor.ts | 6 ++ .../Layers/ProviderRuntimeIngestion.ts | 29 +++++++- apps/server/src/orchestration/Normalizer.ts | 7 +- .../src/provider/Layers/ProviderService.ts | 17 ++++- .../src/provider/Services/ProviderAdapter.ts | 5 ++ apps/server/src/serverModeHooks.ts | 71 +++++++++++++++++++ 10 files changed, 218 insertions(+), 23 deletions(-) create mode 100644 apps/server/src/serverModeHooks.ts diff --git a/apps/server/src/assets/AssetAccess.ts b/apps/server/src/assets/AssetAccess.ts index 05801acca88f..6b7a9f0d4ab5 100644 --- a/apps/server/src/assets/AssetAccess.ts +++ b/apps/server/src/assets/AssetAccess.ts @@ -20,6 +20,7 @@ import { WORKSPACE_BROWSER_PREVIEW_EXTENSIONS, WORKSPACE_IMAGE_PREVIEW_EXTENSIONS, } from "@t3tools/shared/filePreview"; +import { HubModeUnsupportedError } from "@t3tools/contracts/runner"; import { PROJECT_FAVICON_FALLBACK_MARKER } from "@t3tools/shared/projectFavicon"; import * as Clock from "effect/Clock"; import * as Crypto from "effect/Crypto"; @@ -216,6 +217,19 @@ export const issueAssetUrl = Effect.fn("AssetAccess.issueAssetUrl")(function* (i const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const workspacePaths = yield* WorkspacePaths.WorkspacePaths; + // A hub has attachments but no checkout; workspace files live on thread machines. + if ( + input.resource._tag !== "attachment" && + ServerConfig.serverModeOf(yield* ServerConfig.ServerConfig) === "hub" + ) { + return yield* new AssetWorkspaceResolutionError({ + resource: input.resource, + cause: new HubModeUnsupportedError({ + operation: "assets.createUrl", + detail: "Workspace files and favicons are served by thread machines, not the hub.", + }), + }); + } let expiresAt = (yield* Clock.currentTimeMillis) + ASSET_TOKEN_TTL_MS; let claims: AssetClaims; let fileName: string; diff --git a/apps/server/src/cli/config.ts b/apps/server/src/cli/config.ts index d34383d464a1..95f28c549b3c 100644 --- a/apps/server/src/cli/config.ts +++ b/apps/server/src/cli/config.ts @@ -159,6 +159,14 @@ const EnvServerConfig = Config.all({ Config.option, Config.map(Option.getOrUndefined), ), + hubPublicUrl: Config.string("T3CODE_HUB_PUBLIC_URL").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), + hubCheckoutRoot: Config.string("T3CODE_HUB_CHECKOUT_ROOT").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), runnerUrl: Config.string("T3CODE_RUNNER_URL").pipe( Config.option, Config.map(Option.getOrUndefined), @@ -268,29 +276,38 @@ const resolveHubServerConfig = (env: { readonly hubSecretKey?: string | undefined; readonly hubMachinesUrl?: string | undefined; readonly hubMachinesToken?: string | undefined; + readonly hubPublicUrl?: string | undefined; + readonly hubCheckoutRoot?: string | undefined; readonly runnerUrl?: string | undefined; }) => Effect.gen(function* () { - const missing = [ - ["T3CODE_HUB_DATABASE_URL", env.hubDatabaseUrl], - ["T3CODE_HUB_TENANT_ID", env.hubTenantId], - ["T3CODE_HUB_SECRET_KEY", env.hubSecretKey], - ] - .filter(([, value]) => !value) - .map(([name]) => name); + // Without a database URL the hub persists to its SQLite state database. + const missing = env.hubDatabaseUrl + ? [ + ["T3CODE_HUB_TENANT_ID", env.hubTenantId], + ["T3CODE_HUB_SECRET_KEY", env.hubSecretKey], + ] + .filter(([, value]) => !value) + .map(([name]) => name) + : []; if (!env.runnerUrl && (!env.hubMachinesUrl || !env.hubMachinesToken)) { missing.push("T3CODE_HUB_MACHINES_URL and T3CODE_HUB_MACHINES_TOKEN"); } if (missing.length > 0) { return yield* Effect.die(new Error(`Hub mode requires ${missing.join(", ")}.`)); } + if (env.hubCheckoutRoot && !env.hubCheckoutRoot.startsWith("/")) { + return yield* Effect.die(new Error("T3CODE_HUB_CHECKOUT_ROOT must be an absolute path.")); + } return { - databaseUrl: env.hubDatabaseUrl!, + databaseUrl: env.hubDatabaseUrl, databaseAdminUrl: env.hubDatabaseAdminUrl, - tenantId: env.hubTenantId!, - secretKey: env.hubSecretKey!, + tenantId: env.hubTenantId, + secretKey: env.hubSecretKey, machinesUrl: env.hubMachinesUrl, machinesToken: env.hubMachinesToken, + publicUrl: env.hubPublicUrl, + checkoutRoot: env.hubCheckoutRoot, } satisfies ServerConfig.HubServerConfig; }); diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index fc8a09c0b07a..d46f09b60194 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -66,17 +66,32 @@ export const serverModeOf = (config: { }): ServerMode => config.serverMode ?? "standalone"; export interface HubServerConfig { - /** Postgres URL for the hub schema, using a role without BYPASSRLS. */ - readonly databaseUrl: string; + /** + * Postgres URL for the hub schema, using a role without BYPASSRLS. Without + * it a hub persists to its SQLite state database (tests and development). + */ + readonly databaseUrl?: string | undefined; /** Optional role used only to apply hub migrations. */ readonly databaseAdminUrl?: string | undefined; - /** The Aldo user whose rows this process owns. */ - readonly tenantId: string; - /** Base64 32-byte key that encrypts per-user secrets at rest. */ - readonly secretKey: string; + /** The Aldo user whose rows this process owns; required with `databaseUrl`. */ + readonly tenantId?: string | undefined; + /** Base64 32-byte key that encrypts per-user secrets at rest; required with `databaseUrl`. */ + readonly secretKey?: string | undefined; /** Machine directory base URL; absent only with a development `runnerUrl`. */ readonly machinesUrl?: string | undefined; readonly machinesToken?: string | undefined; + /** + * Public base URL at which thread machines reach this hub (its `/mcp` + * endpoint). Without it, runners are given the hub's local MCP endpoint, + * which only a runner on the same host can reach. + */ + readonly publicUrl?: string | undefined; + /** + * Root of thread checkouts on thread machines (`/workspace/t` by default). + * Hub and machines must agree on it; only tests and local development + * change it. + */ + readonly checkoutRoot?: string | undefined; } export class ServerConfig extends Context.Service< diff --git a/apps/server/src/orchestration/CommandDispatcher.ts b/apps/server/src/orchestration/CommandDispatcher.ts index ff04be3f3320..a554e8e25942 100644 --- a/apps/server/src/orchestration/CommandDispatcher.ts +++ b/apps/server/src/orchestration/CommandDispatcher.ts @@ -17,6 +17,7 @@ import { OrchestrationCommandReceiptRepository } from "../persistence/Services/O import * as GitWorkflowService from "../git/GitWorkflowService.ts"; import * as ProjectSetupScriptRunner from "../project/ProjectSetupScriptRunner.ts"; import * as ServerRuntimeStartup from "../serverRuntimeStartup.ts"; +import { HubThreadCheckouts } from "../serverModeHooks.ts"; import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; import * as OrchestrationEngine from "./Services/OrchestrationEngine.ts"; import { ThreadDeletionReactor } from "./Services/ThreadDeletionReactor.ts"; @@ -71,6 +72,8 @@ export const makeOrchestrationCommandDispatcher = Effect.gen(function* () { const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup; const projectSetupScriptRunner = yield* ProjectSetupScriptRunner.ProjectSetupScriptRunner; const threadDeletionReactor = yield* ThreadDeletionReactor; + // Hub mode: each thread checks out on its own machine instead of a local worktree. + const hubThreadCheckouts = yield* HubThreadCheckouts; const toDispatchCommandError = (cause: unknown, fallbackMessage: string) => isOrchestrationDispatchCommandError(cause) @@ -327,7 +330,30 @@ export const makeOrchestrationCommandDispatcher = Effect.gen(function* () { createdThread = createResult.replayed !== true; } - if (bootstrap?.prepareWorktree) { + if (hubThreadCheckouts !== null) { + const branch = + bootstrap?.prepareWorktree?.branch ?? bootstrap?.createThread?.branch ?? null; + const prepared = yield* hubThreadCheckouts.bootstrap({ + threadId: command.threadId, + projectId: targetProjectId, + branch, + baseRef: bootstrap?.prepareWorktree?.baseBranch ?? null, + }); + if (prepared.worktreePath !== targetWorktreePath || prepared.branch !== branch) { + yield* orchestrationEngine.dispatch( + { + type: "thread.meta.update", + commandId: yield* serverCommandId("bootstrap-thread-meta-update"), + threadId: command.threadId, + branch: prepared.branch, + worktreePath: prepared.worktreePath, + }, + options, + ); + } + targetWorktreePath = prepared.worktreePath; + yield* refreshGitStatus(targetWorktreePath); + } else if (bootstrap?.prepareWorktree) { let worktreeBaseRef = bootstrap.prepareWorktree.baseBranch; const startFromOrigin = bootstrap.prepareWorktree.startFromOrigin === true && diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index 1bae49994c34..18d8fb1a35c7 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -52,6 +52,7 @@ import { } from "../../serverSettings.ts"; import { VcsStatusBroadcaster } from "../../vcs/VcsStatusBroadcaster.ts"; import { GitWorkflowService } from "../../git/GitWorkflowService.ts"; +import { HubThreadCheckouts } from "../../serverModeHooks.ts"; const isProviderAdapterRequestError = Schema.is(ProviderAdapterRequestError); const isProviderDriverKind = Schema.is(ProviderDriverKind); @@ -320,6 +321,8 @@ const make = Effect.gen(function* () { const vcsStatusBroadcaster = yield* VcsStatusBroadcaster; const textGeneration = yield* TextGeneration; const serverSettingsService = yield* ServerSettingsService; + // Hub mode: checkouts live on thread machines; never stat or run git locally. + const hubThreadCheckouts = yield* HubThreadCheckouts; const serverCommandId = (tag: string) => crypto.randomUUIDv4.pipe(Effect.map((uuid) => CommandId.make(`server:${tag}:${uuid}`))); const serverEventId = () => crypto.randomUUIDv4.pipe(Effect.map(EventId.make)); @@ -452,6 +455,9 @@ const make = Effect.gen(function* () { readonly branch: string | null; readonly worktreePath: string | null; }) { + if (hubThreadCheckouts !== null) { + return yield* hubThreadCheckouts.ensureForTurn(thread); + } const { worktreePath, branch } = thread; if (!worktreePath || !branch) { return; diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index 424a2f55f2b9..a67d175b535e 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -33,6 +33,7 @@ import { ProviderService } from "../../provider/Services/ProviderService.ts"; import { ProjectionTurnRepository } from "../../persistence/Services/ProjectionTurns.ts"; import { ProjectionTurnRepositoryLive } from "../../persistence/Layers/ProjectionTurns.ts"; import { CheckoutGitProbe } from "../../git/CheckoutGitProbe.ts"; +import { DeterministicIngestionCommandIds } from "../../serverModeHooks.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; import { ThreadBackgroundLivenessService } from "../ThreadBackgroundLiveness.ts"; import { ThreadPlanProgressService } from "../ThreadPlanProgress.ts"; @@ -888,6 +889,9 @@ export function runtimeEventToActivities( return []; } +/** Events whose command ordinals are remembered; far beyond any replay window. */ +const COMMAND_ORDINAL_CAPACITY = 20_000; + const make = Effect.gen(function* () { const threadBackgroundLiveness = yield* ThreadBackgroundLivenessService; const threadPlanProgress = yield* ThreadPlanProgressService; @@ -898,10 +902,29 @@ const make = Effect.gen(function* () { const projectionTurnRepository = yield* ProjectionTurnRepository; const serverSettingsService = yield* ServerSettingsService; const isGitRepository = yield* CheckoutGitProbe; + const deterministicCommandIds = yield* DeterministicIngestionCommandIds; + // Hub mode: `provider:::`, where n counts commands per + // event and tag, so a replayed runner event reproduces the same ids. + const commandOrdinals = new Map(); + const nextCommandOrdinal = (key: string) => { + const ordinal = commandOrdinals.get(key) ?? 0; + commandOrdinals.delete(key); + commandOrdinals.set(key, ordinal + 1); + if (commandOrdinals.size > COMMAND_ORDINAL_CAPACITY) { + commandOrdinals.delete(commandOrdinals.keys().next().value!); + } + return ordinal; + }; const providerCommandId = (event: ProviderRuntimeEvent, tag: string) => - crypto.randomUUIDv4.pipe( - Effect.map((uuid) => CommandId.make(`provider:${event.eventId}:${tag}:${uuid}`)), - ); + deterministicCommandIds + ? Effect.sync(() => + CommandId.make( + `provider:${event.eventId}:${tag}:${nextCommandOrdinal(`${event.eventId}:${tag}`)}`, + ), + ) + : crypto.randomUUIDv4.pipe( + Effect.map((uuid) => CommandId.make(`provider:${event.eventId}:${tag}:${uuid}`)), + ); const turnMessageIdsByTurnKey = yield* Cache.make>({ capacity: TURN_MESSAGE_IDS_BY_TURN_CACHE_CAPACITY, diff --git a/apps/server/src/orchestration/Normalizer.ts b/apps/server/src/orchestration/Normalizer.ts index 1226a6cd25d5..0bbada5e7e66 100644 --- a/apps/server/src/orchestration/Normalizer.ts +++ b/apps/server/src/orchestration/Normalizer.ts @@ -19,6 +19,7 @@ import { } from "../attachmentStore.ts"; import { ServerConfig } from "../config.ts"; import { parseBase64DataUrl } from "../imageMime.ts"; +import { HubThreadCheckouts } from "../serverModeHooks.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; export const canonicalizeClientCommandTimestamps = ( @@ -75,7 +76,11 @@ const removeClaimedAttachmentPaths = Effect.fn("Normalizer.removeClaimedAttachme export const normalizeDispatchCommand = (command: ClientOrchestrationCommand) => Effect.gen(function* () { const receivedAt = DateTime.formatIso(yield* DateTime.now); - const canonicalCommand = canonicalizeClientCommandTimestamps(command, receivedAt); + const hubThreadCheckouts = yield* HubThreadCheckouts; + const canonicalCommand = canonicalizeClientCommandTimestamps( + hubThreadCheckouts ? hubThreadCheckouts.rewriteClientCommand(command) : command, + receivedAt, + ); const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const serverConfig = yield* ServerConfig; diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index ff15b5efbfa9..44d64a3968aa 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -1263,7 +1263,16 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( const runStopAll = Effect.fn("runStopAll")(function* () { const threadIds = yield* directory.listThreadIds(); - const currentAdapters = yield* getAdapterEntries; + const allAdapters = yield* getAdapterEntries; + // Sessions hosted on other machines keep running; their bindings stay live. + const remoteInstanceIds = new Set( + allAdapters + .filter(([, adapter]) => adapter.capabilities.sessionsOutliveServer === true) + .map(([instanceId]) => instanceId), + ); + const currentAdapters = allAdapters.filter( + ([instanceId]) => !remoteInstanceIds.has(instanceId), + ); const activeSessions = yield* Effect.forEach(currentAdapters, ([instanceId, adapter]) => adapter.listSessions().pipe( Effect.map((sessions) => @@ -1285,7 +1294,11 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( yield* Effect.forEach(currentAdapters, ([, adapter]) => adapter.stopAll()).pipe(Effect.asVoid); yield* McpSessionRegistry.revokeAllActiveMcpCredentials(); McpProviderSession.clearAllMcpProviderSessions(); - const bindings = yield* directory.listBindings().pipe(Effect.orElseSucceed(() => [])); + const bindings = (yield* directory.listBindings().pipe(Effect.orElseSucceed(() => []))).filter( + (binding) => + binding.providerInstanceId === undefined || + !remoteInstanceIds.has(binding.providerInstanceId), + ); yield* Effect.forEach(bindings, (binding) => Effect.gen(function* () { const providerInstanceId = dieOnMissingBindingInstanceId( diff --git a/apps/server/src/provider/Services/ProviderAdapter.ts b/apps/server/src/provider/Services/ProviderAdapter.ts index 634745832b37..2f99b9083d48 100644 --- a/apps/server/src/provider/Services/ProviderAdapter.ts +++ b/apps/server/src/provider/Services/ProviderAdapter.ts @@ -32,6 +32,11 @@ export interface ProviderAdapterCapabilities { * Declares whether changing the model on an existing session is supported. */ readonly sessionModelSwitch: ProviderSessionModelSwitchMode; + /** + * Sessions run on another machine and outlive this server process (hub + * mode). Shutting the server down must not stop them or mark them stopped. + */ + readonly sessionsOutliveServer?: boolean; } export interface ProviderThreadTurnSnapshot { diff --git a/apps/server/src/serverModeHooks.ts b/apps/server/src/serverModeHooks.ts new file mode 100644 index 000000000000..ed2580c7e975 --- /dev/null +++ b/apps/server/src/serverModeHooks.ts @@ -0,0 +1,71 @@ +/** + * Narrow hooks through which hub mode changes shared server code. + * + * Each is a `Context.Reference` whose default is standalone behavior, so + * standalone servers and existing tests never provide them. The hub layer set + * (`hub/HubLayers.ts`) provides hub implementations. See + * docs/internals/thread-machines.md. + * + * @module serverModeHooks + */ +import type { + ClientOrchestrationCommand, + OrchestrationDispatchCommandError, + ProjectId, + ThreadId, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import type * as Effect from "effect/Effect"; + +/** + * Provider runtime ingestion derives command ids from the runtime event id + * instead of a random suffix. In hub mode runners replay events after a hub + * restart, and deterministic ids turn every already-committed command into a + * receipt replay instead of a duplicate. + */ +export class DeterministicIngestionCommandIds extends Context.Reference( + "t3/serverModeHooks/DeterministicIngestionCommandIds", + { defaultValue: () => false }, +) {} + +export interface ThreadCheckoutBootstrapInput { + readonly threadId: ThreadId; + readonly projectId: ProjectId | undefined; + /** Branch the thread should work on; null keeps the checkout's branch. */ + readonly branch: string | null; + /** Base ref for a branch that does not exist yet. */ + readonly baseRef: string | null; +} + +export interface ThreadCheckoutBootstrapResult { + readonly worktreePath: string; + readonly branch: string | null; +} + +export interface HubThreadCheckoutsShape { + /** Rewrites a client command before normalization (virtual roots, checkout paths). */ + readonly rewriteClientCommand: ( + command: ClientOrchestrationCommand, + ) => ClientOrchestrationCommand; + /** + * Bootstrap of a thread in hub mode: ensure and wake its machine, then + * prepare the checkout (clone or fetch, branch). Records progress and + * failures as thread activities. + */ + readonly bootstrap: ( + input: ThreadCheckoutBootstrapInput, + ) => Effect.Effect; + /** Before a turn: make sure the thread's checkout exists on its (awake) machine. */ + readonly ensureForTurn: (thread: { + readonly id: ThreadId; + readonly projectId: ProjectId; + readonly branch: string | null; + readonly worktreePath: string | null; + }) => Effect.Effect; +} + +/** Hub thread-checkout provisioning; `null` keeps the standalone local worktree logic. */ +export class HubThreadCheckouts extends Context.Reference( + "t3/serverModeHooks/HubThreadCheckouts", + { defaultValue: () => null }, +) {} From ac011647b371d136809ca8361fd81c0f4979de25 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 22:55:49 +0000 Subject: [PATCH 13/58] feat(server): route hub services to per-thread runners Productionize the prototype's hub and runner split. Runner (t3 runner, one thread per machine): serves RunnerRpcGroup with every call checked against its thread and checkout, commits provider events to a SQLite outbox (WAL, synchronous=FULL) before publishing, prepares the checkout idempotently (clone or init, branch from origin or base ref), and hosts the real providers, git, checkpoints, review, workspace files and terminals. Attachments shipped with a turn are materialized locally and the prompt paths rewritten; the hub's MCP credential is installed per session. Hub: a MachineDirectory (HTTP contract, static development runner, test fake) resolves each thread's runner; RunnerConnectionPool connects per thread on demand, waking only when the caller asks, negotiates the protocol, reconnects busy threads with backoff, and closes idle connections with an idle notification. RunnerEventDelivery dedups by (thread, sequence), persists cursors only at turn-safe points after ingestion drains, and reconciles sessions lost to a runner restart or a failed machine by settling their turns. The remote provider driver answers hasSession/listSessions from a session registry that never wakes machines. Checkpoint diffs, git status and PR state are served from hub caches; terminals, workspace files, git actions and review route to the runner; unsupported operations fail with typed errors. server.ts selects the standalone or hub implementation of each checkout-bound layer group from one ServerModeLayers record; the standalone composition is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/cli/runner.ts | 17 +- apps/server/src/hub/HubLayers.ts | 201 +++++ apps/server/src/hub/HubRunnerLifecycle.ts | 146 ++++ apps/server/src/hub/HubTerminals.ts | 219 ++++++ apps/server/src/hub/HubThreadCheckouts.ts | 266 +++++++ apps/server/src/hub/HubVcs.ts | 511 ++++++++++++ apps/server/src/hub/HubWorkspace.ts | 360 +++++++++ apps/server/src/hub/MachineDirectory.ts | 301 +++++++ apps/server/src/hub/RemoteProviderDriver.ts | 535 +++++++++++++ apps/server/src/hub/RemoteSessionRegistry.ts | 194 +++++ apps/server/src/hub/RunnerConnectionPool.ts | 717 +++++++++++++++++ apps/server/src/hub/RunnerEventDelivery.ts | 404 ++++++++++ apps/server/src/hub/hubRouting.ts | 122 +++ .../ProviderInstanceRegistryHydration.ts | 15 +- apps/server/src/runner/RunnerCheckout.ts | 173 +++++ apps/server/src/runner/RunnerHandlers.ts | 734 ++++++++++++++++++ apps/server/src/runner/RunnerOutbox.ts | 446 ++++++----- .../runner/RunnerProjectionSnapshotQuery.ts | 38 + apps/server/src/runner/RunnerProtocol.ts | 317 -------- apps/server/src/runner/RunnerServer.ts | 390 ++++------ apps/server/src/runner/hub/HubLayers.ts | 225 ------ .../src/runner/hub/RemoteProviderDriver.ts | 314 -------- apps/server/src/runner/hub/RunnerClient.ts | 313 -------- apps/server/src/runner/hub/RunnerDelivery.ts | 50 -- apps/server/src/runner/remoteErrors.ts | 67 ++ apps/server/src/server.ts | 156 ++-- 26 files changed, 5506 insertions(+), 1725 deletions(-) create mode 100644 apps/server/src/hub/HubLayers.ts create mode 100644 apps/server/src/hub/HubRunnerLifecycle.ts create mode 100644 apps/server/src/hub/HubTerminals.ts create mode 100644 apps/server/src/hub/HubThreadCheckouts.ts create mode 100644 apps/server/src/hub/HubVcs.ts create mode 100644 apps/server/src/hub/HubWorkspace.ts create mode 100644 apps/server/src/hub/MachineDirectory.ts create mode 100644 apps/server/src/hub/RemoteProviderDriver.ts create mode 100644 apps/server/src/hub/RemoteSessionRegistry.ts create mode 100644 apps/server/src/hub/RunnerConnectionPool.ts create mode 100644 apps/server/src/hub/RunnerEventDelivery.ts create mode 100644 apps/server/src/hub/hubRouting.ts create mode 100644 apps/server/src/runner/RunnerCheckout.ts create mode 100644 apps/server/src/runner/RunnerHandlers.ts create mode 100644 apps/server/src/runner/RunnerProjectionSnapshotQuery.ts delete mode 100644 apps/server/src/runner/RunnerProtocol.ts delete mode 100644 apps/server/src/runner/hub/HubLayers.ts delete mode 100644 apps/server/src/runner/hub/RemoteProviderDriver.ts delete mode 100644 apps/server/src/runner/hub/RunnerClient.ts delete mode 100644 apps/server/src/runner/hub/RunnerDelivery.ts create mode 100644 apps/server/src/runner/remoteErrors.ts diff --git a/apps/server/src/cli/runner.ts b/apps/server/src/cli/runner.ts index db87ac3f2703..223957fb16b2 100644 --- a/apps/server/src/cli/runner.ts +++ b/apps/server/src/cli/runner.ts @@ -5,9 +5,15 @@ import { ServerConfig } from "../config.ts"; import { runRunner } from "../runner/RunnerServer.ts"; import { resolveServerConfig, sharedServerCommandFlags } from "./config.ts"; -/** Prototype: serve the runner protocol for this machine's checkout. */ +/** + * `t3 runner`: serves the runner protocol for one thread's checkout on a + * thread machine (`T3CODE_RUNNER_THREAD_ID`, `T3CODE_RUNNER_CHECKOUT`, + * `T3CODE_RUNNER_TOKEN`). See docs/internals/thread-machines.md. + */ export const runnerCommand = Command.make("runner", { ...sharedServerCommandFlags }).pipe( - Command.withDescription("Run a T3 runner: provider drivers and checkpoints for a remote hub."), + Command.withDescription( + "Run a T3 runner: one thread's providers, git, terminals and files for a hub.", + ), Command.withHandler((flags) => Effect.gen(function* () { const logLevel = yield* GlobalFlag.LogLevel; @@ -16,7 +22,12 @@ export const runnerCommand = Command.make("runner", { ...sharedServerCommandFlag forceAutoBootstrapProjectFromCwd: false, serverMode: "runner", }); - return yield* runRunner.pipe(Effect.provideService(ServerConfig, config)); + // Services that anchor on the server cwd (review path checks, provider + // fallbacks) anchor on the thread's checkout. + const runnerConfig = config.runnerCheckout + ? { ...config, cwd: config.runnerCheckout } + : config; + return yield* runRunner.pipe(Effect.provideService(ServerConfig, runnerConfig)); }), ), ); diff --git a/apps/server/src/hub/HubLayers.ts b/apps/server/src/hub/HubLayers.ts new file mode 100644 index 000000000000..a0e558be56d2 --- /dev/null +++ b/apps/server/src/hub/HubLayers.ts @@ -0,0 +1,201 @@ +/** + * HubLayers - the hub's replacements for checkout-bound services. + * + * `server.ts` composes the same runtime for every mode from a + * `ServerModeLayers` record. Standalone uses the local implementations; a + * hub (`T3CODE_SERVER_MODE=hub`) uses this set, in which every service that + * would read a checkout, run git, open a PTY or spawn a provider CLI routes + * to the thread's runner instead, or serves from a hub cache, or fails with + * a typed error. Hub-only infrastructure (machine directory, connection + * pool, event delivery, session registry, caches) is provided underneath. + * + * @module hub/HubLayers + */ +import { SourceControlRepositoryError, type SourceControlProviderKind } from "@t3tools/contracts"; +import { HubModeUnsupportedError } from "@t3tools/contracts/runner"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import type * as SqlClient from "effect/unstable/sql/SqlClient"; +import type { SqlError } from "effect/unstable/sql/SqlError"; + +import { ServerConfig } from "../config.ts"; +import { makeHubPgClientLayer } from "../persistence/Layers/Postgres.ts"; +import type { PersistenceSqlError } from "../persistence/Errors.ts"; +import { HubThreadMachineStateSqliteLive } from "../persistence/Layers/HubThreadMachineState.ts"; +import { HUB_MIGRATION_050 } from "../persistence/Postgres/migrations/050_HubThreadMachineState.ts"; +import { HubThreadMachineStatePostgresLive } from "../persistence/Postgres/HubThreadMachineState.ts"; +import { hubTenantLayer } from "../persistence/Postgres/HubTenant.ts"; +import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; +import type { + CheckpointTurnDiffStore, + RunnerCursorStore, + ThreadVcsStatusStore, +} from "../persistence/Services/HubThreadMachineState.ts"; +import { RepositoryIdentityResolver } from "../project/RepositoryIdentityResolver.ts"; +import { resolveBuiltInDrivers } from "../provider/builtInDrivers.ts"; +import { makeProviderInstanceRegistryHydration } from "../provider/Layers/ProviderInstanceRegistryHydration.ts"; +import { DeterministicIngestionCommandIds } from "../serverModeHooks.ts"; +import { SourceControlRepositoryService } from "../sourceControl/SourceControlRepositoryService.ts"; +import * as HubRunnerLifecycle from "./HubRunnerLifecycle.ts"; +import * as HubThreadCheckouts from "./HubThreadCheckouts.ts"; +import { hubTerminalManagerLayer } from "./HubTerminals.ts"; +import { + hubGitManagerLayer, + hubGitWorkflowServiceLayer, + hubReviewServiceLayer, + hubVcsProvisioningServiceLayer, + hubVcsStatusBroadcasterLayer, + hubVcsStatusCacheLayer, +} from "./HubVcs.ts"; +import { + hubCheckoutGitProbeLayer, + hubCheckpointStoreLayer, + hubWorkspaceEntriesLayer, + hubWorkspaceFileSystemLayer, + hubWorkspacePathsLayer, +} from "./HubWorkspace.ts"; +import * as MachineDirectory from "./MachineDirectory.ts"; +import { makeRemoteProviderDriver } from "./RemoteProviderDriver.ts"; +import * as RemoteSessionRegistry from "./RemoteSessionRegistry.ts"; +import * as RunnerConnectionPool from "./RunnerConnectionPool.ts"; +import * as RunnerEventDelivery from "./RunnerEventDelivery.ts"; + +/** + * Hub thread-machine state: Postgres iff `T3CODE_HUB_DATABASE_URL` is set, + * otherwise the server's SQLite database (tests and local development). + * + * TODO(tm-hub integration): use the hub persistence composition's Postgres + * client and tenant instead of this dedicated pool, and run migration 050 + * from the hub migration framework. + */ +type HubStateStores = RunnerCursorStore | CheckpointTurnDiffStore | ThreadVcsStatusStore; + +const hubStateStores = (sqlitePersistence: Layer.Layer) => + Layer.unwrap( + Effect.gen(function* () { + const config = yield* ServerConfig; + const hub = config.hub; + const sqlite: Layer.Layer = + HubThreadMachineStateSqliteLive.pipe(Layer.provide(sqlitePersistence)); + if (!hub?.databaseUrl || !hub.tenantId) return sqlite; + const postgres: Layer.Layer = + HubThreadMachineStatePostgresLive.pipe( + Layer.provide(Layer.effectDiscard(HUB_MIGRATION_050)), + Layer.provide(hubTenantLayer(hub.tenantId)), + Layer.provideMerge( + makeHubPgClientLayer({ + url: hub.databaseUrl, + maxConnections: 2, + applicationName: "t3-hub-thread-machines", + }), + ), + ); + return postgres; + }), + ); + +/** + * Hub-only services underneath the whole runtime: stores, machine directory, + * connection pool, session registry, event delivery and git status cache. + */ +export const makeHubInfrastructureLayer = (parts: { + readonly sqlitePersistence: Layer.Layer; +}) => + Layer.mergeAll(RunnerEventDelivery.layer, hubVcsStatusCacheLayer).pipe( + Layer.provideMerge( + RemoteSessionRegistry.layer.pipe( + Layer.provide(ProviderSessionRuntime.layer.pipe(Layer.provide(parts.sqlitePersistence))), + ), + ), + Layer.provideMerge(RunnerConnectionPool.layer), + Layer.provideMerge(MachineDirectory.layer), + Layer.provideMerge(hubStateStores(parts.sqlitePersistence)), + ); + +/** Every built-in driver kind, each forwarding to the thread's runner. */ +export const hubProviderInstanceRegistryLayer = Layer.unwrap( + Effect.gen(function* () { + const config = yield* ServerConfig; + return makeProviderInstanceRegistryHydration( + resolveBuiltInDrivers({ museCodeEnabled: config.museCodeEnabled }).map( + makeRemoteProviderDriver, + ), + ); + }), +); + +/** + * A hub never runs git locally, so identity comes from what the hub persists + * about the project. TODO(tm-hub integration): the hub persistence branch + * stores repository identity on projects and provides this resolver. + */ +export const hubRepositoryIdentityResolverLayer = Layer.succeed( + RepositoryIdentityResolver, + RepositoryIdentityResolver.of({ resolve: () => Effect.succeed(null) }), +); + +const unsupportedRepositoryOperation = ( + provider: SourceControlProviderKind, + operation: string, + detail: string, +) => + new SourceControlRepositoryError({ + provider, + operation, + detail, + cause: new HubModeUnsupportedError({ operation, detail }), + }); + +/** + * Repository lookup and listing use provider APIs and work on a hub; cloning + * and publishing need a checkout and have no hub implementation yet. + */ +export const hubSourceControlRepositoryServiceLayer = Layer.effect( + SourceControlRepositoryService, + Effect.gen(function* () { + const local = yield* SourceControlRepositoryService; + return SourceControlRepositoryService.of({ + listRepositories: local.listRepositories, + lookupRepository: local.lookupRepository, + cloneRepository: (input) => + Effect.fail( + unsupportedRepositoryOperation( + input.provider ?? "unknown", + "cloneRepository", + "Thread machines clone a project's repository themselves.", + ), + ), + publishRepository: (input) => + Effect.fail( + unsupportedRepositoryOperation( + input.provider ?? "unknown", + "publishRepository", + "Publishing a checkout is not routed to thread machines yet.", + ), + ), + }); + }), +); + +/** Hooks consumed by the orchestration reactors and command dispatch. */ +export const hubReactorHooksLayer = Layer.mergeAll( + Layer.succeed(DeterministicIngestionCommandIds, true), + hubCheckoutGitProbeLayer, + HubThreadCheckouts.layer, +); + +/** Background work that needs the reactors: delivery acks, resume, machine release. */ +export const hubRuntimeBackgroundLayer = HubRunnerLifecycle.layer; + +export { + hubCheckpointStoreLayer, + hubGitManagerLayer, + hubGitWorkflowServiceLayer, + hubReviewServiceLayer, + hubTerminalManagerLayer, + hubVcsProvisioningServiceLayer, + hubVcsStatusBroadcasterLayer, + hubWorkspaceEntriesLayer, + hubWorkspaceFileSystemLayer, + hubWorkspacePathsLayer, +}; diff --git a/apps/server/src/hub/HubRunnerLifecycle.ts b/apps/server/src/hub/HubRunnerLifecycle.ts new file mode 100644 index 000000000000..fd866c54c8e8 --- /dev/null +++ b/apps/server/src/hub/HubRunnerLifecycle.ts @@ -0,0 +1,146 @@ +/** + * HubRunnerLifecycle - hub-wide background work for thread machines. + * + * - Opens runner event delivery once the server is activated (ingestion + * subscribes at the same gate), then resumes delivery from machines that + * were running a turn when the hub stopped. Resuming never wakes a machine. + * - Flushes delivery cursors: every `ACK_INTERVAL`, safe points older than + * `ACK_SETTLE` are persisted after ingestion drains, then acknowledged to + * their runners. A final flush runs on shutdown. + * - Gives the connection pool the project and repository of each thread for + * machine directory requests. + * - Releases a thread's machine when the thread is archived or deleted, and + * drops its hub caches when it is deleted. + * + * @module hub/HubRunnerLifecycle + */ +import type { OrchestrationEvent, ThreadId } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; + +import { isBootstrapCleanupDeletion } from "../orchestration/Layers/ThreadDeletionReactor.ts"; +import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../orchestration/Services/ProjectionSnapshotQuery.ts"; +import { ProviderRuntimeIngestionService } from "../orchestration/Services/ProviderRuntimeIngestion.ts"; +import { + CheckpointTurnDiffStore, + RunnerCursorStore, +} from "../persistence/Services/HubThreadMachineState.ts"; +import { RepositoryIdentityResolver } from "../project/RepositoryIdentityResolver.ts"; +import { forkParked } from "../serverActivation.ts"; +import { HubVcsStatusCache } from "./HubVcs.ts"; +import { RemoteSessionRegistry } from "./RemoteSessionRegistry.ts"; +import { RunnerConnectionPool } from "./RunnerConnectionPool.ts"; +import { RunnerEventDelivery } from "./RunnerEventDelivery.ts"; + +/** Ingestion's stream consumers start at the same activation gate; let them subscribe first. */ +const DELIVERY_START_DELAY = "250 millis"; +const ACK_INTERVAL = "250 millis"; +/** A safe point is persisted once it is this old, so its events have reached ingestion's queue. */ +const ACK_SETTLE_MS = 250; +const SHUTDOWN_FLUSH_TIMEOUT = "5 seconds"; + +type ThreadLifecycleEvent = Extract< + OrchestrationEvent, + { type: "thread.deleted" | "thread.archived" } +>; + +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + const delivery = yield* RunnerEventDelivery; + const registry = yield* RemoteSessionRegistry; + const ingestion = yield* ProviderRuntimeIngestionService; + const engine = yield* OrchestrationEngineService; + const projections = yield* ProjectionSnapshotQuery; + const repositoryIdentity = yield* RepositoryIdentityResolver; + const cursors = yield* RunnerCursorStore; + const diffs = yield* CheckpointTurnDiffStore; + const vcsCache = yield* HubVcsStatusCache; + + yield* pool.setContextResolver((threadId) => + Effect.gen(function* () { + const thread = Option.getOrUndefined( + yield* projections + .getThreadShellById(threadId) + .pipe(Effect.orElseSucceed(() => Option.none())), + ); + if (!thread) return { projectId: null, repository: null, branch: null }; + const project = Option.getOrUndefined( + yield* projections + .getProjectShellById(thread.projectId) + .pipe(Effect.orElseSucceed(() => Option.none())), + ); + const identity = project ? yield* repositoryIdentity.resolve(project.workspaceRoot) : null; + return { + projectId: thread.projectId, + repository: identity ? { url: identity.locator.remoteUrl, ref: null } : null, + branch: thread.branch, + }; + }), + ); + + const flush = (minAgeMs: number) => delivery.flush({ minAgeMs, drain: ingestion.drain }); + + yield* forkParked( + Effect.gen(function* () { + yield* Effect.sleep(DELIVERY_START_DELAY); + yield* delivery.start; + for (const record of yield* registry.list) { + if (record.session.status !== "running" && record.session.activeTurnId === undefined) { + continue; + } + yield* pool + .use(record.threadId, { wake: false, operation: "resume-delivery" }, () => Effect.void) + .pipe( + Effect.catch((error) => + Effect.logInfo("thread machine not resumed for delivery", { + threadId: record.threadId, + reason: error.reason, + }), + ), + Effect.forkScoped, + ); + } + return yield* flush(ACK_SETTLE_MS).pipe(Effect.delay(ACK_INTERVAL), Effect.forever); + }), + ); + yield* Effect.addFinalizer(() => + flush(0).pipe(Effect.timeout(SHUTDOWN_FLUSH_TIMEOUT), Effect.ignore), + ); + + const releaseThread = (event: ThreadLifecycleEvent) => + Effect.gen(function* () { + const threadId: ThreadId = event.payload.threadId; + // A failed bootstrap may retry with the same thread id; its machine stays. + if (event.type === "thread.deleted" && isBootstrapCleanupDeletion(event)) return; + yield* pool.release(threadId); + if (event.type === "thread.deleted") { + yield* registry.remove(threadId); + yield* vcsCache.remove(threadId); + yield* cursors.remove(threadId).pipe(Effect.ignore); + yield* diffs.removeThread(threadId).pipe(Effect.ignore); + } + }).pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.failCause(cause) + : Effect.logWarning("thread machine release failed", { + threadId: event.payload.threadId, + cause: Cause.pretty(cause), + }), + ), + ); + + yield* forkParked( + Stream.runForEach(engine.streamDomainEvents, (event) => + event.type === "thread.deleted" || event.type === "thread.archived" + ? releaseThread(event) + : Effect.void, + ), + ); + }), +); diff --git a/apps/server/src/hub/HubTerminals.ts b/apps/server/src/hub/HubTerminals.ts new file mode 100644 index 000000000000..13a756b258cb --- /dev/null +++ b/apps/server/src/hub/HubTerminals.ts @@ -0,0 +1,219 @@ +/** + * Terminals in hub mode. Every terminal lives on its thread's machine. + * + * - `open`, `attach` and `restart` wake the machine (opening a terminal is an + * explicit request for a live shell). + * - `write`, `resize` and `clear` act on a running terminal and never wake a + * machine; on a sleeping one they fail with `TerminalNotRunningError`. + * `close` on a sleeping machine is a no-op. + * - Terminal events and metadata from every connected runner fan out to hub + * subscribers. Metadata of a machine that went to sleep is kept (paused + * machines keep their shells); a runner's snapshot on reconnect replaces its + * thread's entries. + * + * @module hub/HubTerminals + */ +import { + TerminalCwdStatError, + type TerminalError, + type TerminalEvent, + type TerminalMetadataStreamEvent, + TerminalNotRunningError, + type TerminalSummary, + type ThreadId, + ThreadId as ThreadIdSchema, +} from "@t3tools/contracts"; +import { TerminalError as TerminalErrorSchema } from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +import * as TerminalManager from "../terminal/Manager.ts"; +import { describeCause } from "./hubRouting.ts"; +import { RunnerConnectionPool, type RunnerRpcClient } from "./RunnerConnectionPool.ts"; + +const isTerminalError = Schema.is(TerminalErrorSchema); + +export const hubTerminalManagerLayer = Layer.effect( + TerminalManager.TerminalManager, + Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + const eventListeners = new Set<(event: TerminalEvent) => Effect.Effect>(); + const metadataListeners = new Set< + (event: TerminalMetadataStreamEvent) => Effect.Effect + >(); + const summaries = new Map>(); + + const emitMetadata = (event: TerminalMetadataStreamEvent) => + Effect.forEach([...metadataListeners], (listener) => listener(event), { discard: true }); + const emitEvent = (event: TerminalEvent) => + Effect.forEach([...eventListeners], (listener) => listener(event), { discard: true }); + + /** Applies one runner metadata event to the thread's entries and forwards the delta. */ + const applyMetadata = (threadId: ThreadId, event: TerminalMetadataStreamEvent) => + Effect.gen(function* () { + const entries = summaries.get(threadId) ?? new Map(); + summaries.set(threadId, entries); + switch (event.type) { + case "snapshot": { + const next = new Map( + event.terminals.map((terminal) => [terminal.terminalId, terminal]), + ); + for (const terminalId of entries.keys()) { + if (!next.has(terminalId)) { + yield* emitMetadata({ type: "remove", threadId, terminalId }); + } + } + summaries.set(threadId, next); + for (const terminal of next.values()) { + yield* emitMetadata({ type: "upsert", terminal }); + } + return; + } + case "upsert": + entries.set(event.terminal.terminalId, event.terminal); + return yield* emitMetadata(event); + case "remove": + entries.delete(event.terminalId); + return yield* emitMetadata(event); + } + }); + + yield* pool.onConnection((connection) => + Effect.gen(function* () { + const events = yield* connection.client["runner.terminal.events"]({}).pipe( + Stream.runForEach(emitEvent), + Effect.forkScoped, + ); + yield* connection.client["runner.terminal.metadata"]({}).pipe( + Stream.runForEach((event) => applyMetadata(connection.threadId, event)), + ); + yield* Fiber.join(events); + }).pipe( + Effect.catch((error) => + Effect.logDebug("runner terminal streams ended", { + threadId: connection.threadId, + detail: describeCause(error), + }), + ), + ), + ); + + const unavailableTerminal = ( + input: { + readonly threadId: string; + readonly terminalId?: string | undefined; + readonly cwd?: string | undefined; + }, + cause: unknown, + ): TerminalError => + isTerminalError(cause) + ? cause + : input.cwd !== undefined + ? new TerminalCwdStatError({ cwd: input.cwd, cause }) + : new TerminalNotRunningError({ + threadId: input.threadId, + terminalId: input.terminalId ?? "default", + }); + + const onRunner = ( + input: { + readonly threadId: string; + readonly terminalId?: string | undefined; + readonly cwd?: string | undefined; + }, + wake: boolean, + operation: string, + f: (client: RunnerRpcClient) => Effect.Effect, + ): Effect.Effect => + pool + .use( + ThreadIdSchema.make(input.threadId), + { wake, operation: `terminal.${operation}` }, + (c) => f(c.client), + ) + .pipe(Effect.mapError((error) => unavailableTerminal(input, error))); + + return TerminalManager.TerminalManager.of({ + open: (input) => + onRunner(input, true, "open", (client) => client["runner.terminal.open"](input)), + attachStream: (input, listener) => + Effect.gen(function* () { + // Resolves on the first event (the snapshot) or the first failure, so + // waking and attach failures surface to the caller. + const ready = yield* Deferred.make(); + const fiber = yield* pool + .stream( + ThreadIdSchema.make(input.threadId), + { wake: true, operation: "terminal.attach" }, + (connection) => connection.client["runner.terminal.attach"](input), + ) + .pipe( + Stream.runForEach((event) => + listener(event).pipe(Effect.andThen(Deferred.succeed(ready, undefined))), + ), + Effect.matchEffect({ + onFailure: (error) => + Deferred.fail(ready, unavailableTerminal(input, error)).pipe( + Effect.andThen( + Effect.logDebug("terminal attach stream ended", { + threadId: input.threadId, + terminalId: input.terminalId, + detail: describeCause(error), + }), + ), + ), + onSuccess: () => Deferred.succeed(ready, undefined), + }), + Effect.forkDetach, + ); + yield* Deferred.await(ready); + return () => { + fiber.interruptUnsafe(); + }; + }), + write: (input) => + onRunner(input, false, "write", (client) => client["runner.terminal.write"](input)), + resize: (input) => + onRunner(input, false, "resize", (client) => client["runner.terminal.resize"](input)), + clear: (input) => + onRunner(input, false, "clear", (client) => client["runner.terminal.clear"](input)), + restart: (input) => + onRunner(input, true, "restart", (client) => client["runner.terminal.restart"](input)), + close: (input) => + pool + .current(ThreadIdSchema.make(input.threadId)) + .pipe( + Effect.flatMap((connection) => + Option.isNone(connection) + ? Effect.void + : connection.value.client["runner.terminal.close"](input).pipe( + Effect.mapError((error) => unavailableTerminal(input, error)), + ), + ), + ), + subscribe: (listener) => + Effect.sync(() => { + eventListeners.add(listener); + return () => { + eventListeners.delete(listener); + }; + }), + subscribeMetadata: (listener) => + Effect.gen(function* () { + metadataListeners.add(listener); + yield* listener({ + type: "snapshot", + terminals: [...summaries.values()].flatMap((entries) => [...entries.values()]), + }); + return () => { + metadataListeners.delete(listener); + }; + }), + }); + }), +); diff --git a/apps/server/src/hub/HubThreadCheckouts.ts b/apps/server/src/hub/HubThreadCheckouts.ts new file mode 100644 index 000000000000..0263f7cfb309 --- /dev/null +++ b/apps/server/src/hub/HubThreadCheckouts.ts @@ -0,0 +1,266 @@ +/** + * Thread checkouts in hub mode: every thread works in its own checkout at + * `/` on its own machine. + * + * - Client commands are rewritten before normalization: projects get the + * virtual root `/workspace/p/` (nothing reads it) and threads the + * checkout path as `worktreePath`. + * - Bootstrap (a new thread's first turn, or a thread moved to its own + * branch) ensures and wakes the machine through the directory, passing the + * project's repository identity, then asks the runner to prepare the + * checkout (clone or fetch with the machine's git credentials, create or + * switch the branch). Progress and failures become thread activities: + * `thread-machine.starting`, `thread-machine.checkout.preparing`, + * `thread-machine.failed`. + * - Before every turn the checkout is prepared again; the runner call is + * idempotent, and skipped when this runner boot already prepared it. + * + * @module hub/HubThreadCheckouts + */ +import { + CommandId, + EventId, + OrchestrationDispatchCommandError, + type ProjectId, + type ThreadId, +} from "@t3tools/contracts"; +import { + projectVirtualRoot, + type ThreadMachineRepository, + type ThreadMachineStatus, +} from "@t3tools/contracts/runner"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; + +import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../orchestration/Services/ProjectionSnapshotQuery.ts"; +import { RepositoryIdentityResolver } from "../project/RepositoryIdentityResolver.ts"; +import { HubThreadCheckouts, type HubThreadCheckoutsShape } from "../serverModeHooks.ts"; +import { describeCause } from "./hubRouting.ts"; +import { RunnerConnectionPool, type ThreadMachineContext } from "./RunnerConnectionPool.ts"; + +export const THREAD_MACHINE_ACTIVITY = { + starting: "thread-machine.starting", + preparingCheckout: "thread-machine.checkout.preparing", + failed: "thread-machine.failed", +} as const; + +export const makeHubThreadCheckouts = Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + const engine = yield* OrchestrationEngineService; + const projections = yield* ProjectionSnapshotQuery; + const repositoryIdentity = yield* RepositoryIdentityResolver; + const crypto = yield* Crypto.Crypto; + /** Runner boot that last prepared each thread's checkout. */ + const preparedOnBoot = new Map(); + + const appendActivity = ( + threadId: ThreadId, + activity: { + readonly kind: string; + readonly summary: string; + readonly tone: "info" | "error"; + readonly payload: Record; + }, + ) => + Effect.gen(function* () { + const createdAt = DateTime.formatIso(yield* DateTime.now); + const uuid = yield* crypto.randomUUIDv4; + yield* engine.dispatch({ + type: "thread.activity.append", + commandId: CommandId.make(`server:thread-machine-activity:${uuid}`), + threadId, + activity: { + id: EventId.make(uuid), + tone: activity.tone, + kind: activity.kind, + summary: activity.summary, + payload: activity.payload, + turnId: null, + createdAt, + }, + createdAt, + }); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("failed to record thread machine activity", { + threadId, + kind: activity.kind, + cause: String(cause).slice(0, 300), + }), + ), + ); + + /** Project, repository and branch for the directory and the runner. */ + const machineContext = ( + threadId: ThreadId, + projectId: ProjectId | undefined, + branch: string | null, + baseRef: string | null, + ) => + Effect.gen(function* () { + const resolvedProjectId = + projectId ?? + Option.getOrUndefined( + yield* projections.getThreadShellById(threadId).pipe( + Effect.map(Option.map((thread) => thread.projectId)), + Effect.orElseSucceed(() => Option.none()), + ), + ); + const project = resolvedProjectId + ? Option.getOrUndefined( + yield* projections + .getProjectShellById(resolvedProjectId) + .pipe(Effect.orElseSucceed(() => Option.none())), + ) + : undefined; + const identity = project ? yield* repositoryIdentity.resolve(project.workspaceRoot) : null; + const repository: ThreadMachineRepository | null = identity + ? { url: identity.locator.remoteUrl, ref: baseRef } + : null; + return { + projectId: resolvedProjectId ?? null, + repository, + branch, + } satisfies ThreadMachineContext; + }); + + const prepare = (input: { + readonly threadId: ThreadId; + readonly projectId: ProjectId | undefined; + readonly branch: string | null; + readonly baseRef: string | null; + readonly announce: boolean; + }) => + Effect.gen(function* () { + const { threadId } = input; + const checkout = pool.checkoutFor(threadId); + const context = yield* machineContext(threadId, input.projectId, input.branch, input.baseRef); + let announcedStart = false; + const onWakeProgress = (status: ThreadMachineStatus) => + announcedStart + ? Effect.void + : Effect.suspend(() => { + announcedStart = true; + return appendActivity(threadId, { + kind: THREAD_MACHINE_ACTIVITY.starting, + summary: "Starting machine", + tone: "info", + payload: { state: status.state, detail: status.detail ?? null }, + }); + }); + return yield* pool.use( + threadId, + { wake: true, operation: "checkout.prepare", context, onWakeProgress }, + (connection) => + Effect.gen(function* () { + if (!input.announce && preparedOnBoot.get(threadId) === connection.hello.bootId) { + return null; + } + if (input.announce) { + yield* appendActivity(threadId, { + kind: THREAD_MACHINE_ACTIVITY.preparingCheckout, + summary: "Preparing checkout", + tone: "info", + payload: { + checkout, + branch: input.branch, + repository: context.repository?.url ?? null, + }, + }); + } + const result = yield* connection.client["runner.checkout.prepare"]({ + threadId, + checkout, + repository: context.repository, + branch: input.branch, + baseRef: input.baseRef, + }); + preparedOnBoot.set(threadId, connection.hello.bootId); + return result; + }), + ); + }).pipe( + Effect.tapError((error) => + appendActivity(input.threadId, { + kind: THREAD_MACHINE_ACTIVITY.failed, + summary: "Thread machine is unavailable", + tone: "error", + payload: { detail: describeCause(error) }, + }), + ), + ); + + const rewriteClientCommand: HubThreadCheckoutsShape["rewriteClientCommand"] = (command) => { + switch (command.type) { + case "project.create": + return { + ...command, + workspaceRoot: projectVirtualRoot(command.projectId), + createWorkspaceRootIfMissing: false, + }; + case "project.meta.update": { + // A hub project's root is virtual; there is no folder to move it to. + const { workspaceRoot: _workspaceRoot, ...rest } = command; + return rest; + } + case "thread.create": + return { ...command, worktreePath: pool.checkoutFor(command.threadId) }; + case "thread.turn.start": + return command.bootstrap?.createThread + ? { + ...command, + bootstrap: { + ...command.bootstrap, + createThread: { + ...command.bootstrap.createThread, + worktreePath: pool.checkoutFor(command.threadId), + }, + }, + } + : command; + default: + return command; + } + }; + + return { + rewriteClientCommand, + bootstrap: (input) => + prepare({ ...input, announce: true }).pipe( + Effect.map((result) => ({ + worktreePath: pool.checkoutFor(input.threadId), + branch: result?.branch ?? input.branch, + })), + Effect.mapError( + (error) => + new OrchestrationDispatchCommandError({ + message: `Thread machine could not prepare the checkout: ${describeCause(error)}`, + cause: error, + }), + ), + ), + ensureForTurn: (thread) => + prepare({ + threadId: thread.id, + projectId: thread.projectId, + branch: thread.branch, + baseRef: null, + announce: false, + }).pipe( + Effect.asVoid, + // The turn proceeds; its session start reports the real failure. + Effect.catch((error) => + Effect.logWarning("thread checkout was not prepared before the turn", { + threadId: thread.id, + detail: describeCause(error), + }), + ), + ), + } satisfies HubThreadCheckoutsShape; +}); + +export const layer = Layer.effect(HubThreadCheckouts, makeHubThreadCheckouts); diff --git a/apps/server/src/hub/HubVcs.ts b/apps/server/src/hub/HubVcs.ts new file mode 100644 index 000000000000..77bb3a214968 --- /dev/null +++ b/apps/server/src/hub/HubVcs.ts @@ -0,0 +1,511 @@ +/** + * Git, VCS status and review in hub mode. + * + * Status is served from a hub cache that runners keep current: while a + * thread's machine is connected, the hub subscribes to its runner's status + * stream and stores every update (memory plus `ThreadVcsStatusStore`), so + * sidebar rows, branch drift and auto-settlement read status without waking + * machines. Refreshes use the runner only when it is already connected. + * + * Git actions (pull, commit/push/PR stacked actions, refs, worktrees, init, + * PR thread preparation) run on the thread's runner and wake its machine. + * Read-only git and review calls never wake a machine; on a sleeping machine + * they fail with the service's error type caused by a `ThreadMachineUnavailableError` + * whose reason is `asleep`. + * + * @module hub/HubVcs + */ +import { + GitManagerError, + type GitManagerServiceError, + type GitRunStackedActionResult, + type ThreadId, + type VcsStatusLocalResult, + type VcsStatusRemoteResult, + type VcsStatusResult, + type VcsStatusStreamEvent, +} from "@t3tools/contracts"; +import { ThreadMachineUnavailableError } from "@t3tools/contracts/runner"; +import { mergeGitStatusParts } from "@t3tools/shared/git"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +import * as GitManager from "../git/GitManager.ts"; +import * as GitWorkflowService from "../git/GitWorkflowService.ts"; +import { ThreadVcsStatusStore } from "../persistence/Services/HubThreadMachineState.ts"; +import * as ReviewService from "../review/ReviewService.ts"; +import * as VcsProvisioningService from "../vcs/VcsProvisioningService.ts"; +import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; +import { + describeCause, + onCwdRunner, + streamOnCwdRunner, + threadOfCwd, + toGitCommandError, + toGitManagerServiceError, + toVcsError, +} from "./hubRouting.ts"; +import { RunnerConnectionPool } from "./RunnerConnectionPool.ts"; + +const isThreadMachineUnavailable: (error: unknown) => error is ThreadMachineUnavailableError = + Schema.is(ThreadMachineUnavailableError); + +interface CachedStatus { + readonly local: VcsStatusLocalResult; + readonly remote: VcsStatusRemoteResult | null; +} + +interface CachedStatusEvent { + readonly threadId: ThreadId; + readonly event: VcsStatusStreamEvent; +} + +export interface HubVcsStatusCacheShape { + readonly get: (threadId: ThreadId) => Effect.Effect>; + readonly apply: (threadId: ThreadId, event: VcsStatusStreamEvent) => Effect.Effect; + /** Cached status first (when known), then live updates for the thread. */ + readonly stream: (threadId: ThreadId) => Stream.Stream; + /** The PR of any thread whose checkout is on `branch`. */ + readonly pullRequestForBranch: ( + branch: string, + ) => Effect.Effect; + readonly remove: (threadId: ThreadId) => Effect.Effect; +} + +export class HubVcsStatusCache extends Context.Service()( + "t3/hub/HubVcs/HubVcsStatusCache", +) {} + +export const makeHubVcsStatusCache = Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + const store = yield* ThreadVcsStatusStore; + const cache = new Map(); + const changes = yield* PubSub.unbounded(); + + const seed = yield* Effect.cached( + store.list().pipe( + Effect.map((rows) => { + for (const row of rows) { + if (!cache.has(row.threadId)) { + cache.set(row.threadId, { local: row.local, remote: row.remote }); + } + } + }), + Effect.catch((error) => + Effect.logWarning("hub git status cache could not load persisted status", { + detail: error.message, + }), + ), + ), + ); + + const apply: HubVcsStatusCacheShape["apply"] = (threadId, event) => + Effect.gen(function* () { + yield* seed; + const current = cache.get(threadId); + let next: CachedStatus | undefined; + switch (event._tag) { + case "snapshot": + next = { local: event.local, remote: event.remote }; + break; + case "localUpdated": + next = { local: event.local, remote: current?.remote ?? null }; + break; + case "remoteUpdated": + next = current ? { local: current.local, remote: event.remote } : undefined; + break; + } + if (next) { + cache.set(threadId, next); + yield* store + .put({ + threadId, + local: next.local, + remote: next.remote, + updatedAt: DateTime.formatIso(yield* DateTime.now), + }) + .pipe( + Effect.catch((error) => + Effect.logWarning("hub git status cache write failed", { + threadId, + detail: error.message, + }), + ), + ); + } + yield* PubSub.publish(changes, { threadId, event }); + }); + + // While a machine is connected its runner pushes status into the cache. + yield* pool.onConnection((connection) => + connection.client["runner.vcs.streamStatus"]({ + cwd: pool.checkoutFor(connection.threadId), + }).pipe( + Stream.runForEach((event) => apply(connection.threadId, event)), + Effect.catch((error) => + Effect.logDebug("runner git status stream ended", { + threadId: connection.threadId, + detail: describeCause(error), + }), + ), + ), + ); + + return HubVcsStatusCache.of({ + get: (threadId) => seed.pipe(Effect.map(() => Option.fromNullishOr(cache.get(threadId)))), + apply, + stream: (threadId) => + Stream.unwrap( + Effect.gen(function* () { + yield* seed; + // Subscribe before reading the cache so no update falls in between. + const subscription = yield* PubSub.subscribe(changes); + const cached = cache.get(threadId); + const initial: ReadonlyArray = cached + ? [{ _tag: "snapshot", local: cached.local, remote: cached.remote }] + : []; + return Stream.concat( + Stream.fromIterable(initial), + Stream.fromSubscription(subscription).pipe( + Stream.filter((change) => change.threadId === threadId), + Stream.map((change) => change.event), + ), + ); + }), + ), + pullRequestForBranch: (branch) => + seed.pipe( + Effect.map( + () => + [...cache.values()].find( + (status) => status.local.refName === branch && status.remote?.pr, + )?.remote?.pr ?? null, + ), + ), + remove: (threadId) => + Effect.gen(function* () { + cache.delete(threadId); + yield* store.remove(threadId).pipe(Effect.ignore); + }), + }); +}); + +export const hubVcsStatusCacheLayer = Layer.effect(HubVcsStatusCache, makeHubVcsStatusCache); + +const asleepStatusError = (operation: string, cwd: string, cause: ThreadMachineUnavailableError) => + new GitManagerError({ + operation, + cwd, + detail: `No cached git status and ${cause.message}`, + cause, + }); + +/** `VcsStatusBroadcaster` over the hub cache; refreshes use a connected runner only. */ +export const hubVcsStatusBroadcasterLayer = Layer.effect( + VcsStatusBroadcaster.VcsStatusBroadcaster, + Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + const cache = yield* HubVcsStatusCache; + + const cachedOr = ( + operation: string, + cwd: string, + select: (cached: CachedStatus) => A, + live: Effect.Effect, + ): Effect.Effect => + threadOfCwd(pool, cwd, operation).pipe( + Effect.flatMap((threadId) => + live.pipe( + Effect.catch( + (error): Effect.Effect => + isThreadMachineUnavailable(error) + ? cache + .get(threadId) + .pipe( + Effect.flatMap((cached) => + Option.isSome(cached) + ? Effect.succeed(select(cached.value)) + : Effect.fail(asleepStatusError(operation, cwd, error)), + ), + ) + : Effect.fail(error), + ), + ), + ), + Effect.mapError(toGitManagerServiceError(operation, cwd)), + ); + + const mergedStatus = (cached: CachedStatus): VcsStatusResult => + mergeGitStatusParts(cached.local, cached.remote); + + return VcsStatusBroadcaster.VcsStatusBroadcaster.of({ + getStatus: (input) => + cachedOr( + "getStatus", + input.cwd, + mergedStatus, + threadOfCwd(pool, input.cwd, "getStatus").pipe( + Effect.flatMap((threadId) => cache.get(threadId)), + Effect.flatMap((cached) => + Option.isSome(cached) + ? Effect.succeed(mergedStatus(cached.value)) + : onCwdRunner(pool, input.cwd, { wake: false, operation: "getStatus" }, (client) => + client["runner.vcs.status"](input), + ), + ), + ), + ), + refreshStatus: (cwd) => + cachedOr( + "refreshStatus", + cwd, + mergedStatus, + onCwdRunner(pool, cwd, { wake: false, operation: "refreshStatus" }, (client) => + client["runner.vcs.refreshStatus"]({ cwd }), + ), + ), + refreshLocalStatus: (cwd) => + cachedOr( + "refreshLocalStatus", + cwd, + (cached) => cached.local, + onCwdRunner(pool, cwd, { wake: false, operation: "refreshLocalStatus" }, (client) => + client["runner.vcs.refreshLocalStatus"]({ cwd }), + ), + ), + streamStatus: (input) => + Stream.unwrap( + threadOfCwd(pool, input.cwd, "streamStatus").pipe( + Effect.map((threadId) => cache.stream(threadId)), + Effect.mapError(toGitManagerServiceError("streamStatus", input.cwd)), + ), + ), + }); + }), +); + +/** Git workflow on the thread's runner. Actions wake the machine; reads do not. */ +export const hubGitWorkflowServiceLayer = Layer.effect( + GitWorkflowService.GitWorkflowService, + Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + const cache = yield* HubVcsStatusCache; + const wake = (operation: string) => ({ wake: true, operation: `git.${operation}` }); + const read = (operation: string) => ({ wake: false, operation: `git.${operation}` }); + const cachedFallback = ( + operation: string, + cwd: string, + select: (cached: CachedStatus) => A, + live: Effect.Effect, + ) => + live.pipe( + Effect.catch( + (error): Effect.Effect => + isThreadMachineUnavailable(error) + ? threadOfCwd(pool, cwd, operation).pipe( + Effect.flatMap((threadId) => cache.get(threadId)), + Effect.flatMap((cached) => + Option.isSome(cached) + ? Effect.succeed(select(cached.value)) + : Effect.fail(asleepStatusError(operation, cwd, error)), + ), + ) + : Effect.fail(error), + ), + Effect.mapError(toGitManagerServiceError(operation, cwd)), + ); + const forwardInvalidate = (cwd: string, operation: string) => + threadOfCwd(pool, cwd, operation).pipe( + Effect.flatMap((threadId) => pool.current(threadId)), + Effect.flatMap((connection) => + Option.isNone(connection) + ? Effect.void + : connection.value.client["runner.vcs.refreshStatus"]({ cwd }).pipe(Effect.asVoid), + ), + Effect.ignore, + ); + + return GitWorkflowService.GitWorkflowService.of({ + status: (input) => + cachedFallback( + "status", + input.cwd, + (cached) => mergeGitStatusParts(cached.local, cached.remote), + onCwdRunner(pool, input.cwd, read("status"), (client) => + client["runner.vcs.status"](input), + ), + ), + localStatus: (input) => + cachedFallback( + "localStatus", + input.cwd, + (cached) => cached.local, + onCwdRunner(pool, input.cwd, read("localStatus"), (client) => + client["runner.vcs.localStatus"](input), + ), + ), + remoteStatus: (input) => + cachedFallback( + "remoteStatus", + input.cwd, + (cached) => cached.remote, + onCwdRunner(pool, input.cwd, read("remoteStatus"), (client) => + client["runner.vcs.remoteStatus"](input), + ), + ), + invalidateLocalStatus: (cwd) => forwardInvalidate(cwd, "invalidateLocalStatus"), + invalidateRemoteStatus: (cwd) => forwardInvalidate(cwd, "invalidateRemoteStatus"), + invalidateStatus: (cwd) => forwardInvalidate(cwd, "invalidateStatus"), + pullCurrentBranch: (cwd) => + onCwdRunner(pool, cwd, wake("pull"), (client) => client["runner.git.pull"]({ cwd })).pipe( + Effect.mapError(toGitCommandError("pullCurrentBranch", cwd)), + ), + runStackedAction: (input, options) => + streamOnCwdRunner(pool, input.cwd, wake("runStackedAction"), (client) => + client["runner.git.runStackedAction"](input), + ).pipe( + Stream.runFoldEffect( + (): GitRunStackedActionResult | null => null, + (result, event) => + event._tag === "progress" + ? (options?.progressReporter?.publish(event.event) ?? Effect.void).pipe( + Effect.as(result), + ) + : Effect.succeed(event.result), + ), + Effect.flatMap((result) => + result === null + ? Effect.fail( + new GitManagerError({ + operation: "runStackedAction", + cwd: input.cwd, + detail: "The runner ended the action without a result.", + }), + ) + : Effect.succeed(result), + ), + Effect.mapError(toGitManagerServiceError("runStackedAction", input.cwd)), + ), + resolvePullRequest: (input) => + onCwdRunner(pool, input.cwd, wake("resolvePullRequest"), (client) => + client["runner.git.resolvePullRequest"](input), + ).pipe(Effect.mapError(toGitManagerServiceError("resolvePullRequest", input.cwd))), + preparePullRequestThread: (input) => + onCwdRunner(pool, input.cwd, wake("preparePullRequestThread"), (client) => + client["runner.git.preparePullRequestThread"](input), + ).pipe(Effect.mapError(toGitManagerServiceError("preparePullRequestThread", input.cwd))), + listRefs: (input) => + onCwdRunner(pool, input.cwd, read("listRefs"), (client) => + client["runner.git.listRefs"](input), + ).pipe(Effect.mapError(toGitCommandError("listRefs", input.cwd))), + createWorktree: (input) => + onCwdRunner(pool, input.cwd, wake("createWorktree"), (client) => + client["runner.git.createWorktree"](input), + ).pipe(Effect.mapError(toGitCommandError("createWorktree", input.cwd))), + fetchRemote: (input) => + onCwdRunner(pool, input.cwd, wake("fetchRemote"), (client) => + client["runner.git.fetchRemote"](input), + ).pipe(Effect.mapError(toGitCommandError("fetchRemote", input.cwd))), + remoteExists: (input) => + onCwdRunner(pool, input.cwd, wake("remoteExists"), (client) => + client["runner.git.remoteExists"](input), + ).pipe(Effect.mapError(toGitCommandError("remoteExists", input.cwd))), + resolveRemoteTrackingCommit: (input) => + onCwdRunner(pool, input.cwd, wake("resolveRemoteTrackingCommit"), (client) => + client["runner.git.resolveRemoteTrackingCommit"](input), + ).pipe(Effect.mapError(toGitCommandError("resolveRemoteTrackingCommit", input.cwd))), + removeWorktree: (input) => + onCwdRunner(pool, input.cwd, wake("removeWorktree"), (client) => + client["runner.git.removeWorktree"](input), + ).pipe(Effect.mapError(toGitCommandError("removeWorktree", input.cwd))), + pruneWorktrees: (input) => + onCwdRunner(pool, input.cwd, wake("pruneWorktrees"), (client) => + client["runner.git.pruneWorktrees"](input), + ).pipe(Effect.mapError(toGitCommandError("pruneWorktrees", input.cwd))), + createRef: (input) => + onCwdRunner(pool, input.cwd, wake("createRef"), (client) => + client["runner.git.createRef"](input), + ).pipe(Effect.mapError(toGitCommandError("createRef", input.cwd))), + switchRef: (input) => + onCwdRunner(pool, input.cwd, wake("switchRef"), (client) => + client["runner.git.switchRef"](input), + ).pipe(Effect.mapError(toGitCommandError("switchRef", input.cwd))), + renameBranch: (input) => + onCwdRunner(pool, input.cwd, wake("renameBranch"), (client) => + client["runner.git.renameBranch"](input), + ).pipe(Effect.mapError(toGitManagerServiceError("renameBranch", input.cwd))), + }); + }), +); + +/** + * `GitManager` in hub mode serves only the automatic settlement sweep, from + * the status cache; every other operation goes through `GitWorkflowService`. + */ +export const hubGitManagerLayer = Layer.effect( + GitManager.GitManager, + Effect.gen(function* () { + const workflow = yield* GitWorkflowService.GitWorkflowService; + const cache = yield* HubVcsStatusCache; + return GitManager.GitManager.of({ + status: workflow.status, + localStatus: workflow.localStatus, + remoteStatus: (input) => workflow.remoteStatus(input), + branchPullRequest: ({ branch }) => + cache + .pullRequestForBranch(branch) + .pipe( + Effect.map((pr) => (pr ? { state: pr.state, updatedAt: pr.updatedAt ?? null } : null)), + ), + invalidateLocalStatus: workflow.invalidateLocalStatus, + invalidateRemoteStatus: workflow.invalidateRemoteStatus, + invalidateStatus: workflow.invalidateStatus, + resolvePullRequest: workflow.resolvePullRequest, + preparePullRequestThread: workflow.preparePullRequestThread, + runStackedAction: workflow.runStackedAction, + }); + }), +); + +export const hubVcsProvisioningServiceLayer = Layer.effect( + VcsProvisioningService.VcsProvisioningService, + Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + return VcsProvisioningService.VcsProvisioningService.of({ + initRepository: (input) => + onCwdRunner(pool, input.cwd, { wake: true, operation: "vcs.init" }, (client) => + client["runner.vcs.init"](input), + ).pipe(Effect.mapError(toVcsError("initRepository", input.cwd))), + }); + }), +); + +/** Review diffs are read-only: served by a running machine, never waking one. */ +export const hubReviewServiceLayer = Layer.effect( + ReviewService.ReviewService, + Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + return ReviewService.ReviewService.of({ + getDiffPreview: (input) => + onCwdRunner( + pool, + input.cwd, + { wake: false, operation: "review.getDiffPreview" }, + (client) => client["runner.review.getDiffPreview"](input), + ).pipe(Effect.mapError(toVcsError("getDiffPreview", input.cwd))), + getDiffFileContents: (input) => + onCwdRunner( + pool, + input.cwd, + { wake: false, operation: "review.getDiffFileContents" }, + (client) => client["runner.review.getDiffFileContents"](input), + ).pipe(Effect.mapError(toVcsError("getDiffFileContents", input.cwd))), + }); + }), +); diff --git a/apps/server/src/hub/HubWorkspace.ts b/apps/server/src/hub/HubWorkspace.ts new file mode 100644 index 000000000000..5679147442b3 --- /dev/null +++ b/apps/server/src/hub/HubWorkspace.ts @@ -0,0 +1,360 @@ +/** + * Workspace files and checkpoints in hub mode. + * + * - `WorkspacePaths` normalizes paths purely; a hub never stats a workspace. + * - `WorkspaceEntries` / `WorkspaceFileSystem` route to the thread's runner. + * Writes wake the machine; listing, search and reads are read-only and use a + * running machine only (a sleeping one yields the service's error type with + * a `ThreadMachineUnavailableError { reason: "asleep" }` cause). Filesystem + * browsing for the project picker has no hub implementation. + * - `CheckpointStore` captures and restores on the runner and keeps the patch + * between checkpoints in `CheckpointTurnDiffStore`. Diffs are served from + * that store without waking; only a diff that was never captured wakes the + * machine, and the result is stored. + * - `CheckoutGitProbe`: every thread checkout is a git repository by + * construction (`runner.checkout.prepare` clones or initializes it), so the + * probe answers from the path alone. + * + * @module hub/HubWorkspace + */ +import { type CheckpointRef, ThreadId, type VcsError } from "@t3tools/contracts"; +import { HubModeUnsupportedError, parseThreadCheckoutPath } from "@t3tools/contracts/runner"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Encoding from "effect/Encoding"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; + +import { CheckpointStore } from "../checkpointing/CheckpointStore.ts"; +import { CHECKPOINT_REFS_PREFIX, checkpointRefForThreadTurn } from "../checkpointing/Utils.ts"; +import { CheckoutGitProbe } from "../git/CheckoutGitProbe.ts"; +import { CheckpointTurnDiffStore } from "../persistence/Services/HubThreadMachineState.ts"; +import { fromRunnerRemoteError } from "../runner/remoteErrors.ts"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; +import * as WorkspaceFileSystem from "../workspace/WorkspaceFileSystem.ts"; +import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; +import { describeCause, onCwdRunner, threadOfCwd, toVcsError } from "./hubRouting.ts"; +import { RunnerConnectionPool } from "./RunnerConnectionPool.ts"; + +export const hubWorkspacePathsLayer = Layer.effect( + WorkspacePaths.WorkspacePaths, + Effect.gen(function* () { + // Relative-path resolution is pure; only root validation touches disk. + const local = yield* WorkspacePaths.make; + const path = yield* Path.Path; + return WorkspacePaths.WorkspacePaths.of({ + normalizeWorkspaceRoot: (workspaceRoot) => Effect.succeed(path.resolve(workspaceRoot.trim())), + resolveRelativePathWithinRoot: local.resolveRelativePathWithinRoot, + }); + }), +); + +const WorkspaceFileErrorSchema = WorkspaceFileSystem.WorkspaceFileSystemError; + +export const hubWorkspaceEntriesLayer = Layer.effect( + WorkspaceEntries.WorkspaceEntries, + Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + const toEntriesError = + (cwd: string) => + (error: unknown): WorkspaceEntries.WorkspaceEntriesError => { + const fallback = (cause: unknown) => + new WorkspacePaths.WorkspaceRootStatFailedError({ + workspaceRoot: cwd, + normalizedWorkspaceRoot: cwd, + phase: "validate-existing", + cause, + }); + if ( + error && + typeof error === "object" && + (error as { _tag?: unknown })._tag === "RunnerRemoteError" + ) { + return fromRunnerRemoteError( + WorkspaceEntries.WorkspaceEntriesError, + fallback, + )(error as never); + } + return fallback(error); + }; + const read = (operation: string) => ({ wake: false, operation: `workspace.${operation}` }); + return WorkspaceEntries.WorkspaceEntries.of({ + browse: (input) => + Effect.fail( + new WorkspaceEntries.WorkspaceEntriesReadDirectoryError({ + ...(input.cwd !== undefined ? { cwd: input.cwd } : {}), + partialPath: input.partialPath, + parentPath: input.partialPath, + cause: new HubModeUnsupportedError({ + operation: "filesystem.browse", + detail: "A hub has no filesystem to browse; projects come from repositories.", + }), + }), + ), + list: (input) => + onCwdRunner(pool, input.cwd, read("listEntries"), (client) => + client["runner.workspace.listEntries"](input), + ).pipe(Effect.mapError(toEntriesError(input.cwd))), + search: (input) => + onCwdRunner(pool, input.cwd, read("searchEntries"), (client) => + client["runner.workspace.searchEntries"](input), + ).pipe(Effect.mapError(toEntriesError(input.cwd))), + searchContents: (input) => + onCwdRunner(pool, input.cwd, read("searchContents"), (client) => + client["runner.workspace.searchContents"](input), + ).pipe(Effect.mapError(toEntriesError(input.cwd))), + // Refreshing the runner's index is only useful while it is connected. + refresh: (cwd) => + threadOfCwd(pool, cwd, "workspace.refreshIndex").pipe( + Effect.flatMap((threadId) => pool.current(threadId)), + Effect.flatMap((connection) => + Option.isNone(connection) + ? Effect.void + : connection.value.client["runner.workspace.refreshIndex"]({ cwd }), + ), + Effect.ignore, + ), + }); + }), +); + +export const hubWorkspaceFileSystemLayer = Layer.effect( + WorkspaceFileSystem.WorkspaceFileSystem, + Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + const path = yield* Path.Path; + const toFileError = + ( + input: { readonly cwd: string; readonly relativePath: string }, + operation: "open" | "write-file", + ) => + (error: unknown) => { + const fallback = (cause: unknown) => + new WorkspaceFileSystem.WorkspaceFileSystemOperationError({ + workspaceRoot: input.cwd, + relativePath: input.relativePath, + resolvedPath: path.join(input.cwd, input.relativePath), + operationPath: input.cwd, + operation, + cause, + }); + if ( + error && + typeof error === "object" && + (error as { _tag?: unknown })._tag === "RunnerRemoteError" + ) { + const decoded = fromRunnerRemoteError(WorkspaceFileErrorSchema, (remote) => + remote.errorTag === "WorkspacePathOutsideRootError" + ? new WorkspacePaths.WorkspacePathOutsideRootError({ + workspaceRoot: input.cwd, + relativePath: input.relativePath, + }) + : fallback(remote), + )(error as never); + return decoded; + } + return fallback(error); + }; + return WorkspaceFileSystem.WorkspaceFileSystem.of({ + readFile: (input) => + onCwdRunner(pool, input.cwd, { wake: false, operation: "workspace.readFile" }, (client) => + client["runner.workspace.readFile"](input), + ).pipe(Effect.mapError(toFileError(input, "open"))), + writeFile: (input) => + onCwdRunner(pool, input.cwd, { wake: true, operation: "workspace.writeFile" }, (client) => + client["runner.workspace.writeFile"](input), + ).pipe(Effect.mapError(toFileError(input, "write-file"))), + }); + }), +); + +/** `refs/t3/checkpoints//turn/` → thread and turn count. */ +export const parseCheckpointRef = ( + ref: CheckpointRef, +): { readonly threadId: ThreadId; readonly turnCount: number } | null => { + const prefix = `${CHECKPOINT_REFS_PREFIX}/`; + if (!ref.startsWith(prefix)) return null; + const match = /^([^/]+)\/turn\/(\d+)$/.exec(ref.slice(prefix.length)); + if (!match) return null; + const decoded = Encoding.decodeBase64UrlString(match[1]!); + if (decoded._tag !== "Success" || decoded.success.length === 0) return null; + return { threadId: ThreadId.make(decoded.success), turnCount: Number(match[2]) }; +}; + +/** Diffs worth having before the machine sleeps: the turn and the whole thread, both whitespace modes. */ +const precomputedDiffs = (turnCount: number) => + [turnCount - 1, 0] + .filter((from, index, all) => from >= 0 && from < turnCount && all.indexOf(from) === index) + .flatMap((fromTurnCount) => + [true, false].map((ignoreWhitespace) => ({ fromTurnCount, ignoreWhitespace })), + ); + +export const hubCheckpointStoreLayer = Layer.effect( + CheckpointStore, + Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + const diffs = yield* CheckpointTurnDiffStore; + const wake = (operation: string) => ({ wake: true, operation: `checkpoint.${operation}` }); + + const ignoreStoreFailure = (effect: Effect.Effect) => + effect.pipe( + Effect.asVoid, + Effect.catch((error) => + Effect.logWarning("checkpoint diff store failed", { detail: error.message }), + ), + ); + const invalidateFrom = (threadId: ThreadId, turnCount: number) => + ignoreStoreFailure(diffs.invalidateFromTurn({ threadId, turnCount })); + + const diffOnRunner = ( + input: { + readonly cwd: string; + readonly fromCheckpointRef: CheckpointRef; + readonly toCheckpointRef: CheckpointRef; + readonly fallbackFromToHead?: boolean; + readonly ignoreWhitespace: boolean; + }, + wakeMachine: boolean, + ) => + onCwdRunner(pool, input.cwd, { wake: wakeMachine, operation: "checkpoint.diff" }, (client) => + client["runner.checkpoint.diff"]({ + cwd: input.cwd, + fromCheckpointRef: input.fromCheckpointRef, + toCheckpointRef: input.toCheckpointRef, + ignoreWhitespace: input.ignoreWhitespace, + ...(input.fallbackFromToHead !== undefined + ? { fallbackFromToHead: input.fallbackFromToHead } + : {}), + }), + ); + + const storeDiff = ( + key: { + readonly threadId: ThreadId; + readonly fromTurnCount: number; + readonly toTurnCount: number; + readonly ignoreWhitespace: boolean; + }, + diff: string, + ) => + DateTime.now.pipe( + Effect.flatMap((now) => + ignoreStoreFailure(diffs.put({ ...key, diff, createdAt: DateTime.formatIso(now) })), + ), + ); + + /** While the machine is still connected, capture the diffs clients ask for most. */ + const precompute = (cwd: string, threadId: ThreadId, turnCount: number) => + Effect.forEach( + precomputedDiffs(turnCount), + ({ fromTurnCount, ignoreWhitespace }) => + Effect.gen(function* () { + const key = { threadId, fromTurnCount, toTurnCount: turnCount, ignoreWhitespace }; + const cached = yield* diffs.get(key).pipe(Effect.orElseSucceed(() => Option.none())); + if (Option.isSome(cached)) return; + const diff = yield* diffOnRunner( + { + cwd, + fromCheckpointRef: checkpointRefForThreadTurn(threadId, fromTurnCount), + toCheckpointRef: checkpointRefForThreadTurn(threadId, turnCount), + ignoreWhitespace, + }, + false, + ); + yield* storeDiff(key, diff); + }).pipe( + Effect.catch((error) => + Effect.logDebug("checkpoint diff precompute skipped", { + threadId, + turnCount, + detail: describeCause(error), + }), + ), + ), + { discard: true }, + ); + + return CheckpointStore.of({ + isGitRepository: (cwd) => + Effect.succeed(parseThreadCheckoutPath(cwd, pool.checkoutRoot) !== null), + captureCheckpoint: (input) => + Effect.gen(function* () { + yield* onCwdRunner(pool, input.cwd, wake("capture"), (client) => + client["runner.checkpoint.capture"](input), + ); + const parsed = parseCheckpointRef(input.checkpointRef); + if (!parsed) return; + // A recaptured checkpoint invalidates every diff that read the old one. + yield* invalidateFrom(parsed.threadId, parsed.turnCount); + yield* precompute(input.cwd, parsed.threadId, parsed.turnCount).pipe(Effect.forkDetach); + }).pipe(Effect.mapError(toVcsError("captureCheckpoint", input.cwd))), + hasCheckpointRef: (input) => + onCwdRunner(pool, input.cwd, wake("hasRef"), (client) => + client["runner.checkpoint.hasRef"](input), + ).pipe(Effect.mapError(toVcsError("hasCheckpointRef", input.cwd))), + restoreCheckpoint: (input) => + Effect.gen(function* () { + const restored = yield* onCwdRunner(pool, input.cwd, wake("restore"), (client) => + client["runner.checkpoint.restore"]({ + cwd: input.cwd, + checkpointRef: input.checkpointRef, + ...(input.fallbackToHead !== undefined + ? { fallbackToHead: input.fallbackToHead } + : {}), + }), + ); + const parsed = parseCheckpointRef(input.checkpointRef); + if (restored && parsed) yield* invalidateFrom(parsed.threadId, parsed.turnCount + 1); + return restored; + }).pipe(Effect.mapError(toVcsError("restoreCheckpoint", input.cwd))), + diffCheckpoints: (input): Effect.Effect => + Effect.gen(function* () { + const from = parseCheckpointRef(input.fromCheckpointRef); + const to = parseCheckpointRef(input.toCheckpointRef); + const key = + from && to && from.threadId === to.threadId && input.fallbackFromToHead !== true + ? { + threadId: to.threadId, + fromTurnCount: from.turnCount, + toTurnCount: to.turnCount, + ignoreWhitespace: input.ignoreWhitespace, + } + : null; + if (key) { + const cached = yield* diffs.get(key).pipe(Effect.orElseSucceed(() => Option.none())); + if (Option.isSome(cached)) return cached.value; + } + // Never captured (or not cacheable): the runner computes it, waking the machine. + const diff: string = yield* diffOnRunner(input, true); + if (key) yield* storeDiff(key, diff); + return diff; + }).pipe(Effect.mapError(toVcsError("diffCheckpoints", input.cwd))), + deleteCheckpointRefs: (input) => + Effect.gen(function* () { + yield* onCwdRunner(pool, input.cwd, wake("deleteRefs"), (client) => + client["runner.checkpoint.deleteRefs"](input), + ); + const parsed = input.checkpointRefs.flatMap((ref) => { + const value = parseCheckpointRef(ref); + return value ? [value] : []; + }); + const earliest = parsed.reduce( + (min, value) => (min === null ? value.turnCount : Math.min(min, value.turnCount)), + null, + ); + if (earliest !== null && parsed[0]) yield* invalidateFrom(parsed[0].threadId, earliest); + }).pipe(Effect.mapError(toVcsError("deleteCheckpointRefs", input.cwd))), + }); + }), +); + +/** Thread checkouts are git repositories by construction; no machine is asked. */ +export const hubCheckoutGitProbeLayer = Layer.effect( + CheckoutGitProbe, + Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + return (cwd: string) => + Effect.succeed(parseThreadCheckoutPath(cwd, pool.checkoutRoot) !== null); + }), +); diff --git a/apps/server/src/hub/MachineDirectory.ts b/apps/server/src/hub/MachineDirectory.ts new file mode 100644 index 000000000000..d892e42a90e9 --- /dev/null +++ b/apps/server/src/hub/MachineDirectory.ts @@ -0,0 +1,301 @@ +/** + * MachineDirectory - where a hub finds each thread's machine and runner. + * + * The platform (not T3) creates, pauses, resumes and retires thread machines + * and exposes each runner to the hub through an authenticated tunnel. The hub + * asks the directory at `T3CODE_HUB_MACHINES_URL`: + * + * POST /threads/{threadId}/machine ensure; resume or recreate when `wake` + * GET /threads/{threadId}/machine current state; never wakes + * POST /threads/{threadId}/machine/idle nothing on the hub needs the machine + * POST /threads/{threadId}/machine/release the thread was archived or deleted + * + * Implementations: + * - `layerHttp`: the documented HTTP contract (production). + * - `makeStaticMachineDirectory`: one runner at `T3CODE_RUNNER_URL` that is + * always running (local development against a single `t3 runner`). + * - `makeFakeMachineDirectory`: scriptable states for tests. + * + * @module hub/MachineDirectory + */ +import type { ThreadId } from "@t3tools/contracts"; +import { + type ThreadMachineEnsureRequest, + type ThreadMachineState, + ThreadMachineStatus, +} from "@t3tools/contracts/runner"; +import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; + +import { ServerConfig } from "../config.ts"; + +export class MachineDirectoryError extends Schema.TaggedErrorClass()( + "MachineDirectoryError", + { + operation: Schema.String, + threadId: Schema.String, + status: Schema.optional(Schema.Number), + detail: Schema.String, + }, +) { + override get message(): string { + const status = this.status === undefined ? "" : ` (HTTP ${this.status})`; + return `Machine directory ${this.operation} failed for thread ${this.threadId}${status}: ${this.detail}`; + } +} + +const isMachineDirectoryError = Schema.is(MachineDirectoryError); + +export interface MachineDirectoryShape { + /** Ensures a machine exists; resumes or recreates it when `request.wake`. */ + readonly ensure: ( + threadId: ThreadId, + request: ThreadMachineEnsureRequest, + ) => Effect.Effect; + /** Reports the machine's state without waking it. */ + readonly status: ( + threadId: ThreadId, + ) => Effect.Effect; + /** Tells the platform no session or turn on the hub needs the machine. */ + readonly idle: (threadId: ThreadId) => Effect.Effect; + /** Retires the machine; the thread was archived or deleted. */ + readonly release: (threadId: ThreadId) => Effect.Effect; +} + +export class MachineDirectory extends Context.Service()( + "t3/hub/MachineDirectory", +) {} + +const DEFAULT_REQUEST_TIMEOUT = Duration.seconds(30); + +export const makeHttpMachineDirectory = (options: { + readonly baseUrl: string; + readonly token: string; + readonly requestTimeout?: Duration.Input; +}) => + Effect.gen(function* () { + const client = yield* HttpClient.HttpClient; + const base = options.baseUrl.replace(/\/+$/, ""); + const timeout = options.requestTimeout ?? DEFAULT_REQUEST_TIMEOUT; + const url = (threadId: ThreadId, suffix = "") => + `${base}/threads/${encodeURIComponent(threadId)}/machine${suffix}`; + + const toDirectoryError = (operation: string, threadId: ThreadId) => (cause: unknown) => + isMachineDirectoryError(cause) + ? cause + : new MachineDirectoryError({ + operation, + threadId, + detail: + cause && typeof cause === "object" && "message" in cause + ? String((cause as { readonly message: unknown }).message) + : String(cause), + }); + + const send = ( + operation: string, + threadId: ThreadId, + request: Effect.Effect, + decode: (response: HttpClientResponse.HttpClientResponse) => Effect.Effect, + ): Effect.Effect => + Effect.gen(function* () { + const response = yield* request.pipe( + Effect.map(HttpClientRequest.bearerToken(options.token)), + Effect.flatMap(client.execute), + Effect.mapError(toDirectoryError(operation, threadId)), + ); + if (response.status < 200 || response.status >= 300) { + const body = yield* response.text.pipe(Effect.orElseSucceed(() => "")); + return yield* new MachineDirectoryError({ + operation, + threadId, + status: response.status, + detail: body.slice(0, 500) || "request failed", + }); + } + return yield* decode(response).pipe(Effect.mapError(toDirectoryError(operation, threadId))); + }).pipe( + Effect.timeoutOrElse({ + duration: timeout, + orElse: () => + Effect.fail( + new MachineDirectoryError({ + operation, + threadId, + detail: `no response within ${Duration.format(Duration.fromInputUnsafe(timeout))}`, + }), + ), + }), + ); + + const decodeStatus = HttpClientResponse.schemaBodyJson(ThreadMachineStatus); + + return MachineDirectory.of({ + ensure: (threadId, request) => + send( + "ensure", + threadId, + HttpClientRequest.post(url(threadId)).pipe(HttpClientRequest.bodyJson(request)), + decodeStatus, + ), + status: (threadId) => + send( + "status", + threadId, + Effect.succeed(HttpClientRequest.get(url(threadId))), + decodeStatus, + ), + idle: (threadId) => + send( + "idle", + threadId, + HttpClientRequest.post(url(threadId, "/idle")).pipe(HttpClientRequest.bodyJson({})), + () => Effect.void, + ), + release: (threadId) => + send( + "release", + threadId, + HttpClientRequest.post(url(threadId, "/release")).pipe(HttpClientRequest.bodyJson({})), + () => Effect.void, + ), + }); + }); + +/** + * Development directory: every thread resolves to one always-running runner. + * A runner serves exactly one thread, so other threads fail its handshake with + * a typed error rather than sharing its checkout. + */ +export const makeStaticMachineDirectory = (options: { + readonly runnerUrl: string; + readonly runnerToken: string | undefined; +}) => + MachineDirectory.of({ + ensure: () => Effect.succeed(staticStatus(options)), + status: () => Effect.succeed(staticStatus(options)), + idle: () => Effect.void, + release: () => Effect.void, + }); + +const staticStatus = (options: { + readonly runnerUrl: string; + readonly runnerToken: string | undefined; +}): ThreadMachineStatus => ({ + state: "running", + // The static runner's token is optional in development; the protocol + // endpoint is open when the runner has none. + runner: { url: options.runnerUrl, token: options.runnerToken ?? "development" }, + bootId: null, + detail: "static development runner", +}); + +export interface FakeMachine { + readonly state: ThreadMachineState; + readonly runnerUrl?: string; + readonly runnerToken?: string; + readonly bootId?: string | null; + readonly detail?: string; +} + +export interface FakeMachineDirectoryCall { + readonly method: "ensure" | "status" | "idle" | "release"; + readonly threadId: ThreadId; + readonly wake?: boolean; +} + +/** + * Scriptable directory for tests. `onWake` decides what an ensure-with-wake + * does to a machine (by default it becomes `running`). + */ +export const makeFakeMachineDirectory = (options?: { + readonly initial?: ReadonlyArray; + readonly onWake?: (threadId: ThreadId, machine: FakeMachine | undefined) => FakeMachine; +}) => + Effect.gen(function* () { + const machines = yield* Ref.make(new Map(options?.initial ?? [])); + const calls = yield* Ref.make>([]); + const record = (call: FakeMachineDirectoryCall) => + Ref.update(calls, (current) => [...current, call]); + const toStatus = (machine: FakeMachine | undefined): ThreadMachineStatus => + machine === undefined + ? { state: "none", runner: null, bootId: null, detail: null } + : { + state: machine.state, + runner: + machine.state === "running" && machine.runnerUrl + ? { url: machine.runnerUrl, token: machine.runnerToken ?? "test-token" } + : null, + bootId: machine.bootId ?? null, + detail: machine.detail ?? null, + }; + const directory = MachineDirectory.of({ + ensure: (threadId, request) => + Effect.gen(function* () { + yield* record({ method: "ensure", threadId, wake: request.wake }); + const current = (yield* Ref.get(machines)).get(threadId); + if (!request.wake) return toStatus(current); + const next = + options?.onWake?.(threadId, current) ?? + ({ ...current, state: "running" } satisfies FakeMachine); + yield* Ref.update(machines, (map) => new Map(map).set(threadId, next)); + return toStatus(next); + }), + status: (threadId) => + record({ method: "status", threadId }).pipe( + Effect.andThen(Ref.get(machines)), + Effect.map((map) => toStatus(map.get(threadId))), + ), + idle: (threadId) => record({ method: "idle", threadId }), + release: (threadId) => + record({ method: "release", threadId }).pipe( + Effect.andThen( + Ref.update(machines, (map) => { + const next = new Map(map); + next.delete(threadId); + return next; + }), + ), + ), + }); + return { + directory, + calls: Ref.get(calls), + set: (threadId: ThreadId, machine: FakeMachine) => + Ref.update(machines, (map) => new Map(map).set(threadId, machine)), + get: (threadId: ThreadId) => Ref.get(machines).pipe(Effect.map((map) => map.get(threadId))), + }; + }); + +/** + * The configured directory: HTTP when `T3CODE_HUB_MACHINES_URL` is set, else + * the static development runner at `T3CODE_RUNNER_URL`. + */ +export const layer = Layer.effect( + MachineDirectory, + Effect.gen(function* () { + const config = yield* ServerConfig; + if (config.hub?.machinesUrl && config.hub.machinesToken) { + return yield* makeHttpMachineDirectory({ + baseUrl: config.hub.machinesUrl, + token: config.hub.machinesToken, + }); + } + if (config.runnerUrl) { + return makeStaticMachineDirectory({ + runnerUrl: config.runnerUrl, + runnerToken: config.runnerToken, + }); + } + return yield* Effect.die( + new Error( + "Hub mode requires T3CODE_HUB_MACHINES_URL and T3CODE_HUB_MACHINES_TOKEN, or T3CODE_RUNNER_URL for development.", + ), + ); + }), +); diff --git a/apps/server/src/hub/RemoteProviderDriver.ts b/apps/server/src/hub/RemoteProviderDriver.ts new file mode 100644 index 000000000000..c788601f2b5e --- /dev/null +++ b/apps/server/src/hub/RemoteProviderDriver.ts @@ -0,0 +1,535 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Reads attachment bytes from the hub's own attachment store. +/** + * RemoteProviderDriver - the hub's provider driver. It runs nothing locally. + * + * It wraps each built-in driver's identity (kind, config schema, defaults) so + * the hub keeps the same instance ids, model selections and session + * directory rows, while its adapter forwards every call to the runner on the + * thread's own machine. `ProviderService` above it is unchanged: it still + * persists resume cursors, mints MCP credentials, recovers sessions and + * serializes per-thread lifecycle work. + * + * Wake semantics (see docs/internals/thread-machines.md): + * - `startSession`, `sendTurn`, `respondToRequest`, `respondToUserInput`, + * `rollbackThread`, `readThread`, and interrupting an active turn wake the + * machine. + * - `hasSession` and `listSessions` answer from the `RemoteSessionRegistry` + * and never contact a machine: a session on a sleeping machine is + * resumable, so it still exists. + * - `stopSession` stops a session only on a running machine; a sleeping + * machine is left asleep and the hub forgets the session. + * - `stopAll` does nothing: the sessions run on other machines and outlive + * the hub process (`capabilities.sessionsOutliveServer`). + * + * A session the runner lost while its machine slept (the machine was + * recreated) is restarted from the recorded resume cursor before a turn is + * sent, after boot reconciliation has settled the old one. + * + * @module hub/RemoteProviderDriver + */ +import * as NodeFS from "node:fs"; + +import { + type ChatAttachment, + type ProviderDriverKind, + type ProviderInstanceId, + type ProviderSession, + type ServerProvider, + TextGenerationError, + type ThreadId, +} from "@t3tools/contracts"; +import { + parseThreadCheckoutPath, + type RunnerAttachmentFile, + type RunnerMcpSession, + ThreadMachineUnavailableError, +} from "@t3tools/contracts/runner"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +import { resolveAttachmentPath } from "../attachmentStore.ts"; +import { ServerConfig } from "../config.ts"; +import * as McpProviderSession from "../mcp/McpProviderSession.ts"; +import { + type ProviderAdapterError, + ProviderAdapterRequestError, + ProviderAdapterSessionNotFoundError, +} from "../provider/Errors.ts"; +import { + defaultProviderContinuationIdentity, + type AnyProviderDriver, + type ProviderInstance, +} from "../provider/ProviderDriver.ts"; +import type { + ProviderAdapterCapabilities, + ProviderAdapterShape, + ProviderThreadSnapshot, +} from "../provider/Services/ProviderAdapter.ts"; +import type { ServerProviderShape } from "../provider/Services/ServerProvider.ts"; +import { buildUnavailableProviderSnapshot } from "../provider/unavailableProviderSnapshot.ts"; +import type { TextGenerationShape } from "../textGeneration/TextGeneration.ts"; +import { fromRunnerRemoteError, ProviderAdapterErrorSchema } from "../runner/remoteErrors.ts"; +import { RemoteSessionRegistry } from "./RemoteSessionRegistry.ts"; +import { RunnerConnectionPool, type RunnerConnection } from "./RunnerConnectionPool.ts"; +import { RunnerEventDelivery } from "./RunnerEventDelivery.ts"; + +export type RemoteProviderDriverEnv = + | RunnerConnectionPool + | RunnerEventDelivery + | RemoteSessionRegistry + | ServerConfig; + +const isThreadMachineUnavailable = Schema.is(ThreadMachineUnavailableError); +const isProviderAdapterError = Schema.is(ProviderAdapterErrorSchema); +const isTextGenerationError = Schema.is(TextGenerationError); + +const describe = (cause: unknown): string => + cause && typeof cause === "object" && "message" in cause + ? String((cause as { readonly message: unknown }).message) + : String(cause); + +/** Rewrites the hub-minted MCP session to the hub's public endpoint. */ +export const mcpSessionForRunner = ( + threadId: ThreadId, + publicUrl: string | undefined, +): RunnerMcpSession | null => { + const session = McpProviderSession.readMcpProviderSession(threadId); + if (!session) return null; + return { + ...session, + endpoint: publicUrl ? `${publicUrl.replace(/\/+$/, "")}/mcp` : session.endpoint, + }; +}; + +export class RunnerAttachmentReadError extends Schema.TaggedErrorClass()( + "RunnerAttachmentReadError", + { attachmentId: Schema.String, cause: Schema.Defect() }, +) { + override get message(): string { + return `Attachment ${this.attachmentId} could not be read from the hub attachment store.`; + } +} + +/** Reads attachment bytes from the hub's attachment store for shipping with a turn. */ +export const attachmentFilesForRunner = ( + attachmentsDir: string, + attachments: ReadonlyArray | undefined, +) => + Effect.forEach(attachments ?? [], (attachment) => + Effect.try({ + try: (): ReadonlyArray => { + const hubPath = resolveAttachmentPath({ attachmentsDir, attachment }); + if (hubPath === null || !NodeFS.existsSync(hubPath)) return []; + return [ + { attachment, hubPath, bytesBase64: NodeFS.readFileSync(hubPath).toString("base64") }, + ]; + }, + catch: (cause) => new RunnerAttachmentReadError({ attachmentId: attachment.id, cause }), + }), + ).pipe(Effect.map((files) => files.flat())); + +interface InstanceContext { + readonly driverKind: ProviderDriverKind; + readonly instanceId: ProviderInstanceId; + readonly pool: RunnerConnectionPool["Service"]; + readonly delivery: RunnerEventDelivery["Service"]; + readonly registry: RemoteSessionRegistry["Service"]; + readonly attachmentsDir: string; + readonly publicUrl: string | undefined; +} + +export function makeRemoteProviderDriver( + base: AnyProviderDriver, +): AnyProviderDriver { + const driverKind: ProviderDriverKind = base.driverKind; + return { + driverKind, + metadata: base.metadata, + configSchema: base.configSchema, + defaultConfig: base.defaultConfig, + create: ({ instanceId, displayName, accentColor, enabled }) => + Effect.gen(function* () { + const config = yield* ServerConfig; + const context: InstanceContext = { + driverKind, + instanceId, + pool: yield* RunnerConnectionPool, + delivery: yield* RunnerEventDelivery, + registry: yield* RemoteSessionRegistry, + attachmentsDir: config.attachmentsDir, + publicUrl: config.hub?.publicUrl, + }; + const capabilities: { + current: Omit; + } = { current: { sessionModelSwitch: "in-session" } }; + const snapshot = yield* makeRemoteSnapshot({ + context, + displayName, + accentColor, + onCapabilities: (sessionModelSwitch) => + Effect.sync(() => { + capabilities.current = { ...capabilities.current, sessionModelSwitch }; + }), + }); + return { + instanceId, + driverKind, + continuationIdentity: defaultProviderContinuationIdentity({ driverKind, instanceId }), + displayName, + accentColor, + enabled, + snapshot, + adapter: makeRemoteAdapter(context, () => capabilities.current), + textGeneration: makeRemoteTextGeneration(context), + } satisfies ProviderInstance; + }), + }; +} + +function makeRemoteAdapter( + context: InstanceContext, + currentCapabilities: () => Omit, +): ProviderAdapterShape { + const { driverKind, instanceId, pool, delivery, registry } = context; + + const toAdapterError = + (method: string) => + (error: unknown): ProviderAdapterError => { + if (isProviderAdapterError(error)) return error; + if ( + error && + typeof error === "object" && + (error as { _tag?: unknown })._tag === "RunnerRemoteError" + ) { + return fromRunnerRemoteError( + ProviderAdapterErrorSchema, + (remote) => + new ProviderAdapterRequestError({ + provider: driverKind, + method, + detail: remote.message, + cause: remote, + }), + )(error as never); + } + return new ProviderAdapterRequestError({ + provider: driverKind, + method, + detail: isThreadMachineUnavailable(error) + ? error.message + : `Runner call failed: ${describe(error)}`, + cause: error, + }); + }; + + /** A call on the thread's runner after boot reconciliation of the connection. */ + const onRunner = ( + threadId: ThreadId, + method: string, + wake: boolean, + f: (connection: RunnerConnection) => Effect.Effect, + ): Effect.Effect => + pool + .use(threadId, { wake, operation: `provider.${method}` }, (connection) => + delivery.awaitReconciled(connection).pipe(Effect.andThen(f(connection))), + ) + .pipe(Effect.mapError(toAdapterError(method))); + + const sessionNotFound = (threadId: ThreadId) => + new ProviderAdapterSessionNotFoundError({ provider: driverKind, threadId }); + + const recordSession = (session: ProviderSession, connection: RunnerConnection) => + registry.upsert({ + threadId: session.threadId, + instanceId, + provider: driverKind, + session: { ...session, providerInstanceId: instanceId }, + bootId: connection.hello.bootId, + }); + + const startOnRunner = ( + connection: RunnerConnection, + input: Parameters["startSession"]>[0], + ) => + connection.client["runner.provider.startSession"]({ + instanceId, + input, + mcp: mcpSessionForRunner(input.threadId, context.publicUrl), + }).pipe(Effect.tap((session) => recordSession(session, connection))); + + const hostedSession = (threadId: ThreadId) => + registry + .get(threadId) + .pipe(Effect.map(Option.filter((record) => record.instanceId === instanceId))); + + return { + provider: driverKind, + get capabilities() { + return { ...currentCapabilities(), sessionsOutliveServer: true }; + }, + startSession: (input) => + onRunner(input.threadId, "startSession", true, (connection) => + startOnRunner(connection, input), + ), + sendTurn: (input) => + Effect.gen(function* () { + // Captured before waking: if the machine was recreated while asleep, + // reconciliation settles the old session and this restarts it. + const before = yield* hostedSession(input.threadId); + const attachments = yield* attachmentFilesForRunner( + context.attachmentsDir, + input.attachments, + ).pipe(Effect.mapError(toAdapterError("sendTurn"))); + return yield* onRunner(input.threadId, "sendTurn", true, (connection) => + Effect.gen(function* () { + const current = yield* hostedSession(input.threadId); + if (Option.isNone(current)) { + if (Option.isNone(before)) return yield* sessionNotFound(input.threadId); + const previous = before.value.session; + yield* Effect.logInfo("restarting a session its recreated machine lost", { + threadId: input.threadId, + instanceId, + }); + yield* startOnRunner(connection, { + threadId: input.threadId, + provider: driverKind, + providerInstanceId: instanceId, + runtimeMode: previous.runtimeMode, + ...(previous.cwd !== undefined ? { cwd: previous.cwd } : {}), + ...(previous.resumeCursor !== undefined + ? { resumeCursor: previous.resumeCursor } + : {}), + ...(input.modelSelection !== undefined + ? { modelSelection: input.modelSelection } + : {}), + }); + } + return yield* connection.client["runner.provider.sendTurn"]({ + instanceId, + input, + attachments, + }); + }), + ); + }), + interruptTurn: (threadId, turnId) => + hostedSession(threadId).pipe( + Effect.flatMap((record) => + // Nothing runs on a machine without an active turn; do not wake it. + Option.isNone(record) || record.value.session.activeTurnId === undefined + ? Effect.void + : onRunner(threadId, "interruptTurn", true, (connection) => + connection.client["runner.provider.interruptTurn"]({ + instanceId, + threadId, + ...(turnId !== undefined ? { turnId } : {}), + }), + ), + ), + ), + respondToRequest: (threadId, requestId, decision) => + onRunner(threadId, "respondToRequest", true, (connection) => + Effect.gen(function* () { + if (Option.isNone(yield* hostedSession(threadId))) { + return yield* sessionNotFound(threadId); + } + return yield* connection.client["runner.provider.respondToRequest"]({ + instanceId, + threadId, + requestId, + decision, + }); + }), + ), + respondToUserInput: (threadId, requestId, answers) => + onRunner(threadId, "respondToUserInput", true, (connection) => + Effect.gen(function* () { + if (Option.isNone(yield* hostedSession(threadId))) { + return yield* sessionNotFound(threadId); + } + return yield* connection.client["runner.provider.respondToUserInput"]({ + instanceId, + threadId, + requestId, + answers, + }); + }), + ), + stopSession: (threadId) => + Effect.gen(function* () { + const connection = yield* pool.current(threadId); + yield* registry.remove(threadId); + if (Option.isNone(connection)) return; + yield* connection.value.client["runner.provider.stopSession"]({ + instanceId, + threadId, + }).pipe(Effect.mapError(toAdapterError("stopSession"))); + }), + listSessions: () => + registry + .listForInstance(instanceId) + .pipe(Effect.map((records) => records.map((record) => record.session))), + hasSession: (threadId) => hostedSession(threadId).pipe(Effect.map(Option.isSome)), + readThread: (threadId) => + onRunner(threadId, "readThread", true, (connection) => + connection.client["runner.provider.readThread"]({ instanceId, threadId }), + ).pipe(Effect.map((snapshot) => snapshot as ProviderThreadSnapshot)), + rollbackThread: (threadId, numTurns) => + onRunner(threadId, "rollbackThread", true, (connection) => + connection.client["runner.provider.rollbackThread"]({ instanceId, threadId, numTurns }), + ).pipe(Effect.map((snapshot) => snapshot as ProviderThreadSnapshot)), + stopAll: () => Effect.void, + streamEvents: delivery.events.pipe( + Stream.filter((event) => event.providerInstanceId === instanceId), + ), + }; +} + +/** + * Provider status and models as the most recent runner reported them. A hub + * cannot probe provider CLIs; until a thread machine reports this instance + * the snapshot says so. + */ +const makeRemoteSnapshot = (input: { + readonly context: InstanceContext; + readonly displayName: string | undefined; + readonly accentColor: string | undefined; + readonly onCapabilities: ( + sessionModelSwitch: ProviderAdapterCapabilities["sessionModelSwitch"], + ) => Effect.Effect; +}) => + Effect.gen(function* () { + const { context } = input; + const unavailable = (reason: string) => + buildUnavailableProviderSnapshot({ + driverKind: context.driverKind, + instanceId: context.instanceId, + displayName: input.displayName, + accentColor: input.accentColor, + reason, + }); + const current = yield* Ref.make( + yield* unavailable("No thread machine has reported this provider yet."), + ); + const changes = yield* PubSub.unbounded(); + + const fetchFrom = (connection: RunnerConnection, refresh: boolean) => + connection.hello.instances.includes(context.instanceId) + ? connection.client["runner.provider.getCapabilities"]({ + instanceId: context.instanceId, + ...(refresh ? { refresh } : {}), + }).pipe( + Effect.tap((result) => input.onCapabilities(result.sessionModelSwitch)), + Effect.map((result) => result.snapshot), + Effect.tap((snapshot) => Ref.set(current, snapshot)), + Effect.tap((snapshot) => PubSub.publish(changes, snapshot)), + Effect.asVoid, + Effect.catch((error) => + Effect.logWarning("runner did not report provider capabilities", { + instanceId: context.instanceId, + detail: describe(error), + }), + ), + ) + : Effect.void; + + yield* context.pool.onConnection((connection) => fetchFrom(connection, false)); + + const refresh = Effect.gen(function* () { + const [connection] = yield* context.pool.connections; + if (connection) yield* fetchFrom(connection, true); + return yield* Ref.get(current); + }); + + return { + maintenanceCapabilities: { provider: context.driverKind, packageName: null, update: null }, + getSnapshot: Ref.get(current), + refresh, + streamChanges: Stream.fromPubSub(changes), + } satisfies ServerProviderShape; + }); + +/** + * Title and branch-name generation run on the thread's runner (whose cwd the + * request names). Commit and PR text are generated on the runner inside git + * actions, so they are never requested from a hub. + */ +function makeRemoteTextGeneration(context: InstanceContext): TextGenerationShape { + const { pool, instanceId } = context; + const toTextGenerationError = (operation: string) => (error: unknown) => + isTextGenerationError(error) + ? error + : new TextGenerationError({ + operation, + detail: isThreadMachineUnavailable(error) + ? error.message + : `Runner call failed: ${describe(error)}`, + cause: error, + }); + const onThreadRunner = ( + operation: string, + cwd: string, + f: (connection: RunnerConnection) => Effect.Effect, + ) => { + const threadId = parseThreadCheckoutPath(cwd, pool.checkoutRoot); + return threadId === null + ? Effect.fail( + new TextGenerationError({ + operation, + detail: `${cwd} is not a thread checkout, so no thread machine can generate this text.`, + }), + ) + : pool + .use(threadId, { wake: true, operation: `text.${operation}` }, f) + .pipe(Effect.mapError(toTextGenerationError(operation))); + }; + const onRunnerOnly = (operation: string) => + Effect.fail( + new TextGenerationError({ + operation, + detail: "Runs on the thread's runner as part of git actions, never on the hub.", + }), + ); + return { + generateThreadTitle: (request) => + attachmentFilesForRunner(context.attachmentsDir, request.attachments).pipe( + Effect.mapError(toTextGenerationError("generateThreadTitle")), + Effect.flatMap((attachments) => + onThreadRunner("generateThreadTitle", request.cwd, (connection) => + connection.client["runner.text.generateThreadTitle"]({ + instanceId, + cwd: request.cwd, + message: request.message, + attachments, + modelSelection: request.modelSelection, + ...(request.previousTitle !== undefined + ? { previousTitle: request.previousTitle } + : {}), + }), + ), + ), + ), + generateBranchName: (request) => + attachmentFilesForRunner(context.attachmentsDir, request.attachments).pipe( + Effect.mapError(toTextGenerationError("generateBranchName")), + Effect.flatMap((attachments) => + onThreadRunner("generateBranchName", request.cwd, (connection) => + connection.client["runner.text.generateBranchName"]({ + instanceId, + cwd: request.cwd, + message: request.message, + attachments, + modelSelection: request.modelSelection, + }), + ), + ), + ), + generateCommitMessage: () => onRunnerOnly("generateCommitMessage"), + generatePrContent: () => onRunnerOnly("generatePrContent"), + generateStructured: () => onRunnerOnly("generateStructured"), + }; +} diff --git a/apps/server/src/hub/RemoteSessionRegistry.ts b/apps/server/src/hub/RemoteSessionRegistry.ts new file mode 100644 index 000000000000..6969e37350f9 --- /dev/null +++ b/apps/server/src/hub/RemoteSessionRegistry.ts @@ -0,0 +1,194 @@ +/** + * RemoteSessionRegistry - the hub's answer to "does this thread have a + * provider session?" without asking (or waking) a machine. + * + * A session started on a thread's runner stays live on that machine while it + * is paused or saved; the hub treats it as resumable. The registry records + * every remote session the hub started, the runner boot that hosts it, and + * its active turn, updated from the delivered event stream. After a hub + * restart it is seeded from the persisted session directory rows that were + * still active, with the boot the hub last reconciled. + * + * A session disappears only when its runner reports it exited, the hub stops + * it, or boot reconciliation finds the runner restarted without it. + * + * @module hub/RemoteSessionRegistry + */ +import { + type ProviderDriverKind, + type ProviderInstanceId, + type ProviderRuntimeEvent, + type ProviderSession, + type RuntimeMode, + type ThreadId, + TurnId, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; + +import { RunnerCursorStore } from "../persistence/Services/HubThreadMachineState.ts"; +import { + ProviderSessionRuntimeRepository, + type ProviderSessionRuntime, +} from "../persistence/ProviderSessionRuntime.ts"; + +export interface RemoteSessionRecord { + readonly threadId: ThreadId; + readonly instanceId: ProviderInstanceId; + readonly provider: ProviderDriverKind; + readonly session: ProviderSession; + /** Runner boot hosting the session; null when unknown (seeded without a cursor). */ + readonly bootId: string | null; +} + +export interface RemoteSessionRegistryShape { + readonly get: (threadId: ThreadId) => Effect.Effect>; + readonly listForInstance: ( + instanceId: ProviderInstanceId, + ) => Effect.Effect>; + readonly list: Effect.Effect>; + readonly upsert: (record: RemoteSessionRecord) => Effect.Effect; + readonly remove: (threadId: ThreadId) => Effect.Effect; + /** Tracks status and active turn from a delivered runtime event. */ + readonly applyEvent: (event: ProviderRuntimeEvent) => Effect.Effect; + /** Marks every session of a thread as hosted by `bootId`. */ + readonly adoptBoot: (threadId: ThreadId, bootId: string) => Effect.Effect; +} + +export class RemoteSessionRegistry extends Context.Service< + RemoteSessionRegistry, + RemoteSessionRegistryShape +>()("t3/hub/RemoteSessionRegistry") {} + +const readPayloadString = (payload: unknown, key: string): string | undefined => { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return undefined; + const value = (payload as Record)[key]; + return typeof value === "string" && value.trim().length > 0 ? value : undefined; +}; + +/** Rebuilds the live-session view of a persisted, still-active binding. */ +export const sessionFromRuntimeRow = (row: ProviderSessionRuntime): ProviderSession | null => { + if (row.providerInstanceId === null) return null; + if (row.status !== "starting" && row.status !== "running") return null; + const cwd = readPayloadString(row.runtimePayload, "cwd"); + const model = readPayloadString(row.runtimePayload, "model"); + const activeTurnId = readPayloadString(row.runtimePayload, "activeTurnId"); + return { + provider: row.providerName as ProviderDriverKind, + providerInstanceId: row.providerInstanceId, + status: activeTurnId ? "running" : row.status === "starting" ? "connecting" : "ready", + runtimeMode: row.runtimeMode as RuntimeMode, + ...(cwd ? { cwd } : {}), + ...(model ? { model } : {}), + threadId: row.threadId, + ...(row.resumeCursor !== null ? { resumeCursor: row.resumeCursor } : {}), + ...(activeTurnId ? { activeTurnId: TurnId.make(activeTurnId) } : {}), + createdAt: row.lastSeenAt, + updatedAt: row.lastSeenAt, + }; +}; + +export const make = Effect.gen(function* () { + const runtimeRows = yield* ProviderSessionRuntimeRepository; + const cursors = yield* RunnerCursorStore; + const records = new Map(); + + const seed = yield* Effect.cached( + Effect.gen(function* () { + const rows = yield* runtimeRows.list(); + const bootByThread = new Map( + (yield* cursors.list()).map((cursor) => [cursor.threadId, cursor.bootId] as const), + ); + for (const row of rows) { + const session = sessionFromRuntimeRow(row); + if (!session || !session.providerInstanceId || records.has(row.threadId)) continue; + records.set(row.threadId, { + threadId: row.threadId, + instanceId: session.providerInstanceId, + provider: session.provider, + session, + bootId: bootByThread.get(row.threadId) ?? null, + }); + } + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("remote session registry could not read persisted sessions", { + cause: String(cause).slice(0, 400), + }), + ), + ), + ); + + const update = (threadId: ThreadId, f: (record: RemoteSessionRecord) => RemoteSessionRecord) => + Effect.sync(() => { + const current = records.get(threadId); + if (current) records.set(threadId, f(current)); + }); + + return RemoteSessionRegistry.of({ + get: (threadId) => seed.pipe(Effect.map(() => Option.fromNullishOr(records.get(threadId)))), + listForInstance: (instanceId) => + seed.pipe( + Effect.map(() => + [...records.values()].filter((record) => record.instanceId === instanceId), + ), + ), + list: seed.pipe(Effect.map(() => [...records.values()])), + upsert: (record) => + seed.pipe( + Effect.andThen( + Effect.sync(() => { + records.set(record.threadId, record); + }), + ), + ), + remove: (threadId) => + seed.pipe( + Effect.andThen( + Effect.sync(() => { + records.delete(threadId); + }), + ), + ), + applyEvent: (event) => + seed.pipe( + Effect.andThen(() => { + switch (event.type) { + case "session.exited": + return Effect.sync(() => { + const current = records.get(event.threadId); + if (current && current.instanceId === event.providerInstanceId) { + records.delete(event.threadId); + } + }); + case "turn.started": + return update(event.threadId, (record) => ({ + ...record, + session: { + ...record.session, + status: "running", + ...(event.turnId ? { activeTurnId: event.turnId } : {}), + updatedAt: event.createdAt, + }, + })); + case "turn.completed": + case "turn.aborted": + return update(event.threadId, (record) => { + const { activeTurnId: _activeTurnId, ...session } = record.session; + return { + ...record, + session: { ...session, status: "ready", updatedAt: event.createdAt }, + }; + }); + default: + return Effect.void; + } + }), + ), + adoptBoot: (threadId, bootId) => update(threadId, (record) => ({ ...record, bootId })), + }); +}); + +export const layer = Layer.effect(RemoteSessionRegistry, make); diff --git a/apps/server/src/hub/RunnerConnectionPool.ts b/apps/server/src/hub/RunnerConnectionPool.ts new file mode 100644 index 000000000000..5c47075652c6 --- /dev/null +++ b/apps/server/src/hub/RunnerConnectionPool.ts @@ -0,0 +1,717 @@ +/** + * RunnerConnectionPool - the hub's connections to thread machines. + * + * One connection per thread, opened on demand through the machine directory: + * + * - `use`/`stream` with `wake: true` resume or create the thread's machine, + * wait (bounded) until it is running, then connect. With `wake: false` a + * call only uses a machine that is already running; a paused, saved or + * missing machine fails with `ThreadMachineUnavailableError { reason: + * "asleep" }`, which read paths turn into cached results or a typed + * "machine asleep" answer. Which calls wake is decided by the callers and + * documented in docs/internals/thread-machines.md. + * - The handshake negotiates the protocol version and proves the runner + * serves this thread. A new `bootId` is reported to connection handlers, + * which reconcile sessions that were active on the previous boot. + * - Connection handlers (event delivery, git status, terminals) run in the + * connection's scope for as long as it is open. + * - A connection is closed after `idleTimeout` without calls, open streams, + * or busy marks (a running turn), and the directory is told the machine is + * idle. An unexpected disconnect of a busy thread reconnects with backoff, + * without waking the machine. + * + * @module hub/RunnerConnectionPool + */ +import { type ProjectId, type ThreadId } from "@t3tools/contracts"; +import { + RUNNER_MIN_PROTOCOL_VERSION, + RUNNER_PROTOCOL_VERSION, + RUNNER_WS_PATH, + type RunnerHello, + RunnerRpcGroup, + THREAD_CHECKOUT_ROOT, + type ThreadMachineEnsureRequest, + type ThreadMachineRepository, + type ThreadMachineStatus, + ThreadMachineUnavailableError, + threadCheckoutPath, +} from "@t3tools/contracts/runner"; +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; +import * as Schedule from "effect/Schedule"; +import * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import { RpcClient, RpcSerialization } from "effect/unstable/rpc"; +import type { RpcClientError } from "effect/unstable/rpc/RpcClientError"; +import * as Socket from "effect/unstable/socket/Socket"; + +import { ServerConfig } from "../config.ts"; +import { MachineDirectory, type MachineDirectoryError } from "./MachineDirectory.ts"; + +export type RunnerRpcClient = RpcClient.FromGroup; + +export interface RunnerConnection { + readonly threadId: ThreadId; + readonly client: RunnerRpcClient; + readonly hello: RunnerHello; + /** Resolves when the socket closes for any reason. */ + readonly closed: Effect.Effect; +} + +/** What the directory needs to create a machine: from the thread and project. */ +export interface ThreadMachineContext { + readonly projectId: ProjectId | null; + readonly repository: ThreadMachineRepository | null; + readonly branch: string | null; +} + +export type ThreadMachineContextResolver = ( + threadId: ThreadId, +) => Effect.Effect; + +export interface RunnerCallOptions { + /** Resume or create the machine when it is not running. */ + readonly wake: boolean; + /** Names the call in errors and logs. */ + readonly operation: string; + /** Richer ensure context than the registered resolver provides. */ + readonly context?: ThreadMachineContext; + /** Progress while waking (directory states before `running`). */ + readonly onWakeProgress?: (status: ThreadMachineStatus) => Effect.Effect; +} + +export type ConnectionHandler = ( + connection: RunnerConnection, +) => Effect.Effect; + +export type PoolLifecycleEvent = + | { readonly _tag: "connected"; readonly connection: RunnerConnection } + | { readonly _tag: "disconnected"; readonly threadId: ThreadId; readonly expected: boolean } + /** The directory reports the machine failed or no longer exists. */ + | { readonly _tag: "lost"; readonly threadId: ThreadId; readonly status: ThreadMachineStatus }; + +export interface RunnerConnectionPoolShape { + readonly checkoutRoot: string; + readonly checkoutFor: (threadId: ThreadId) => string; + readonly use: ( + threadId: ThreadId, + options: RunnerCallOptions, + f: (connection: RunnerConnection) => Effect.Effect, + ) => Effect.Effect; + /** Like `use`; the connection counts as active until the stream ends. */ + readonly stream: ( + threadId: ThreadId, + options: RunnerCallOptions, + f: (connection: RunnerConnection) => Stream.Stream, + ) => Stream.Stream; + /** The open connection, if any. Never connects or wakes. */ + readonly current: (threadId: ThreadId) => Effect.Effect>; + readonly connections: Effect.Effect>; + /** Directory state without waking. */ + readonly machineStatus: ( + threadId: ThreadId, + ) => Effect.Effect; + /** Runs `handler` in the scope of every current and future connection. */ + readonly onConnection: (handler: ConnectionHandler) => Effect.Effect; + readonly lifecycle: Stream.Stream; + /** Busy marks keep a connection open (and reconnecting) while set. */ + readonly setBusy: (threadId: ThreadId, reason: string, busy: boolean) => Effect.Effect; + /** Records activity so the idle timer restarts. */ + readonly touch: (threadId: ThreadId) => Effect.Effect; + readonly setContextResolver: (resolver: ThreadMachineContextResolver) => Effect.Effect; + /** Closes the connection and retires the machine. */ + readonly release: (threadId: ThreadId) => Effect.Effect; +} + +export class RunnerConnectionPool extends Context.Service< + RunnerConnectionPool, + RunnerConnectionPoolShape +>()("t3/hub/RunnerConnectionPool") {} + +export interface RunnerConnectionPoolOptions { + readonly checkoutRoot?: string; + readonly wakeTimeout?: Duration.Input; + readonly wakePollInterval?: Duration.Input; + readonly idleTimeout?: Duration.Input; + readonly idleCheckInterval?: Duration.Input; + readonly connectTimeout?: Duration.Input; + readonly reconnectMaxDelay?: Duration.Input; +} + +const DEFAULTS = { + wakeTimeout: Duration.minutes(5), + wakePollInterval: Duration.seconds(2), + idleTimeout: Duration.minutes(10), + idleCheckInterval: Duration.seconds(30), + connectTimeout: Duration.seconds(15), + reconnectMaxDelay: Duration.seconds(15), +} as const; + +const COMING_UP: ReadonlySet = new Set(["preparing", "starting"]); + +interface Slot { + attempt: Deferred.Deferred; + wake: boolean; + connection: RunnerConnection | null; + scope: Scope.Closeable | null; + closing: boolean; +} + +interface Activity { + inFlight: number; + lastActivityAt: number; + readonly busy: Set; +} + +const describe = (cause: unknown): string => + cause && typeof cause === "object" && "message" in cause + ? String((cause as { readonly message: unknown }).message) + : String(cause); + +/** A WebSocket constructor that presents the runner token as a bearer header. */ +const bearerWebSocketConstructor = (token: string) => + Layer.succeed(Socket.WebSocketConstructor)( + (url, protocols) => + // Node's and Bun's WebSocket accept an init object with headers. + new globalThis.WebSocket(url, { + ...(protocols !== undefined ? { protocols } : {}), + headers: { authorization: `Bearer ${token}` }, + } as unknown as string[]), + ); + +const runnerSocketUrl = (url: string) => { + const parsed = new URL(url); + if (parsed.pathname === "/" || parsed.pathname === "") parsed.pathname = RUNNER_WS_PATH; + return parsed.toString(); +}; + +export const make = (options: RunnerConnectionPoolOptions = {}) => + Effect.gen(function* () { + const directory = yield* MachineDirectory; + const poolScope = yield* Effect.scope; + const checkoutRoot = options.checkoutRoot ?? THREAD_CHECKOUT_ROOT; + const wakeTimeout = Duration.fromInputUnsafe(options.wakeTimeout ?? DEFAULTS.wakeTimeout); + const wakePollInterval = Duration.fromInputUnsafe( + options.wakePollInterval ?? DEFAULTS.wakePollInterval, + ); + const idleTimeout = Duration.fromInputUnsafe(options.idleTimeout ?? DEFAULTS.idleTimeout); + const idleCheckInterval = Duration.fromInputUnsafe( + options.idleCheckInterval ?? DEFAULTS.idleCheckInterval, + ); + const connectTimeout = Duration.fromInputUnsafe( + options.connectTimeout ?? DEFAULTS.connectTimeout, + ); + const reconnectMaxDelay = Duration.fromInputUnsafe( + options.reconnectMaxDelay ?? DEFAULTS.reconnectMaxDelay, + ); + + const slots = new Map(); + const activity = new Map(); + const handlers: Array = []; + const lifecycle = yield* PubSub.unbounded(); + let contextResolver: ThreadMachineContextResolver | null = null; + + const checkoutFor = (threadId: ThreadId) => threadCheckoutPath(threadId, checkoutRoot); + const activityOf = (threadId: ThreadId, now: number): Activity => { + let entry = activity.get(threadId); + if (!entry) { + entry = { inFlight: 0, lastActivityAt: now, busy: new Set() }; + activity.set(threadId, entry); + } + return entry; + }; + + const unavailable = ( + threadId: ThreadId, + operation: string, + reason: ThreadMachineUnavailableError["reason"], + detail: string, + state?: ThreadMachineStatus["state"], + ) => + new ThreadMachineUnavailableError({ + threadId, + reason, + operation, + detail, + ...(state !== undefined ? { state } : {}), + }); + const fromDirectoryError = + (threadId: ThreadId, operation: string) => (error: MachineDirectoryError) => + unavailable(threadId, operation, "directory", error.message); + + const ensureRequest = ( + threadId: ThreadId, + context: ThreadMachineContext | undefined, + ): Effect.Effect => + Effect.gen(function* () { + const resolved = + context ?? + (contextResolver + ? yield* contextResolver(threadId).pipe( + Effect.catchCause((cause) => + Effect.logWarning("thread machine context lookup failed", { + threadId, + cause: Cause.pretty(cause), + }).pipe(Effect.as(undefined)), + ), + ) + : undefined); + return { + projectId: resolved?.projectId ?? null, + repository: resolved?.repository ?? null, + branch: resolved?.branch ?? null, + checkout: checkoutFor(threadId), + wake: true, + }; + }); + + /** Resolves a running machine through the directory, waking it when asked. */ + const resolveRunning = (threadId: ThreadId, callOptions: RunnerCallOptions) => + Effect.gen(function* () { + const { operation } = callOptions; + let status: ThreadMachineStatus = callOptions.wake + ? yield* directory + .ensure(threadId, yield* ensureRequest(threadId, callOptions.context)) + .pipe(Effect.mapError(fromDirectoryError(threadId, operation))) + : yield* directory + .status(threadId) + .pipe(Effect.mapError(fromDirectoryError(threadId, operation))); + if (callOptions.wake) { + const deadline = (yield* Clock.currentTimeMillis) + Duration.toMillis(wakeTimeout); + while (status.state !== "running" && status.state !== "failed") { + if (callOptions.onWakeProgress) yield* callOptions.onWakeProgress(status); + if ((yield* Clock.currentTimeMillis) >= deadline) { + return yield* unavailable( + threadId, + operation, + "wake-timeout", + `The machine did not start within ${Duration.format(wakeTimeout)} (last state ${status.state}${status.detail ? `: ${status.detail}` : ""}).`, + status.state, + ); + } + yield* Effect.sleep(wakePollInterval); + status = yield* directory + .status(threadId) + .pipe(Effect.mapError(fromDirectoryError(threadId, operation))); + } + } + switch (status.state) { + case "running": + if (!status.runner) { + return yield* unavailable( + threadId, + operation, + "unreachable", + "The directory reports the machine running without a runner endpoint.", + status.state, + ); + } + return { status, runner: status.runner }; + case "failed": + yield* PubSub.publish(lifecycle, { _tag: "lost", threadId, status }); + return yield* unavailable( + threadId, + operation, + "failed", + status.detail ?? "The machine failed.", + status.state, + ); + case "none": + yield* PubSub.publish(lifecycle, { _tag: "lost", threadId, status }); + return yield* unavailable( + threadId, + operation, + "asleep", + "The thread has no machine.", + status.state, + ); + default: + return yield* unavailable( + threadId, + operation, + "asleep", + COMING_UP.has(status.state) ? "The machine is starting." : "The machine is asleep.", + status.state, + ); + } + }); + + /** Opens the socket, runs the handshake, and starts connection handlers. */ + const open = ( + threadId: ThreadId, + runner: { readonly url: string; readonly token: string }, + operation: string, + scope: Scope.Closeable, + ) => + Effect.gen(function* () { + const closed = yield* Deferred.make(); + const hooks = RpcClient.ConnectionHooks.of({ + onConnect: Effect.void, + onDisconnect: Deferred.succeed(closed, undefined).pipe(Effect.asVoid), + }); + const protocolContext = yield* Layer.buildWithScope( + Layer.effect( + RpcClient.Protocol, + RpcClient.makeProtocolSocket({ retryTransientErrors: false }), + ).pipe( + Layer.provide( + Layer.mergeAll( + Socket.layerWebSocket(runnerSocketUrl(runner.url), { + openTimeout: connectTimeout, + }).pipe(Layer.provide(bearerWebSocketConstructor(runner.token))), + RpcSerialization.layerJson, + Layer.succeed(RpcClient.ConnectionHooks, hooks), + ), + ), + ), + scope, + ); + const client = yield* RpcClient.make(RunnerRpcGroup, { disableTracing: true }).pipe( + Effect.provide(protocolContext), + Scope.provide(scope), + ); + const hello = yield* client["runner.hello"]({ + protocolVersion: RUNNER_PROTOCOL_VERSION, + minProtocolVersion: RUNNER_MIN_PROTOCOL_VERSION, + threadId, + }).pipe( + Effect.timeoutOrElse({ + duration: connectTimeout, + orElse: () => + Effect.fail( + unavailable( + threadId, + operation, + "unreachable", + `The runner did not answer within ${Duration.format(connectTimeout)}.`, + "running", + ), + ), + }), + Effect.mapError((error) => { + switch (error._tag) { + case "ThreadMachineUnavailableError": + return error; + case "RunnerProtocolMismatchError": + case "RunnerThreadMismatchError": + return unavailable(threadId, operation, "incompatible", error.message, "running"); + default: + return unavailable( + threadId, + operation, + "unreachable", + `The runner connection failed: ${describe(error)}`, + "running", + ); + } + }), + ); + return { + threadId, + client, + hello, + closed: Deferred.await(closed), + } satisfies RunnerConnection; + }); + + const runHandlers = (connection: RunnerConnection, scope: Scope.Closeable) => + Effect.forEach( + [...handlers], + (handler) => + handler(connection).pipe( + Scope.provide(scope), + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.void + : Effect.logWarning("runner connection handler failed", { + threadId: connection.threadId, + cause: Cause.pretty(cause), + }), + ), + Effect.forkIn(scope), + ), + { discard: true }, + ); + + const closeSlot = (threadId: ThreadId, slot: Slot, expected: boolean) => + Effect.gen(function* () { + if (slot.closing) return; + slot.closing = true; + if (slots.get(threadId) === slot) slots.delete(threadId); + if (slot.scope) yield* Scope.close(slot.scope, Exit.void); + if (slot.connection) { + yield* PubSub.publish(lifecycle, { _tag: "disconnected", threadId, expected }); + } + }); + + /** Reconnects a busy thread after an unexpected disconnect, never waking it. */ + const reconnectWhileBusy = (threadId: ThreadId) => + Effect.gen(function* () { + const busy = () => (activity.get(threadId)?.busy.size ?? 0) > 0; + if (!busy()) return; + yield* acquire(threadId, { wake: false, operation: "reconnect" }).pipe( + Effect.retry({ + while: (error) => + busy() && (error.reason === "unreachable" || error.reason === "directory"), + schedule: Schedule.exponential("250 millis").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, reconnectMaxDelay)), + ), + ), + }), + Effect.tap(() => Effect.logInfo("runner reconnected", { threadId })), + Effect.catch((error) => + Effect.logWarning("runner reconnect stopped", { + threadId, + reason: error.reason, + detail: error.detail, + }), + ), + ); + }); + + const connectAttempt = (threadId: ThreadId, slot: Slot, callOptions: RunnerCallOptions) => + Effect.gen(function* () { + const { runner } = yield* resolveRunning(threadId, callOptions); + const scope = yield* Scope.make(); + slot.scope = scope; + const connection = yield* open(threadId, runner, callOptions.operation, scope).pipe( + Effect.onError(() => Scope.close(scope, Exit.void)), + ); + slot.connection = connection; + const now = yield* Clock.currentTimeMillis; + activityOf(threadId, now).lastActivityAt = now; + yield* Effect.logInfo("runner connected", { + threadId, + runnerId: connection.hello.runnerId, + bootId: connection.hello.bootId, + protocolVersion: connection.hello.protocolVersion, + headSequence: connection.hello.headSequence, + ackedSequence: connection.hello.ackedSequence, + }); + yield* PubSub.publish(lifecycle, { _tag: "connected", connection }); + yield* runHandlers(connection, scope); + // Watch for the socket closing; unexpected closes of busy threads reconnect. + yield* connection.closed.pipe( + Effect.andThen( + Effect.suspend(() => { + const expected = slot.closing; + return closeSlot(threadId, slot, expected).pipe( + Effect.andThen( + expected + ? Effect.void + : Effect.logWarning("runner connection lost", { threadId }).pipe( + Effect.andThen(reconnectWhileBusy(threadId)), + ), + ), + ); + }), + ), + Effect.forkIn(poolScope), + ); + return connection; + }); + + const acquire = ( + threadId: ThreadId, + callOptions: RunnerCallOptions, + ): Effect.Effect => + Effect.suspend(() => { + const existing = slots.get(threadId); + if (existing && !existing.closing) { + if (existing.connection) return Effect.succeed(existing.connection); + const waitExisting = Deferred.await(existing.attempt); + // A read-only attempt in flight cannot satisfy a caller that must wake. + return callOptions.wake && !existing.wake + ? waitExisting.pipe( + Effect.catchIf( + (error) => error.reason === "asleep", + () => acquire(threadId, callOptions), + ), + ) + : waitExisting; + } + return Effect.gen(function* () { + const attempt = yield* Deferred.make(); + const slot: Slot = { + attempt, + wake: callOptions.wake, + connection: null, + scope: null, + closing: false, + }; + slots.set(threadId, slot); + yield* connectAttempt(threadId, slot, callOptions).pipe( + Effect.exit, + Effect.flatMap((exit) => { + if (Exit.isFailure(exit) && slots.get(threadId) === slot) slots.delete(threadId); + return Deferred.done(attempt, exit); + }), + // The attempt outlives any single caller being interrupted. + Effect.forkIn(poolScope), + ); + return yield* Deferred.await(attempt); + }); + }); + + const track = (threadId: ThreadId, effect: Effect.Effect) => + Effect.acquireUseRelease( + Clock.currentTimeMillis.pipe( + Effect.map((now) => { + const entry = activityOf(threadId, now); + entry.inFlight += 1; + entry.lastActivityAt = now; + }), + ), + () => effect, + () => + Clock.currentTimeMillis.pipe( + Effect.map((now) => { + const entry = activityOf(threadId, now); + entry.inFlight = Math.max(0, entry.inFlight - 1); + entry.lastActivityAt = now; + }), + ), + ); + + const use: RunnerConnectionPoolShape["use"] = (threadId, callOptions, f) => + track(threadId, acquire(threadId, callOptions).pipe(Effect.flatMap(f))); + + const stream: RunnerConnectionPoolShape["stream"] = (threadId, callOptions, f) => + Stream.unwrap( + Effect.gen(function* () { + const connection = yield* acquire(threadId, callOptions); + const now = yield* Clock.currentTimeMillis; + const entry = activityOf(threadId, now); + entry.inFlight += 1; + entry.lastActivityAt = now; + return f(connection).pipe( + Stream.ensuring( + Clock.currentTimeMillis.pipe( + Effect.map((end) => { + entry.inFlight = Math.max(0, entry.inFlight - 1); + entry.lastActivityAt = end; + }), + ), + ), + ); + }), + ); + + // Idle connections close and tell the platform the machine may sleep. + yield* Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + for (const [threadId, slot] of slots) { + if (!slot.connection || slot.closing) continue; + const entry = activityOf(threadId, now); + if (entry.inFlight > 0 || entry.busy.size > 0) continue; + if (now - entry.lastActivityAt < Duration.toMillis(idleTimeout)) continue; + yield* Effect.logInfo("closing idle runner connection", { threadId }); + yield* closeSlot(threadId, slot, true); + yield* directory.idle(threadId).pipe( + Effect.catch((error) => + Effect.logWarning("machine directory idle notification failed", { + threadId, + detail: error.message, + }), + ), + ); + } + }).pipe(Effect.delay(idleCheckInterval), Effect.forever, Effect.forkIn(poolScope)); + + yield* Effect.addFinalizer(() => + Effect.forEach([...slots], ([threadId, slot]) => closeSlot(threadId, slot, true), { + discard: true, + }), + ); + + return RunnerConnectionPool.of({ + checkoutRoot, + checkoutFor, + use, + stream, + current: (threadId) => + Effect.sync(() => { + const slot = slots.get(threadId); + return slot?.connection && !slot.closing + ? Option.some(slot.connection) + : Option.none(); + }), + connections: Effect.sync(() => + [...slots.values()].flatMap((slot) => + slot.connection && !slot.closing ? [slot.connection] : [], + ), + ), + machineStatus: (threadId) => + directory.status(threadId).pipe(Effect.mapError(fromDirectoryError(threadId, "status"))), + onConnection: (handler) => + Effect.gen(function* () { + handlers.push(handler); + for (const slot of slots.values()) { + if (slot.connection && slot.scope && !slot.closing) { + yield* handler(slot.connection).pipe( + Scope.provide(slot.scope), + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.void + : Effect.logWarning("runner connection handler failed", { + cause: Cause.pretty(cause), + }), + ), + Effect.forkIn(slot.scope), + ); + } + } + }), + lifecycle: Stream.fromPubSub(lifecycle), + setBusy: (threadId, reason, busy) => + Clock.currentTimeMillis.pipe( + Effect.map((now) => { + const entry = activityOf(threadId, now); + if (busy) entry.busy.add(reason); + else entry.busy.delete(reason); + entry.lastActivityAt = now; + }), + ), + touch: (threadId) => + Clock.currentTimeMillis.pipe( + Effect.map((now) => { + activityOf(threadId, now).lastActivityAt = now; + }), + ), + setContextResolver: (resolver) => + Effect.sync(() => { + contextResolver = resolver; + }), + release: (threadId) => + Effect.gen(function* () { + const slot = slots.get(threadId); + if (slot) yield* closeSlot(threadId, slot, true); + activity.delete(threadId); + yield* directory.release(threadId).pipe( + Effect.catch((error) => + Effect.logWarning("machine directory release failed", { + threadId, + detail: error.message, + }), + ), + ); + }), + }); + }); + +export const layer = Layer.effect( + RunnerConnectionPool, + Effect.gen(function* () { + const config = yield* ServerConfig; + return yield* make(config.hub?.checkoutRoot ? { checkoutRoot: config.hub.checkoutRoot } : {}); + }), +); diff --git a/apps/server/src/hub/RunnerEventDelivery.ts b/apps/server/src/hub/RunnerEventDelivery.ts new file mode 100644 index 000000000000..97611d122456 --- /dev/null +++ b/apps/server/src/hub/RunnerEventDelivery.ts @@ -0,0 +1,404 @@ +/** + * RunnerEventDelivery - provider runtime events from thread runners into the + * hub, exactly once in effect. + * + * For every runner connection the hub subscribes to the runner outbox from + * its cursor, drops any `(thread, sequence)` it already delivered, and + * publishes the events to the remote provider adapters, which feed + * `ProviderService` and ingestion unchanged. + * + * Durability contract: + * - The persisted cursor (`RunnerCursorStore`) only advances to a *safe + * point*: a sequence at which the thread had no open turn. Ingestion + * buffers assistant text and plans in memory while a turn runs, so a cursor + * inside a turn could lose buffered text on a hub crash. After a crash the + * runner replays the open turn from its start. + * - Replays are idempotent: in hub mode ingestion derives its command ids + * from the runner event id, so commands that already committed are + * answered from their receipts instead of being applied twice. + * - A cursor is written only after ingestion has drained and the safe point + * has aged `flush`'s `minAge`, then acknowledged to the runner so it can + * compact its outbox. Acknowledgement is a notification, not a correctness + * step. + * + * Boot reconciliation: when a runner's `bootId` differs from the boot the hub + * last saw, sessions from the previous boot are gone. Once the replayed + * backlog is delivered, the hub asks the runner which sessions it still + * hosts and settles the rest with an interrupted `turn.completed` and a + * `session.exited`, with deterministic event ids. The same happens when the + * directory reports a thread's machine failed or no longer exists. + * + * @module hub/RunnerEventDelivery + */ +import { EventId, type ProviderRuntimeEvent, type ThreadId } from "@t3tools/contracts"; +import type { ThreadMachineStatus } from "@t3tools/contracts/runner"; +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; +import * as Stream from "effect/Stream"; + +import { RunnerCursorStore } from "../persistence/Services/HubThreadMachineState.ts"; +import { RemoteSessionRegistry, type RemoteSessionRecord } from "./RemoteSessionRegistry.ts"; +import { RunnerConnectionPool, type RunnerConnection } from "./RunnerConnectionPool.ts"; + +export interface ThreadDeliveryState { + readonly outboxId: string | null; + readonly bootId: string | null; + /** Highest runner sequence published to the hub. */ + readonly delivered: number; + /** Highest sequence at which the thread had no open turn. */ + readonly safe: number; + /** Highest sequence persisted in the cursor store. */ + readonly durable: number; + readonly duplicates: number; + readonly openTurnSince: number | null; +} + +export interface RunnerEventDeliveryShape { + /** Delivered and reconciliation events, in runner order per thread. */ + readonly events: Stream.Stream; + /** Opens delivery. Subscriptions wait for it so ingestion listens first. */ + readonly start: Effect.Effect; + /** + * Persists safe points older than `minAgeMs` once `drain` has returned, + * then acknowledges them to connected runners. + */ + readonly flush: (input: { + readonly minAgeMs: number; + readonly drain: Effect.Effect; + }) => Effect.Effect; + readonly state: Effect.Effect>; + /** + * Waits until boot reconciliation for `connection` has run, so callers see + * whether sessions survived a runner restart. Bounded; delivery that never + * starts does not block calls forever. + */ + readonly awaitReconciled: (connection: RunnerConnection) => Effect.Effect; +} + +export class RunnerEventDelivery extends Context.Service< + RunnerEventDelivery, + RunnerEventDeliveryShape +>()("t3/hub/RunnerEventDelivery") {} + +interface MutableThreadState { + outboxId: string | null; + bootId: string | null; + persistedBootId: string | null; + delivered: number; + safe: number; + safeReachedAt: number; + durable: number; + duplicates: number; + openTurnSince: number | null; +} + +const closesTurn = (event: ProviderRuntimeEvent) => + event.type === "turn.completed" || + event.type === "turn.aborted" || + event.type === "session.exited"; + +const MACHINE_RESTARTED = "The thread machine restarted; the turn was interrupted."; +const RECONCILE_WAIT = "30 seconds"; + +export const make = Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + const cursors = yield* RunnerCursorStore; + const registry = yield* RemoteSessionRegistry; + const bus = yield* PubSub.unbounded(); + const enabled = yield* Deferred.make(); + const states = new Map(); + const reconciledByConnection = new WeakMap>(); + const reconciledSignal = (connection: RunnerConnection) => + Effect.gen(function* () { + const existing = reconciledByConnection.get(connection); + if (existing) return existing; + const created = yield* Deferred.make(); + reconciledByConnection.set(connection, created); + return created; + }); + + const stateOf = (threadId: ThreadId) => { + let state = states.get(threadId); + if (!state) { + state = { + outboxId: null, + bootId: null, + persistedBootId: null, + delivered: 0, + safe: 0, + safeReachedAt: 0, + durable: 0, + duplicates: 0, + openTurnSince: null, + }; + states.set(threadId, state); + } + return state; + }; + + /** Publishes one event and advances turn tracking; `sequence` is null for synthesized events. */ + const publish = (threadId: ThreadId, sequence: number | null, event: ProviderRuntimeEvent) => + Effect.gen(function* () { + const state = stateOf(threadId); + const now = yield* Clock.currentTimeMillis; + if (sequence !== null) { + state.delivered = sequence; + if (event.type === "turn.started" && state.openTurnSince === null) { + state.openTurnSince = sequence; + } + } + if (closesTurn(event)) state.openTurnSince = null; + if (state.openTurnSince === null && state.safe !== state.delivered) { + state.safe = state.delivered; + state.safeReachedAt = now; + } + yield* registry.applyEvent(event); + yield* PubSub.publish(bus, event); + yield* pool.setBusy(threadId, "turn", state.openTurnSince !== null); + }); + + /** Settles a session its runner no longer hosts. */ + const settle = (record: RemoteSessionRecord, reason: string, key: string) => + Effect.gen(function* () { + const createdAt = DateTime.formatIso(yield* DateTime.now); + const base = { + provider: record.provider, + providerInstanceId: record.instanceId, + threadId: record.threadId, + createdAt, + } as const; + yield* Effect.logWarning("settling a thread session its runner no longer hosts", { + threadId: record.threadId, + instanceId: record.instanceId, + activeTurnId: record.session.activeTurnId, + reason, + }); + if (record.session.activeTurnId !== undefined) { + yield* publish(record.threadId, null, { + ...base, + type: "turn.completed", + eventId: EventId.make(`hub-reconcile:${record.threadId}:${key}:turn-completed`), + turnId: record.session.activeTurnId, + payload: { state: "interrupted", errorMessage: reason }, + }); + } + yield* publish(record.threadId, null, { + ...base, + type: "session.exited", + eventId: EventId.make(`hub-reconcile:${record.threadId}:${key}:session-exited`), + payload: { reason, recoverable: true, exitKind: "error" }, + }); + yield* registry.remove(record.threadId); + }); + + /** After a boot change: keep sessions the runner still hosts, settle the rest. */ + const reconcileBoot = (connection: RunnerConnection) => + Effect.gen(function* () { + const { threadId, hello, client } = connection; + const record = Option.getOrUndefined(yield* registry.get(threadId)); + if (!record || record.bootId === hello.bootId) return; + const live = hello.instances.includes(record.instanceId) + ? yield* client["runner.provider.listSessions"]({ instanceId: record.instanceId }).pipe( + Effect.map((sessions) => sessions.some((session) => session.threadId === threadId)), + Effect.orElseSucceed(() => false), + ) + : false; + if (live) { + yield* registry.adoptBoot(threadId, hello.bootId); + return; + } + yield* settle(record, MACHINE_RESTARTED, hello.bootId); + }); + + const deliverFrom = (connection: RunnerConnection) => + Effect.gen(function* () { + const { threadId, hello, client } = connection; + const cursor = Option.getOrUndefined( + yield* cursors.get(threadId).pipe( + Effect.catch((error) => + Effect.logWarning("runner cursor read failed; resuming from memory", { + threadId, + detail: error.message, + }).pipe(Effect.as(Option.none())), + ), + ), + ); + const state = stateOf(threadId); + if (state.outboxId !== hello.runnerId) { + // First connection this hub lifetime, or a new outbox (recreated machine). + const persisted = cursor?.outboxId === hello.runnerId ? cursor.ackedSequence : 0; + state.outboxId = hello.runnerId; + state.delivered = persisted; + state.safe = persisted; + state.durable = persisted; + state.openTurnSince = null; + state.persistedBootId = cursor?.outboxId === hello.runnerId ? cursor.bootId : null; + } + const previousBoot = state.bootId ?? cursor?.bootId ?? null; + state.bootId = hello.bootId; + if (hello.firstRetainedSequence > state.delivered + 1) { + yield* Effect.logError("runner outbox dropped undelivered events past retention", { + threadId, + delivered: state.delivered, + firstRetainedSequence: hello.firstRetainedSequence, + }); + } + + yield* Deferred.await(enabled); + + const done = yield* reconciledSignal(connection); + const record = Option.getOrUndefined(yield* registry.get(threadId)); + let reconciled = record === undefined || record.bootId === hello.bootId; + if (!reconciled) { + yield* Effect.logInfo("runner boot changed; reconciling sessions after replay", { + threadId, + previousBootId: previousBoot, + bootId: hello.bootId, + }); + } else { + yield* Deferred.succeed(done, undefined); + } + const reconcileWhenCaughtUp = (sequence: number) => + reconciled || sequence < hello.headSequence + ? Effect.void + : Effect.suspend(() => { + reconciled = true; + return reconcileBoot(connection).pipe( + Effect.ensuring(Deferred.succeed(done, undefined)), + ); + }); + + yield* reconcileWhenCaughtUp(state.delivered); + yield* client["runner.events.subscribe"]({ afterSequence: state.delivered }).pipe( + Stream.runForEach((envelope) => + Effect.gen(function* () { + if (envelope.sequence <= state.delivered) { + state.duplicates += 1; + return; + } + yield* publish(threadId, envelope.sequence, envelope.event); + yield* pool.touch(threadId); + yield* reconcileWhenCaughtUp(envelope.sequence); + }), + ), + ); + }); + + yield* pool.onConnection((connection) => + deliverFrom(connection).pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.void + : Effect.logWarning("runner event subscription ended", { + threadId: connection.threadId, + cause: Cause.pretty(cause).slice(0, 400), + }), + ), + ), + ); + + // The directory reports the machine failed or gone: nothing will replay. + yield* pool.lifecycle.pipe( + Stream.runForEach((event) => + event._tag !== "lost" + ? Effect.void + : Effect.gen(function* () { + yield* Deferred.await(enabled); + const record = Option.getOrUndefined(yield* registry.get(event.threadId)); + if (!record) return; + yield* settle(record, lostReason(event.status), `lost:${record.bootId ?? "unknown"}`); + }), + ), + Effect.forkScoped, + ); + + const flush: RunnerEventDeliveryShape["flush"] = ({ minAgeMs, drain }) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + const candidates = [...states].filter( + ([, state]) => + state.outboxId !== null && + state.bootId !== null && + (state.safe > state.durable || state.bootId !== state.persistedBootId) && + now - state.safeReachedAt >= minAgeMs, + ); + if (candidates.length === 0) return; + // Everything published up to each safe point has reached ingestion's + // queue by now; drain so its effects are committed before the cursor. + yield* drain; + const updatedAt = DateTime.formatIso(yield* DateTime.now); + const rows = candidates.map(([threadId, state]) => ({ + threadId, + outboxId: state.outboxId!, + bootId: state.bootId!, + ackedSequence: state.safe, + updatedAt, + })); + yield* cursors.saveAll(rows); + for (const row of rows) { + const state = stateOf(row.threadId); + state.durable = Math.max(state.durable, row.ackedSequence); + state.persistedBootId = row.bootId; + const connection = yield* pool.current(row.threadId); + if (Option.isSome(connection) && connection.value.hello.runnerId === row.outboxId) { + yield* connection.value.client["runner.events.ack"]({ + throughSequence: row.ackedSequence, + }).pipe(Effect.ignore); + } + } + }).pipe( + Effect.catch((error) => + Effect.logWarning("runner cursor flush failed; retried on the next flush", { + detail: error.message, + }), + ), + ); + + return RunnerEventDelivery.of({ + awaitReconciled: (connection) => + reconciledSignal(connection).pipe( + Effect.flatMap(Deferred.await), + Effect.timeoutOrElse({ + duration: RECONCILE_WAIT, + orElse: () => + Effect.logWarning("runner boot reconciliation is taking long; proceeding", { + threadId: connection.threadId, + }), + }), + ), + events: Stream.fromPubSub(bus), + start: Deferred.succeed(enabled, undefined).pipe(Effect.asVoid), + flush, + state: Effect.sync( + () => + new Map( + [...states].map(([threadId, state]) => [ + threadId, + { + outboxId: state.outboxId, + bootId: state.bootId, + delivered: state.delivered, + safe: state.safe, + durable: state.durable, + duplicates: state.duplicates, + openTurnSince: state.openTurnSince, + } satisfies ThreadDeliveryState, + ]), + ), + ), + }); +}); + +const lostReason = (status: ThreadMachineStatus) => + status.state === "failed" + ? `The thread machine failed${status.detail ? `: ${status.detail}` : ""}; the turn was interrupted.` + : "The thread machine no longer exists; the turn was interrupted."; + +export const layer = Layer.effect(RunnerEventDelivery, make); diff --git a/apps/server/src/hub/hubRouting.ts b/apps/server/src/hub/hubRouting.ts new file mode 100644 index 000000000000..ca315c016d30 --- /dev/null +++ b/apps/server/src/hub/hubRouting.ts @@ -0,0 +1,122 @@ +/** + * Routing helpers shared by the hub's checkout services. + * + * Every checkout-keyed call names a `cwd`; in hub mode a thread's checkout is + * `/`, so the cwd alone names the thread and its + * machine. A cwd that is not a thread checkout fails with a typed error; the + * hub never falls back to its own filesystem. + * + * Each hub service must fail with its own service's error type, so transport + * failures and sleeping machines are wrapped into that type with the + * original `ThreadMachineUnavailableError` (or RPC error) as the cause. + * + * @module hub/hubRouting + */ +import { + GitCommandError, + GitManagerError, + GitManagerServiceError, + type ThreadId, + VcsError, + VcsRepositoryDetectionError, +} from "@t3tools/contracts"; +import { parseThreadCheckoutPath, ThreadMachineUnavailableError } from "@t3tools/contracts/runner"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +import type { + RunnerCallOptions, + RunnerConnection, + RunnerConnectionPoolShape, + RunnerRpcClient, +} from "./RunnerConnectionPool.ts"; + +export const describeCause = (cause: unknown): string => + cause && typeof cause === "object" && "message" in cause + ? String((cause as { readonly message: unknown }).message) + : String(cause); + +export const notAThreadCheckout = (cwd: string, operation: string) => + new ThreadMachineUnavailableError({ + threadId: null, + reason: "not-a-thread-checkout", + operation, + detail: `${cwd} is not a thread checkout, and a hub never reads its own filesystem.`, + }); + +/** The thread that owns `cwd`, or a typed failure. */ +export const threadOfCwd = ( + pool: RunnerConnectionPoolShape, + cwd: string, + operation: string, +): Effect.Effect => { + const threadId = parseThreadCheckoutPath(cwd, pool.checkoutRoot); + return threadId === null + ? Effect.fail(notAThreadCheckout(cwd, operation)) + : Effect.succeed(threadId); +}; + +/** Runs `f` on the runner that owns `cwd`. */ +export const onCwdRunner = ( + pool: RunnerConnectionPoolShape, + cwd: string, + options: RunnerCallOptions, + f: (client: RunnerRpcClient, connection: RunnerConnection) => Effect.Effect, +): Effect.Effect => + threadOfCwd(pool, cwd, options.operation).pipe( + Effect.flatMap((threadId) => + pool.use(threadId, options, (connection) => f(connection.client, connection)), + ), + ); + +/** Streams from the runner that owns `cwd`. */ +export const streamOnCwdRunner = ( + pool: RunnerConnectionPoolShape, + cwd: string, + options: RunnerCallOptions, + f: (client: RunnerRpcClient) => Stream.Stream, +): Stream.Stream => + Stream.unwrap( + threadOfCwd(pool, cwd, options.operation).pipe( + Effect.map((threadId) => + pool.stream(threadId, options, (connection) => f(connection.client)), + ), + ), + ); + +const isVcsError = Schema.is(VcsError); +const isGitCommandError = Schema.is(GitCommandError); +const isGitManagerServiceError = Schema.is(GitManagerServiceError); + +export const toVcsError = + (operation: string, cwd: string) => + (error: unknown): VcsError => + isVcsError(error) + ? error + : new VcsRepositoryDetectionError({ + operation, + cwd, + detail: describeCause(error), + cause: error, + }); + +export const toGitCommandError = + (operation: string, cwd: string) => + (error: unknown): GitCommandError => + isGitCommandError(error) + ? error + : new GitCommandError({ + operation, + command: "runner", + cwd, + detail: describeCause(error), + cause: error, + }); + +export const toGitManagerServiceError = + (operation: string, cwd: string) => + (error: unknown): GitManagerServiceError => + isGitManagerServiceError(error) + ? error + : new GitManagerError({ operation, cwd, detail: describeCause(error), cause: error }); diff --git a/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts b/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts index 350dc8011144..003c2bcf428c 100644 --- a/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts +++ b/apps/server/src/provider/Layers/ProviderInstanceRegistryHydration.ts @@ -62,7 +62,6 @@ import type { AnyProviderDriver } from "../ProviderDriver.ts"; import { ProviderInstanceRegistry } from "../Services/ProviderInstanceRegistry.ts"; import { ProviderInstanceRegistryMutator } from "../Services/ProviderInstanceRegistryMutator.ts"; import { ProviderInstanceRegistryMutableLayer } from "./ProviderInstanceRegistryLive.ts"; -import { makeRemoteProviderDriver } from "../../runner/hub/RemoteProviderDriver.ts"; /** * Synthesize a `ProviderInstanceConfigMap` from a `ServerSettings` snapshot. @@ -193,15 +192,11 @@ export const ProviderInstanceRegistryHydrationLive: Layer.Layer< > = Layer.unwrap( Effect.gen(function* () { const serverConfig = yield* ServerConfig; - const drivers = resolveBuiltInDrivers({ - museCodeEnabled: serverConfig.museCodeEnabled, - }); - // Hub mode: every provider runs on the thread's runner. The hub keeps the - // same driver kinds and instance ids, but each adapter forwards over the - // runner protocol instead of spawning a provider CLI locally. - return serverConfig.serverMode === "hub" - ? makeProviderInstanceRegistryHydration(drivers.map(makeRemoteProviderDriver)) - : makeProviderInstanceRegistryHydration(drivers); + return makeProviderInstanceRegistryHydration( + resolveBuiltInDrivers({ + museCodeEnabled: serverConfig.museCodeEnabled, + }), + ); }), ) as Layer.Layer< ProviderInstanceRegistry, diff --git a/apps/server/src/runner/RunnerCheckout.ts b/apps/server/src/runner/RunnerCheckout.ts new file mode 100644 index 000000000000..61e8bf50003a --- /dev/null +++ b/apps/server/src/runner/RunnerCheckout.ts @@ -0,0 +1,173 @@ +/** + * RunnerCheckout - prepares a thread's checkout on its machine. + * + * `prepare` is idempotent and cheap when nothing changed, so the hub calls it + * at bootstrap and before every turn: + * + * 1. Missing checkout: clone the project's repository with the machine's own + * git configuration and credential helper, or `git init` a blank project. + * 2. Existing checkout: left as is (local work is never discarded). + * 3. When `branch` is not checked out: switch to it if it exists locally; + * otherwise create it from `origin/` (a branch pushed from an + * earlier machine), else from `baseRef` (remote-tracking first), else HEAD. + * + * Network access happens only when the checkout is cloned or a branch has to + * be created, so steady-state calls run two or three local git commands. + * + * @module runner/RunnerCheckout + */ +import { + RunnerCheckoutError, + type RunnerPrepareCheckoutInput, + type RunnerPrepareCheckoutResult, +} from "@t3tools/contracts/runner"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; + +import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; + +const CLONE_TIMEOUT_MS = 15 * 60_000; +const FETCH_TIMEOUT_MS = 5 * 60_000; +const DEFAULT_BLANK_BRANCH = "main"; + +export interface RunnerCheckoutShape { + readonly prepare: ( + input: RunnerPrepareCheckoutInput, + ) => Effect.Effect; +} + +export class RunnerCheckout extends Context.Service()( + "t3/runner/RunnerCheckout", +) {} + +export const make = Effect.gen(function* () { + const git = yield* GitVcsDriver.GitVcsDriver; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + + const prepare = Effect.fn("RunnerCheckout.prepare")(function* ( + input: RunnerPrepareCheckoutInput, + ) { + const checkout = input.checkout; + const fail = (operation: string) => (cause: unknown) => + new RunnerCheckoutError({ + operation, + checkout, + detail: + cause && typeof cause === "object" && "message" in cause + ? String((cause as { readonly message: unknown }).message) + : String(cause), + }); + const run = (operation: string, cwd: string, args: ReadonlyArray, timeoutMs?: number) => + git + .execute({ + operation: `RunnerCheckout.${operation}`, + cwd, + args, + ...(timeoutMs !== undefined ? { timeoutMs } : {}), + }) + .pipe( + Effect.map((result) => result.stdout.trim()), + Effect.mapError(fail(operation)), + ); + const probe = (cwd: string, args: ReadonlyArray) => + git.execute({ operation: "RunnerCheckout.probe", cwd, args, allowNonZeroExit: true }).pipe( + Effect.map((result) => (result.exitCode === 0 ? result.stdout.trim() : null)), + Effect.orElseSucceed(() => null), + ); + + const exists = yield* fileSystem.exists(checkout).pipe(Effect.mapError(fail("stat"))); + const isRepository = exists + ? yield* fileSystem.exists(path.join(checkout, ".git")).pipe(Effect.mapError(fail("stat"))) + : false; + + let created = false; + if (!isRepository) { + const entries = exists + ? yield* fileSystem.readDirectory(checkout).pipe(Effect.mapError(fail("readDirectory"))) + : []; + if (input.repository !== null) { + if (entries.length > 0) { + return yield* new RunnerCheckoutError({ + operation: "clone", + checkout, + detail: "The checkout directory exists, is not empty, and is not a git repository.", + }); + } + const parent = path.dirname(checkout); + yield* fileSystem + .makeDirectory(parent, { recursive: true }) + .pipe(Effect.mapError(fail("makeDirectory"))); + if (exists) { + yield* fileSystem.remove(checkout).pipe(Effect.mapError(fail("removeEmptyCheckout"))); + } + yield* run( + "clone", + parent, + ["clone", "--origin", "origin", input.repository.url, checkout], + CLONE_TIMEOUT_MS, + ); + } else { + yield* fileSystem + .makeDirectory(checkout, { recursive: true }) + .pipe(Effect.mapError(fail("makeDirectory"))); + yield* run("init", checkout, [ + "init", + `--initial-branch=${input.branch ?? DEFAULT_BLANK_BRANCH}`, + ]); + } + created = true; + } + + const currentBranch = () => probe(checkout, ["symbolic-ref", "--quiet", "--short", "HEAD"]); + const hasOrigin = (yield* probe(checkout, ["remote", "get-url", "origin"])) !== null; + const refExists = (ref: string) => + probe(checkout, ["rev-parse", "--verify", "--quiet", `${ref}^{commit}`]).pipe( + Effect.map((sha) => sha !== null), + ); + + if (input.branch !== null && (yield* currentBranch()) !== input.branch) { + const branch = input.branch; + if (yield* refExists(`refs/heads/${branch}`)) { + yield* run("switchBranch", checkout, ["checkout", branch]); + } else { + if (hasOrigin && !created) { + yield* run("fetch", checkout, ["fetch", "--prune", "origin"], FETCH_TIMEOUT_MS); + } + const candidates = [ + ...(hasOrigin ? [`refs/remotes/origin/${branch}`] : []), + ...(input.baseRef !== null + ? [...(hasOrigin ? [`refs/remotes/origin/${input.baseRef}`] : []), input.baseRef] + : []), + "HEAD", + ]; + let startPoint: string | null = null; + for (const candidate of candidates) { + if (yield* refExists(candidate)) { + startPoint = candidate; + break; + } + } + yield* startPoint === null + ? // An empty repository has no commit to branch from; point HEAD at the branch. + run("createUnbornBranch", checkout, ["symbolic-ref", "HEAD", `refs/heads/${branch}`]) + : run("createBranch", checkout, ["checkout", "-b", branch, startPoint]); + } + } + + return { + checkout, + isRepository: true, + branch: yield* currentBranch(), + headCommit: yield* probe(checkout, ["rev-parse", "--verify", "--quiet", "HEAD"]), + created, + } satisfies RunnerPrepareCheckoutResult; + }); + + return RunnerCheckout.of({ prepare }); +}); + +export const layer = Layer.effect(RunnerCheckout, make); diff --git a/apps/server/src/runner/RunnerHandlers.ts b/apps/server/src/runner/RunnerHandlers.ts new file mode 100644 index 000000000000..1a1d626b94ff --- /dev/null +++ b/apps/server/src/runner/RunnerHandlers.ts @@ -0,0 +1,734 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Attachment bytes are materialized synchronously before the provider reads them. +/** + * RunnerHandlers - serves `RunnerRpcGroup` for the runner's single thread. + * + * Every call is checked against the runner's binding: thread-keyed calls must + * name `T3CODE_RUNNER_THREAD_ID`, and cwd-keyed calls must stay inside + * `T3CODE_RUNNER_CHECKOUT`. A misrouted call fails with the RPC's own error + * type and never touches another path. + * + * Provider calls run the real adapters from the runner's provider instance + * registry; checkout calls run the same services the standalone server uses. + * + * @module runner/RunnerHandlers + */ +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + +import { + GitCommandError, + GitManagerError, + type ChatAttachment, + type GitManagerServiceError, + type ProviderInstanceId, + ProviderSendTurnInput, + type TerminalAttachStreamEvent, + type TerminalError, + TerminalCwdNotFoundError, + type TerminalEvent, + type TerminalMetadataStreamEvent, + TerminalSessionLookupError, + TextGenerationError, + ThreadId, + VcsRepositoryDetectionError, +} from "@t3tools/contracts"; +import { + RUNNER_MIN_PROTOCOL_VERSION, + RUNNER_PROTOCOL_VERSION, + type RunnerAttachmentFile, + RunnerCheckoutError, + type RunnerGitStackedActionEvent, + RunnerProtocolMismatchError, + RunnerRemoteError, + RunnerRpcGroup, + RunnerThreadMismatchError, + runnerProtocolsOverlap, +} from "@t3tools/contracts/runner"; +import { resolveServerBackgroundActivitySettings } from "@t3tools/shared/backgroundActivitySettings"; +import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Queue from "effect/Queue"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +import { resolveAttachmentPath } from "../attachmentStore.ts"; +import * as CheckpointStore from "../checkpointing/CheckpointStore.ts"; +import { ServerConfig } from "../config.ts"; +import * as GitWorkflowService from "../git/GitWorkflowService.ts"; +import * as McpProviderSession from "../mcp/McpProviderSession.ts"; +import { ProviderAdapterValidationError } from "../provider/Errors.ts"; +import { ProviderInstanceRegistry } from "../provider/Services/ProviderInstanceRegistry.ts"; +import * as ReviewService from "../review/ReviewService.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; +import * as TerminalManager from "../terminal/Manager.ts"; +import * as VcsProvisioningService from "../vcs/VcsProvisioningService.ts"; +import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; +import * as WorkspaceFileSystem from "../workspace/WorkspaceFileSystem.ts"; +import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; +import { RunnerCheckout } from "./RunnerCheckout.ts"; +import { RunnerOutbox } from "./RunnerOutbox.ts"; +import { ProviderAdapterErrorSchema, toRunnerRemoteError } from "./remoteErrors.ts"; + +const DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL = Duration.minutes(5); + +const WorkspaceFileErrorSchema = Schema.Union([ + ...WorkspaceFileSystem.WorkspaceFileSystemError.members, + WorkspacePaths.WorkspacePathOutsideRootError, +]); + +const toProviderRemote = toRunnerRemoteError(ProviderAdapterErrorSchema); +const isTextGenerationError = Schema.is(TextGenerationError); +const toEntriesRemote = toRunnerRemoteError(WorkspaceEntries.WorkspaceEntriesError); +const toFileRemote = toRunnerRemoteError(WorkspaceFileErrorSchema); + +export interface RunnerBinding { + readonly threadId: ThreadId; + readonly checkout: string; +} + +/** Reads the runner's thread binding from its configuration; fails loudly when absent. */ +export const resolveRunnerBinding = Effect.gen(function* () { + const config = yield* ServerConfig; + if (!config.runnerThreadId || !config.runnerCheckout) { + return yield* Effect.die( + new Error("t3 runner requires T3CODE_RUNNER_THREAD_ID and T3CODE_RUNNER_CHECKOUT."), + ); + } + if (!NodePath.isAbsolute(config.runnerCheckout)) { + return yield* Effect.die(new Error("T3CODE_RUNNER_CHECKOUT must be an absolute path.")); + } + return { + threadId: ThreadId.make(config.runnerThreadId), + checkout: NodePath.resolve(config.runnerCheckout), + } satisfies RunnerBinding; +}); + +const isWithin = (root: string, candidate: string) => { + const relative = NodePath.relative(root, NodePath.resolve(candidate)); + return relative === "" || (!relative.startsWith("..") && !NodePath.isAbsolute(relative)); +}; + +export const RunnerRpcHandlersLive = RunnerRpcGroup.toLayer( + Effect.gen(function* () { + const config = yield* ServerConfig; + const binding = yield* resolveRunnerBinding; + const registry = yield* ProviderInstanceRegistry; + const outbox = yield* RunnerOutbox; + const runnerCheckout = yield* RunnerCheckout; + const checkpointStore = yield* CheckpointStore.CheckpointStore; + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const workspaceFileSystem = yield* WorkspaceFileSystem.WorkspaceFileSystem; + const gitWorkflow = yield* GitWorkflowService.GitWorkflowService; + const vcsStatus = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const vcsProvisioning = yield* VcsProvisioningService.VcsProvisioningService; + const review = yield* ReviewService.ReviewService; + const terminals = yield* TerminalManager.TerminalManager; + const serverSettings = yield* ServerSettingsService; + + const automaticGitFetchInterval = serverSettings.getSettings.pipe( + Effect.map( + (settings) => resolveServerBackgroundActivitySettings(settings).automaticGitFetchInterval, + ), + Effect.orElseSucceed(() => DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL), + ); + + // ── Binding checks, each failing with the RPC's own error type ────── + + const outside = (cwd: string) => !isWithin(binding.checkout, cwd); + const outsideDetail = (cwd: string) => + `${cwd} is outside this runner's checkout ${binding.checkout}.`; + const guardVcs = (operation: string, cwd: string, effect: Effect.Effect) => + outside(cwd) + ? Effect.fail( + new VcsRepositoryDetectionError({ operation, cwd, detail: outsideDetail(cwd) }), + ) + : effect; + const guardGitCommand = ( + operation: string, + cwd: string, + effect: Effect.Effect, + ) => + outside(cwd) + ? Effect.fail( + new GitCommandError({ operation, command: "runner", cwd, detail: outsideDetail(cwd) }), + ) + : effect; + const guardGitManager = ( + operation: string, + cwd: string, + effect: Effect.Effect, + ): Effect.Effect => + outside(cwd) + ? Effect.fail(new GitManagerError({ operation, cwd, detail: outsideDetail(cwd) })) + : effect; + const guardTerminal = ( + input: { + readonly threadId: string; + readonly terminalId?: string | undefined; + readonly cwd?: string | undefined; + }, + effect: Effect.Effect, + ): Effect.Effect => { + if (input.threadId !== binding.threadId) { + return Effect.fail( + new TerminalSessionLookupError({ + threadId: input.threadId, + terminalId: input.terminalId ?? "*", + }), + ); + } + if (input.cwd !== undefined && outside(input.cwd)) { + // The checkout is the only directory this runner opens terminals in. + return Effect.fail(new TerminalCwdNotFoundError({ cwd: input.cwd })); + } + return effect; + }; + const guardThread = (method: string, threadId: string) => + threadId === binding.threadId + ? Effect.void + : Effect.fail( + toProviderRemote( + new ProviderAdapterValidationError({ + provider: "runner", + operation: method, + issue: `This runner serves thread '${binding.threadId}', not '${threadId}'.`, + }), + ), + ); + const guardRemote = (operation: string, cwd: string) => + outside(cwd) + ? Effect.fail( + new RunnerRemoteError({ + errorTag: "RunnerCwdOutsideCheckoutError", + detail: `${operation}: ${outsideDetail(cwd)}`, + encoded: null, + }), + ) + : Effect.void; + + // ── Provider instances ────────────────────────────────────────────── + + const instanceOf = (instanceId: ProviderInstanceId, operation: string) => + registry.getInstance(instanceId).pipe( + Effect.flatMap((instance) => + instance + ? Effect.succeed(instance) + : Effect.fail( + toProviderRemote( + new ProviderAdapterValidationError({ + provider: String(instanceId), + operation, + issue: `This runner does not host provider instance '${instanceId}'.`, + }), + ), + ), + ), + ); + const withAdapter = ( + instanceId: ProviderInstanceId, + operation: string, + use: ( + adapter: Effect.Success>["adapter"], + ) => Effect.Effect>, + ) => + instanceOf(instanceId, operation).pipe( + Effect.flatMap((instance) => use(instance.adapter).pipe(Effect.mapError(toProviderRemote))), + ); + + /** Logs a state-changing call with its outcome and duration. */ + const logged = ( + method: string, + attributes: Record, + effect: Effect.Effect, + ) => + Effect.gen(function* () { + const startedAt = yield* Clock.currentTimeMillis; + const exit = yield* Effect.exit(effect); + yield* Effect.logInfo(`runner rpc ${method}`, { + ...attributes, + ok: exit._tag === "Success", + ms: (yield* Clock.currentTimeMillis) - startedAt, + }); + return yield* exit; + }); + + // ── Attachments ───────────────────────────────────────────────────── + + /** + * Writes shipped attachment bytes where this runner's adapters read them + * and returns the prompt with the hub's attachment paths replaced. + */ + const materializeAttachments = (files: ReadonlyArray, text?: string) => + Effect.sync(() => { + let rewritten = text; + const attachments: Array = []; + for (const file of files) { + const localPath = resolveAttachmentPath({ + attachmentsDir: config.attachmentsDir, + attachment: file.attachment, + }); + if (localPath === null) continue; + NodeFS.mkdirSync(NodePath.dirname(localPath), { recursive: true }); + NodeFS.writeFileSync(localPath, Buffer.from(file.bytesBase64, "base64"), { + mode: 0o600, + }); + attachments.push(file.attachment); + if (rewritten !== undefined) rewritten = rewritten.replaceAll(file.hubPath, localPath); + } + return { attachments, text: rewritten }; + }); + + const toTextGenerationError = (operation: string) => (cause: unknown) => + isTextGenerationError(cause) + ? cause + : new TextGenerationError({ + operation, + detail: + cause && typeof cause === "object" && "message" in cause + ? String((cause as { readonly message: unknown }).message) + : String(cause), + }); + const textGenerationOf = (instanceId: ProviderInstanceId, operation: string) => + instanceOf(instanceId, operation).pipe( + Effect.map((instance) => instance.textGeneration), + Effect.mapError(toTextGenerationError(operation)), + ); + + const decodeSendTurnInput = Schema.decodeUnknownEffect(ProviderSendTurnInput); + + return RunnerRpcGroup.of({ + "runner.hello": (input) => + Effect.gen(function* () { + if ( + !runnerProtocolsOverlap({ + runnerProtocolVersion: RUNNER_PROTOCOL_VERSION, + runnerMinProtocolVersion: RUNNER_MIN_PROTOCOL_VERSION, + hubProtocolVersion: input.protocolVersion, + hubMinProtocolVersion: input.minProtocolVersion, + }) + ) { + return yield* new RunnerProtocolMismatchError({ + runnerProtocolVersion: RUNNER_PROTOCOL_VERSION, + runnerMinProtocolVersion: RUNNER_MIN_PROTOCOL_VERSION, + hubProtocolVersion: input.protocolVersion, + hubMinProtocolVersion: input.minProtocolVersion, + }); + } + if (input.threadId !== binding.threadId) { + return yield* new RunnerThreadMismatchError({ + requestedThreadId: input.threadId, + runnerThreadId: binding.threadId, + }); + } + const stats = yield* outbox.stats; + const instances = yield* registry.listInstances; + return { + protocolVersion: Math.min(RUNNER_PROTOCOL_VERSION, input.protocolVersion), + runnerId: stats.runnerId, + bootId: stats.bootId, + threadId: binding.threadId, + checkout: binding.checkout, + headSequence: stats.headSequence, + ackedSequence: stats.ackedSequence, + firstRetainedSequence: stats.firstRetainedSequence, + instances: instances.map((instance) => instance.instanceId), + }; + }), + + // ── Provider sessions ──────────────────────────────────────────── + "runner.provider.startSession": ({ instanceId, input, mcp }) => + guardThread("startSession", input.threadId).pipe( + Effect.andThen( + Effect.sync(() => + mcp === null + ? McpProviderSession.clearMcpProviderSession(input.threadId) + : McpProviderSession.setMcpProviderSession(mcp), + ), + ), + Effect.andThen( + logged( + "startSession", + { + threadId: input.threadId, + cwd: input.cwd, + resume: input.resumeCursor !== undefined, + }, + withAdapter(instanceId, "startSession", (adapter) => adapter.startSession(input)), + ), + ), + ), + "runner.provider.sendTurn": ({ instanceId, input, attachments }) => + guardThread("sendTurn", input.threadId).pipe( + Effect.andThen(materializeAttachments(attachments, input.input)), + Effect.flatMap((materialized) => + decodeSendTurnInput({ + ...input, + ...(materialized.text !== undefined ? { input: materialized.text } : {}), + ...(input.attachments !== undefined ? { attachments: materialized.attachments } : {}), + }).pipe( + Effect.mapError((cause) => + toProviderRemote( + new ProviderAdapterValidationError({ + provider: String(instanceId), + operation: "sendTurn", + issue: `Rewritten turn input is invalid: ${cause.message}`, + }), + ), + ), + ), + ), + Effect.flatMap((turnInput) => + logged( + "sendTurn", + { threadId: input.threadId, attachments: attachments.length }, + withAdapter(instanceId, "sendTurn", (adapter) => adapter.sendTurn(turnInput)), + ), + ), + ), + "runner.provider.interruptTurn": ({ instanceId, threadId, turnId }) => + guardThread("interruptTurn", threadId).pipe( + Effect.andThen( + logged( + "interruptTurn", + { threadId, turnId }, + withAdapter(instanceId, "interruptTurn", (adapter) => + adapter.interruptTurn(threadId, turnId), + ), + ), + ), + ), + "runner.provider.respondToRequest": ({ instanceId, threadId, requestId, decision }) => + guardThread("respondToRequest", threadId).pipe( + Effect.andThen( + logged( + "respondToRequest", + { threadId, requestId, decision }, + withAdapter(instanceId, "respondToRequest", (adapter) => + adapter.respondToRequest(threadId, requestId, decision), + ), + ), + ), + ), + "runner.provider.respondToUserInput": ({ instanceId, threadId, requestId, answers }) => + guardThread("respondToUserInput", threadId).pipe( + Effect.andThen( + withAdapter(instanceId, "respondToUserInput", (adapter) => + adapter.respondToUserInput(threadId, requestId, answers), + ), + ), + ), + "runner.provider.stopSession": ({ instanceId, threadId }) => + guardThread("stopSession", threadId).pipe( + Effect.andThen( + logged( + "stopSession", + { threadId }, + withAdapter(instanceId, "stopSession", (adapter) => adapter.stopSession(threadId)), + ), + ), + ), + "runner.provider.listSessions": ({ instanceId }) => + withAdapter(instanceId, "listSessions", (adapter) => adapter.listSessions()), + "runner.provider.readThread": ({ instanceId, threadId }) => + guardThread("readThread", threadId).pipe( + Effect.andThen( + withAdapter(instanceId, "readThread", (adapter) => adapter.readThread(threadId)), + ), + ), + "runner.provider.rollbackThread": ({ instanceId, threadId, numTurns }) => + guardThread("rollbackThread", threadId).pipe( + Effect.andThen( + withAdapter(instanceId, "rollbackThread", (adapter) => + adapter.rollbackThread(threadId, numTurns), + ), + ), + ), + "runner.provider.getCapabilities": ({ instanceId, refresh }) => + instanceOf(instanceId, "getCapabilities").pipe( + Effect.flatMap((instance) => + (refresh ? instance.snapshot.refresh : instance.snapshot.getSnapshot).pipe( + Effect.map((snapshot) => ({ + snapshot, + sessionModelSwitch: instance.adapter.capabilities.sessionModelSwitch, + })), + ), + ), + ), + + // ── Text generation ────────────────────────────────────────────── + "runner.text.generateThreadTitle": ({ instanceId, attachments, ...request }) => + (outside(request.cwd) + ? Effect.fail( + new TextGenerationError({ + operation: "generateThreadTitle", + detail: outsideDetail(request.cwd), + }), + ) + : Effect.void + ).pipe( + Effect.andThen(materializeAttachments(attachments)), + Effect.flatMap((materialized) => + logged( + "text.generateThreadTitle", + { model: request.modelSelection.model }, + textGenerationOf(instanceId, "generateThreadTitle").pipe( + Effect.flatMap((textGeneration) => + textGeneration.generateThreadTitle({ + ...request, + ...(materialized.attachments.length > 0 + ? { attachments: materialized.attachments } + : {}), + }), + ), + ), + ), + ), + ), + "runner.text.generateBranchName": ({ instanceId, attachments, ...request }) => + (outside(request.cwd) + ? Effect.fail( + new TextGenerationError({ + operation: "generateBranchName", + detail: outsideDetail(request.cwd), + }), + ) + : Effect.void + ).pipe( + Effect.andThen(materializeAttachments(attachments)), + Effect.flatMap((materialized) => + textGenerationOf(instanceId, "generateBranchName").pipe( + Effect.flatMap((textGeneration) => + textGeneration.generateBranchName({ + ...request, + ...(materialized.attachments.length > 0 + ? { attachments: materialized.attachments } + : {}), + }), + ), + ), + ), + ), + + // ── Event outbox ───────────────────────────────────────────────── + "runner.events.subscribe": ({ afterSequence }) => outbox.subscribe(afterSequence), + "runner.events.ack": ({ throughSequence }) => outbox.ack(throughSequence), + + // ── Checkout ───────────────────────────────────────────────────── + "runner.checkout.prepare": (input) => + input.threadId !== binding.threadId || NodePath.resolve(input.checkout) !== binding.checkout + ? Effect.fail( + new RunnerCheckoutError({ + operation: "validate", + checkout: input.checkout, + detail: `This runner serves thread '${binding.threadId}' at ${binding.checkout}.`, + }), + ) + : logged( + "checkout.prepare", + { branch: input.branch, baseRef: input.baseRef, repository: input.repository?.url }, + runnerCheckout.prepare(input), + ), + + // ── Checkpoints ────────────────────────────────────────────────── + "runner.checkpoint.isGitRepository": ({ cwd }) => + guardVcs("isGitRepository", cwd, checkpointStore.isGitRepository(cwd)), + "runner.checkpoint.capture": (input) => + guardVcs( + "captureCheckpoint", + input.cwd, + logged( + "checkpoint.capture", + { checkpointRef: input.checkpointRef }, + checkpointStore.captureCheckpoint(input), + ), + ), + "runner.checkpoint.hasRef": (input) => + guardVcs("hasCheckpointRef", input.cwd, checkpointStore.hasCheckpointRef(input)), + "runner.checkpoint.restore": ({ cwd, checkpointRef, fallbackToHead }) => + guardVcs( + "restoreCheckpoint", + cwd, + checkpointStore.restoreCheckpoint({ + cwd, + checkpointRef, + ...(fallbackToHead !== undefined ? { fallbackToHead } : {}), + }), + ), + "runner.checkpoint.diff": ({ fallbackFromToHead, ...input }) => + guardVcs( + "diffCheckpoints", + input.cwd, + checkpointStore.diffCheckpoints({ + ...input, + ...(fallbackFromToHead !== undefined ? { fallbackFromToHead } : {}), + }), + ), + "runner.checkpoint.deleteRefs": (input) => + guardVcs("deleteCheckpointRefs", input.cwd, checkpointStore.deleteCheckpointRefs(input)), + + // ── Workspace files ────────────────────────────────────────────── + "runner.workspace.listEntries": (input) => + guardRemote("listEntries", input.cwd).pipe( + Effect.andThen(workspaceEntries.list(input).pipe(Effect.mapError(toEntriesRemote))), + ), + "runner.workspace.searchEntries": (input) => + guardRemote("searchEntries", input.cwd).pipe( + Effect.andThen(workspaceEntries.search(input).pipe(Effect.mapError(toEntriesRemote))), + ), + "runner.workspace.searchContents": (input) => + guardRemote("searchContents", input.cwd).pipe( + Effect.andThen( + workspaceEntries.searchContents(input).pipe(Effect.mapError(toEntriesRemote)), + ), + ), + "runner.workspace.readFile": (input) => + guardRemote("readFile", input.cwd).pipe( + Effect.andThen(workspaceFileSystem.readFile(input).pipe(Effect.mapError(toFileRemote))), + ), + "runner.workspace.writeFile": (input) => + guardRemote("writeFile", input.cwd).pipe( + Effect.andThen( + logged( + "workspace.writeFile", + { relativePath: input.relativePath }, + workspaceFileSystem.writeFile(input).pipe(Effect.mapError(toFileRemote)), + ), + ), + ), + "runner.workspace.refreshIndex": ({ cwd }) => + outside(cwd) ? Effect.void : workspaceEntries.refresh(cwd), + + // ── VCS and git ────────────────────────────────────────────────── + "runner.vcs.status": (input) => + guardGitManager("status", input.cwd, vcsStatus.getStatus(input)), + "runner.vcs.localStatus": (input) => + guardGitManager("localStatus", input.cwd, gitWorkflow.localStatus(input)), + "runner.vcs.remoteStatus": (input) => + guardGitManager("remoteStatus", input.cwd, gitWorkflow.remoteStatus(input)), + "runner.vcs.refreshStatus": ({ cwd }) => + guardGitManager("refreshStatus", cwd, vcsStatus.refreshStatus(cwd)), + "runner.vcs.refreshLocalStatus": ({ cwd }) => + guardGitManager("refreshLocalStatus", cwd, vcsStatus.refreshLocalStatus(cwd)), + "runner.vcs.streamStatus": (input) => + outside(input.cwd) + ? Stream.fail( + new GitManagerError({ + operation: "streamStatus", + cwd: input.cwd, + detail: outsideDetail(input.cwd), + }), + ) + : vcsStatus.streamStatus(input, { + automaticRemoteRefreshInterval: automaticGitFetchInterval, + }), + "runner.vcs.init": (input) => + guardVcs("initRepository", input.cwd, vcsProvisioning.initRepository(input)), + "runner.git.pull": ({ cwd }) => + guardGitCommand("pull", cwd, logged("git.pull", {}, gitWorkflow.pullCurrentBranch(cwd))), + "runner.git.runStackedAction": (input) => + outside(input.cwd) + ? Stream.fail( + new GitManagerError({ + operation: "runStackedAction", + cwd: input.cwd, + detail: outsideDetail(input.cwd), + }), + ) + : Stream.callback((queue) => + gitWorkflow + .runStackedAction(input, { + actionId: input.actionId, + progressReporter: { + publish: (event) => + Queue.offer(queue, { _tag: "progress", event }).pipe(Effect.asVoid), + }, + }) + .pipe( + Effect.matchCauseEffect({ + onFailure: (cause) => Queue.failCause(queue, cause), + onSuccess: (result) => + Queue.offer(queue, { _tag: "result", result }).pipe( + Effect.andThen(Queue.end(queue)), + ), + }), + ), + ), + "runner.git.resolvePullRequest": (input) => + guardGitManager("resolvePullRequest", input.cwd, gitWorkflow.resolvePullRequest(input)), + "runner.git.preparePullRequestThread": (input) => + guardGitManager( + "preparePullRequestThread", + input.cwd, + gitWorkflow.preparePullRequestThread(input), + ), + "runner.git.listRefs": (input) => + guardGitCommand("listRefs", input.cwd, gitWorkflow.listRefs(input)), + "runner.git.createWorktree": (input) => + guardGitCommand("createWorktree", input.cwd, gitWorkflow.createWorktree(input)), + "runner.git.removeWorktree": (input) => + guardGitCommand("removeWorktree", input.cwd, gitWorkflow.removeWorktree(input)), + "runner.git.pruneWorktrees": (input) => + guardGitCommand("pruneWorktrees", input.cwd, gitWorkflow.pruneWorktrees(input)), + "runner.git.createRef": (input) => + guardGitCommand("createRef", input.cwd, gitWorkflow.createRef(input)), + "runner.git.switchRef": (input) => + guardGitCommand("switchRef", input.cwd, gitWorkflow.switchRef(input)), + "runner.git.renameBranch": (input) => + guardGitManager("renameBranch", input.cwd, gitWorkflow.renameBranch(input)), + "runner.git.fetchRemote": (input) => + guardGitCommand("fetchRemote", input.cwd, gitWorkflow.fetchRemote(input)), + "runner.git.remoteExists": (input) => + guardGitCommand("remoteExists", input.cwd, gitWorkflow.remoteExists(input)), + "runner.git.resolveRemoteTrackingCommit": (input) => + guardGitCommand( + "resolveRemoteTrackingCommit", + input.cwd, + gitWorkflow.resolveRemoteTrackingCommit(input), + ), + + // ── Review ─────────────────────────────────────────────────────── + "runner.review.getDiffPreview": (input) => + guardVcs("getDiffPreview", input.cwd, review.getDiffPreview(input)), + "runner.review.getDiffFileContents": (input) => + guardVcs("getDiffFileContents", input.cwd, review.getDiffFileContents(input)), + + // ── Terminals ──────────────────────────────────────────────────── + "runner.terminal.open": (input) => guardTerminal(input, terminals.open(input)), + "runner.terminal.attach": (input) => + input.threadId !== binding.threadId + ? Stream.fail( + new TerminalSessionLookupError({ + threadId: input.threadId, + terminalId: input.terminalId, + }), + ) + : Stream.callback((queue) => + Effect.acquireRelease( + terminals.attachStream(input, (event) => Queue.offer(queue, event)), + (unsubscribe) => Effect.sync(unsubscribe), + ), + ), + "runner.terminal.write": (input) => guardTerminal(input, terminals.write(input)), + "runner.terminal.resize": (input) => guardTerminal(input, terminals.resize(input)), + "runner.terminal.clear": (input) => guardTerminal(input, terminals.clear(input)), + "runner.terminal.restart": (input) => guardTerminal(input, terminals.restart(input)), + "runner.terminal.close": (input) => guardTerminal(input, terminals.close(input)), + "runner.terminal.events": () => + Stream.callback((queue) => + Effect.acquireRelease( + terminals.subscribe((event) => Queue.offer(queue, event)), + (unsubscribe) => Effect.sync(unsubscribe), + ), + ), + "runner.terminal.metadata": () => + Stream.callback((queue) => + Effect.acquireRelease( + terminals.subscribeMetadata((event) => Queue.offer(queue, event)), + (unsubscribe) => Effect.sync(unsubscribe), + ), + ), + }); + }), +); diff --git a/apps/server/src/runner/RunnerOutbox.ts b/apps/server/src/runner/RunnerOutbox.ts index dd128769fc43..16942466baf6 100644 --- a/apps/server/src/runner/RunnerOutbox.ts +++ b/apps/server/src/runner/RunnerOutbox.ts @@ -1,30 +1,34 @@ -// @effect-diagnostics nodeBuiltinImport:off -- Append order is the durability contract; synchronous appends keep it trivially ordered. +// @effect-diagnostics nodeBuiltinImport:off -- Append order and fsync-before-publish are the durability contract; synchronous node:sqlite keeps both trivially true. /** - * RunnerOutbox - durable, replayable provider runtime event log (prototype). + * RunnerOutbox - the runner's durable, replayable provider event log. * - * Provider adapters publish runtime events to an in-memory PubSub. A hub on - * another machine can disconnect, restart, or lag, so the runner writes each - * event to an append-only NDJSON file with a monotonically increasing - * `sequence` before it is offered to subscribers. The hub subscribes from its - * last acknowledged sequence and acknowledges after ingestion; acknowledged - * events are compacted away. + * Provider adapters publish runtime events to in-memory streams, but the hub + * that ingests them lives on another machine and can disconnect, restart or + * lag. Every event is appended to a SQLite table on the machine's state disk + * with a strictly increasing `sequence` and committed (WAL, `synchronous = + * FULL`, so the commit is fsynced) before any subscriber sees it. The hub + * subscribes from its cursor and acknowledges once the events' effects are + * durable on the hub; acknowledged rows are deleted. * - * Files under `/runner/`: - * - `runner-id` stable runner identity (survives restarts) - * - `outbox.ndjson` one encoded `RunnerEventEnvelope` per line - * - `outbox-ack.json` `{ "ackedSequence": n }` - * - * Guarantees: at-least-once delivery of every event appended before a crash - * (a torn final line is discarded on load), strictly increasing sequences - * across restarts, and in-order replay. The hub deduplicates by sequence. + * Guarantees: + * - At-least-once, in-order delivery of every committed event. + * - Sequences never repeat for this outbox (`runnerId`), across restarts. + * - Event ids are unique among retained rows; a duplicate id from an adapter + * is suffixed with its sequence, so hub-side deterministic ingestion ids + * never collide. + * - Retention: when more than `maxRetained` events are unacknowledged, the + * oldest are dropped and `firstRetainedSequence` moves past them. A hub + * whose cursor is below it has lost events and must resynchronize. * * @module runner/RunnerOutbox */ import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodePath from "node:path"; +import * as NodeSqlite from "node:sqlite"; -import type { ProviderRuntimeEvent } from "@t3tools/contracts"; +import { EventId, type ProviderRuntimeEvent } from "@t3tools/contracts"; +import { RunnerEventEnvelope } from "@t3tools/contracts/runner"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -33,208 +37,268 @@ import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { ServerConfig } from "../config.ts"; -import { RunnerEventEnvelope } from "./RunnerProtocol.ts"; -const EnvelopeLine = Schema.fromJsonString(RunnerEventEnvelope); -const encodeEnvelopeLine = Schema.encodeSync(EnvelopeLine); -const decodeEnvelopeLine = Schema.decodeUnknownSync(EnvelopeLine); -const AckFile = Schema.fromJsonString(Schema.Struct({ ackedSequence: Schema.Number })); -const encodeAckFile = Schema.encodeSync(AckFile); -const decodeAckFile = Schema.decodeUnknownSync(AckFile); +const EnvelopeJson = Schema.fromJsonString(RunnerEventEnvelope); +const encodeEnvelopeJson = Schema.encodeSync(EnvelopeJson); +const decodeEnvelopeJson = Schema.decodeUnknownSync(EnvelopeJson); -/** Rewrite the file once this many acknowledged lines have accumulated. */ -const COMPACT_AFTER_ACKED_LINES = 500; +/** Unacknowledged events kept before the oldest are dropped. */ +export const DEFAULT_MAX_RETAINED_EVENTS = 200_000; +const REPLAY_PAGE_SIZE = 500; export interface RunnerOutboxStats { readonly runnerId: string; readonly bootId: string; readonly headSequence: number; readonly ackedSequence: number; + readonly firstRetainedSequence: number; readonly retained: number; readonly rejected: number; + readonly dropped: number; } -export class RunnerOutbox extends Context.Service< - RunnerOutbox, - { - readonly append: (event: ProviderRuntimeEvent) => Effect.Effect; - readonly subscribe: (afterSequence: number) => Stream.Stream; - readonly ack: ( - throughSequence: number, - ) => Effect.Effect<{ readonly ackedSequence: number; readonly retained: number }>; - readonly stats: Effect.Effect; - } ->()("t3/runner/RunnerOutbox") {} - -interface LoadedOutbox { - readonly entries: Array; - readonly headSequence: number; - readonly ackedSequence: number; - readonly discardedLines: number; +export interface RunnerOutboxShape { + readonly append: (event: ProviderRuntimeEvent) => Effect.Effect; + /** Retained events with `sequence > afterSequence`, then live events, without gaps or overlap. */ + readonly subscribe: (afterSequence: number) => Stream.Stream; + readonly ack: ( + throughSequence: number, + ) => Effect.Effect<{ readonly ackedSequence: number; readonly retained: number }>; + readonly stats: Effect.Effect; } -function readAckedSequence(ackPath: string): number { - try { - return decodeAckFile(NodeFS.readFileSync(ackPath, "utf8")).ackedSequence; - } catch { - return 0; - } -} +export class RunnerOutbox extends Context.Service()( + "t3/runner/RunnerOutbox", +) {} -function loadOutbox(outboxPath: string, ackPath: string): LoadedOutbox { - const ackedSequence = readAckedSequence(ackPath); - const entries: Array = []; - let headSequence = ackedSequence; - let discardedLines = 0; - let raw = ""; - try { - raw = NodeFS.readFileSync(outboxPath, "utf8"); - } catch { - raw = ""; - } - for (const line of raw.split("\n")) { - if (line.trim().length === 0) continue; - try { - const envelope = decodeEnvelopeLine(line); - headSequence = Math.max(headSequence, envelope.sequence); - if (envelope.sequence > ackedSequence) entries.push(envelope); - } catch { - // A crash can tear the final line; anything unparseable is dropped. - discardedLines += 1; - } - } - entries.sort((left, right) => left.sequence - right.sequence); - return { entries, headSequence, ackedSequence, discardedLines }; +export interface RunnerOutboxOptions { + readonly databasePath: string; + readonly maxRetained?: number; } -function readOrCreateRunnerId(idPath: string): string { - try { - const existing = NodeFS.readFileSync(idPath, "utf8").trim(); - if (existing.length > 0) return existing; - } catch { - // fall through - } - const created = `runner-${NodeCrypto.randomUUID()}`; - NodeFS.writeFileSync(idPath, `${created}\n`, { mode: 0o600 }); - return created; -} +const SCHEMA = ` + PRAGMA journal_mode = WAL; + PRAGMA synchronous = FULL; + CREATE TABLE IF NOT EXISTS runner_outbox ( + sequence INTEGER PRIMARY KEY, + boot_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + event_id TEXT NOT NULL UNIQUE, + event_type TEXT NOT NULL, + envelope_json TEXT NOT NULL, + created_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS runner_outbox_state ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); +`; -export const make = Effect.gen(function* () { - const config = yield* ServerConfig; - const dir = NodePath.join(config.stateDir, "runner"); - NodeFS.mkdirSync(dir, { recursive: true }); - const outboxPath = NodePath.join(dir, "outbox.ndjson"); - const ackPath = NodePath.join(dir, "outbox-ack.json"); - const runnerId = readOrCreateRunnerId(NodePath.join(dir, "runner-id")); - const bootId = `boot-${NodeCrypto.randomUUID()}`; - const fsyncEnabled = process.env.T3CODE_RUNNER_OUTBOX_FSYNC !== "0"; - // Evidence/debug switch: keep acknowledged lines instead of compacting. - const keepAcked = process.env.T3CODE_RUNNER_OUTBOX_KEEP === "1"; - - const loaded = loadOutbox(outboxPath, ackPath); - const entries = loaded.entries; - let headSequence = loaded.headSequence; - let ackedSequence = loaded.ackedSequence; - let ackedLinesSinceCompaction = 0; - let rejected = 0; - yield* Effect.logInfo("runner outbox loaded", { - runnerId, - bootId, - headSequence, - ackedSequence, - retained: entries.length, - discardedLines: loaded.discardedLines, - }); +export const makeWithOptions = (options: RunnerOutboxOptions) => + Effect.gen(function* () { + NodeFS.mkdirSync(NodePath.dirname(options.databasePath), { recursive: true }); + const db = yield* Effect.acquireRelease( + Effect.sync(() => { + const database = new NodeSqlite.DatabaseSync(options.databasePath); + database.exec(SCHEMA); + return database; + }), + (database) => Effect.sync(() => database.close()), + ); + const maxRetained = options.maxRetained ?? DEFAULT_MAX_RETAINED_EVENTS; - let fd = NodeFS.openSync(outboxPath, "a"); - yield* Effect.addFinalizer(() => Effect.sync(() => NodeFS.closeSync(fd))); - - const live = yield* PubSub.unbounded(); - - const compact = () => { - const tmpPath = `${outboxPath}.tmp`; - const body = entries.map((entry) => `${encodeEnvelopeLine(entry)}\n`).join(""); - NodeFS.writeFileSync(tmpPath, body); - NodeFS.closeSync(fd); - NodeFS.renameSync(tmpPath, outboxPath); - fd = NodeFS.openSync(outboxPath, "a"); - ackedLinesSinceCompaction = 0; - }; - - const append = (event: ProviderRuntimeEvent) => - Effect.suspend(() => { - const envelope: RunnerEventEnvelope = { sequence: headSequence + 1, bootId, event }; - let line: string; + const readState = db.prepare("SELECT value FROM runner_outbox_state WHERE key = ?"); + const writeState = db.prepare( + "INSERT INTO runner_outbox_state (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value", + ); + const stateValue = (key: string): string | undefined => { + const row = readState.get(key) as { value?: unknown } | undefined; + return typeof row?.value === "string" ? row.value : undefined; + }; + const numericState = (key: string) => Number(stateValue(key) ?? "0") || 0; + + const runnerId = + stateValue("runner_id") ?? + (() => { + const created = `runner-${NodeCrypto.randomUUID()}`; + writeState.run("runner_id", created); + return created; + })(); + const bootId = `boot-${NodeCrypto.randomUUID()}`; + + const maxRow = db.prepare("SELECT MAX(sequence) AS max FROM runner_outbox").get() as { + max: number | null; + }; + let headSequence = Math.max(numericState("head_sequence"), maxRow.max ?? 0); + let ackedSequence = numericState("acked_sequence"); + let firstRetainedSequence = Math.max( + numericState("first_retained_sequence"), + ackedSequence + 1, + ); + let rejected = 0; + let dropped = 0; + + const insertEvent = db.prepare( + "INSERT INTO runner_outbox (sequence, boot_id, thread_id, event_id, event_type, envelope_json, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ); + const eventIdTaken = db.prepare("SELECT 1 AS taken FROM runner_outbox WHERE event_id = ?"); + const countRetained = db.prepare("SELECT COUNT(*) AS count FROM runner_outbox"); + const deleteThrough = db.prepare("DELETE FROM runner_outbox WHERE sequence <= ?"); + const selectPage = db.prepare( + "SELECT envelope_json FROM runner_outbox WHERE sequence > ? ORDER BY sequence ASC LIMIT ?", + ); + const selectOldest = db.prepare( + "SELECT sequence FROM runner_outbox ORDER BY sequence ASC LIMIT 1 OFFSET ?", + ); + + const retainedCount = () => (countRetained.get() as { count: number }).count; + + const transaction = (body: () => A): A => { + db.exec("BEGIN IMMEDIATE"); try { - line = `${encodeEnvelopeLine(envelope)}\n`; + const result = body(); + db.exec("COMMIT"); + return result; } catch (cause) { - // An event that does not satisfy the wire schema could never be - // delivered; count it instead of poisoning every subscription. - rejected += 1; - return Effect.logWarning("runner outbox rejected an event that fails the wire schema", { - eventType: event.type, - eventId: event.eventId, - cause: String(cause), - }); + db.exec("ROLLBACK"); + throw cause; } - headSequence = envelope.sequence; - NodeFS.writeSync(fd, line); - if (fsyncEnabled) NodeFS.fsyncSync(fd); - entries.push(envelope); - return PubSub.publish(live, envelope).pipe(Effect.asVoid); - }); + }; - const subscribe = (afterSequence: number) => - Stream.unwrap( - Effect.gen(function* () { - // Subscribe before snapshotting the backlog so no append can fall - // between replay and the live tail; overlap is filtered by sequence. - const subscription = yield* PubSub.subscribe(live); - const backlog = entries.filter((entry) => entry.sequence > afterSequence); - const replayedThrough = backlog.at(-1)?.sequence ?? afterSequence; - return Stream.concat( - Stream.fromIterable(backlog), - Stream.fromSubscription(subscription).pipe( - Stream.filter((entry) => entry.sequence > replayedThrough), - ), - ); - }), - ); + /** Drops the oldest unacknowledged events beyond the retention bound. */ + const enforceRetention = () => { + const excess = retainedCount() - maxRetained; + if (excess <= 0) return; + const boundary = selectOldest.get(excess - 1) as { sequence: number } | undefined; + if (!boundary) return; + transaction(() => { + deleteThrough.run(boundary.sequence); + firstRetainedSequence = boundary.sequence + 1; + writeState.run("first_retained_sequence", String(firstRetainedSequence)); + }); + dropped += excess; + }; - const ack = (throughSequence: number) => - Effect.sync(() => { - const next = Math.min(Math.max(ackedSequence, throughSequence), headSequence); - if (next > ackedSequence) { - ackedSequence = next; - let dropped = 0; - while (entries.length > 0 && entries[0]!.sequence <= ackedSequence) { - entries.shift(); - dropped += 1; - } - ackedLinesSinceCompaction += dropped; - NodeFS.writeFileSync(ackPath, encodeAckFile({ ackedSequence })); - if ( - !keepAcked && - (ackedLinesSinceCompaction >= COMPACT_AFTER_ACKED_LINES || entries.length === 0) - ) { - compact(); - } - } - return { ackedSequence, retained: entries.length }; - }); + const live = yield* PubSub.unbounded(); - const stats = Effect.sync( - (): RunnerOutboxStats => ({ + yield* Effect.logInfo("runner outbox opened", { runnerId, bootId, headSequence, ackedSequence, - retained: entries.length, - rejected, - }), - ); + retained: retainedCount(), + }); + + const append: RunnerOutboxShape["append"] = (event) => + Effect.suspend(() => { + const sequence = headSequence + 1; + const eventId = eventIdTaken.get(event.eventId) + ? EventId.make(`${event.eventId}:${sequence}`) + : event.eventId; + const envelope: RunnerEventEnvelope = { + sequence, + bootId, + event: eventId === event.eventId ? event : { ...event, eventId }, + }; + let envelopeJson: string; + try { + envelopeJson = encodeEnvelopeJson(envelope); + } catch (cause) { + // An event that fails the wire schema could never be delivered; + // count it rather than poisoning every subscription. + rejected += 1; + return Effect.logWarning("runner outbox rejected an event that fails the wire schema", { + eventType: event.type, + eventId: event.eventId, + cause: String(cause).slice(0, 400), + }); + } + transaction(() => { + insertEvent.run( + sequence, + bootId, + event.threadId, + eventId, + event.type, + envelopeJson, + event.createdAt, + ); + writeState.run("head_sequence", String(sequence)); + }); + headSequence = sequence; + enforceRetention(); + return PubSub.publish(live, envelope).pipe(Effect.asVoid); + }); + + const readAfter = (afterSequence: number): Array => + (selectPage.all(afterSequence, REPLAY_PAGE_SIZE) as Array<{ envelope_json: string }>).map( + (row) => decodeEnvelopeJson(row.envelope_json), + ); + + const subscribe: RunnerOutboxShape["subscribe"] = (afterSequence) => + Stream.unwrap( + Effect.gen(function* () { + // Subscribe before reading the backlog so no append falls between + // replay and the live tail; overlap is filtered by sequence. + const subscription = yield* PubSub.subscribe(live); + let cursor = afterSequence; + const backlog = Stream.unfold(afterSequence, (after) => + Effect.sync(() => { + const page = readAfter(after); + const last = page.at(-1)?.sequence; + if (last === undefined) return undefined; + cursor = last; + return [page, last] as const; + }), + ).pipe(Stream.flatMap((page) => Stream.fromIterable(page))); + return Stream.concat( + backlog, + Stream.suspend(() => + Stream.fromSubscription(subscription).pipe( + Stream.filter((entry) => entry.sequence > cursor), + ), + ), + ); + }), + ); - return RunnerOutbox.of({ append, subscribe, ack, stats }); -}); + const ack: RunnerOutboxShape["ack"] = (throughSequence) => + Effect.sync(() => { + const next = Math.min(Math.max(ackedSequence, throughSequence), headSequence); + if (next > ackedSequence) { + transaction(() => { + deleteThrough.run(next); + writeState.run("acked_sequence", String(next)); + }); + ackedSequence = next; + firstRetainedSequence = Math.max(firstRetainedSequence, next + 1); + } + return { ackedSequence, retained: retainedCount() }; + }); -export const layer = Layer.effect(RunnerOutbox, make); + const stats = Effect.sync( + (): RunnerOutboxStats => ({ + runnerId, + bootId, + headSequence, + ackedSequence, + firstRetainedSequence, + retained: retainedCount(), + rejected, + dropped, + }), + ); + + return RunnerOutbox.of({ append, subscribe, ack, stats }); + }); + +/** `/runner/outbox.sqlite` on the machine's state disk. */ +export const layer = Layer.effect( + RunnerOutbox, + Effect.gen(function* () { + const config = yield* ServerConfig; + return yield* makeWithOptions({ + databasePath: NodePath.join(config.stateDir, "runner", "outbox.sqlite"), + }); + }), +); diff --git a/apps/server/src/runner/RunnerProjectionSnapshotQuery.ts b/apps/server/src/runner/RunnerProjectionSnapshotQuery.ts new file mode 100644 index 000000000000..39d9963790a0 --- /dev/null +++ b/apps/server/src/runner/RunnerProjectionSnapshotQuery.ts @@ -0,0 +1,38 @@ +/** + * The runner has no orchestration state. A few checkout services it hosts + * (review path authorization, setup scripts inside git actions) look up + * projects by id or root; on a runner those lookups find nothing, and the + * services fall back to the runner's own checkout root. Whole-model reads are + * never valid on a runner and die loudly. + * + * @module runner/RunnerProjectionSnapshotQuery + */ +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; + +import { ProjectionSnapshotQuery } from "../orchestration/Services/ProjectionSnapshotQuery.ts"; + +const noProjections = (method: string) => + Effect.die(new Error(`ProjectionSnapshotQuery.${method} is not available on a runner.`)); + +export const layer = Layer.succeed( + ProjectionSnapshotQuery, + ProjectionSnapshotQuery.of({ + getCommandReadModel: () => noProjections("getCommandReadModel"), + getSnapshot: () => noProjections("getSnapshot"), + getShellSnapshot: () => noProjections("getShellSnapshot"), + getArchivedShellSnapshot: () => noProjections("getArchivedShellSnapshot"), + searchThreads: () => noProjections("searchThreads"), + getSnapshotSequence: () => noProjections("getSnapshotSequence"), + getCounts: () => noProjections("getCounts"), + getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()), + getProjectShellById: () => Effect.succeed(Option.none()), + getFirstActiveThreadIdByProjectId: () => Effect.succeed(Option.none()), + getThreadCheckpointContext: () => Effect.succeed(Option.none()), + getFullThreadDiffContext: () => Effect.succeed(Option.none()), + getThreadShellById: () => Effect.succeed(Option.none()), + getThreadDetailById: () => Effect.succeed(Option.none()), + getThreadDetailSnapshot: () => Effect.succeed(Option.none()), + }), +); diff --git a/apps/server/src/runner/RunnerProtocol.ts b/apps/server/src/runner/RunnerProtocol.ts deleted file mode 100644 index 77f0e053a170..000000000000 --- a/apps/server/src/runner/RunnerProtocol.ts +++ /dev/null @@ -1,317 +0,0 @@ -/** - * RunnerProtocol - the hub <-> runner wire contract (prototype). - * - * The hub is a T3 server with no local checkout: it owns orchestration, - * projections, settings and the provider session directory. A runner owns - * one machine's checkout and runs the real provider drivers, git and - * checkpoints for it. The hub reaches the runner through this Effect RPC - * group over one authenticated WebSocket. - * - * Every payload reuses a `@t3tools/contracts` schema where one exists, so - * the runner speaks exactly the shapes the in-process adapter already uses. - * - * Event delivery is the only stateful part: runners append every provider - * runtime event to a durable outbox with a monotonically increasing - * `sequence`; the hub subscribes from its last acknowledged sequence and - * acknowledges once the events are ingested. See `RunnerOutbox.ts`. - * - * @module runner/RunnerProtocol - */ -import { - ApprovalRequestId, - ChatAttachment, - CheckpointRef, - ModelSelection, - NonNegativeInt, - ProviderApprovalDecision, - ProviderInstanceId, - ProviderRuntimeEvent, - ProviderSendTurnInput, - ProviderSession, - ProviderSessionStartInput, - ProviderTurnStartResult, - ProviderUserInputAnswers, - ServerProvider, - TextGenerationError, - ThreadId, - TurnId, - VcsError, -} from "@t3tools/contracts"; -import * as Schema from "effect/Schema"; -import * as Rpc from "effect/unstable/rpc/Rpc"; -import * as RpcGroup from "effect/unstable/rpc/RpcGroup"; - -import { - ProviderAdapterProcessError, - ProviderAdapterRequestError, - ProviderAdapterSessionClosedError, - ProviderAdapterSessionNotFoundError, - ProviderAdapterValidationError, -} from "../provider/Errors.ts"; - -export const RUNNER_PROTOCOL_VERSION = 1; -export const RUNNER_WS_PATH = "/runner/ws"; - -/** Adapter errors cross the wire with their original tags. */ -export const RunnerProviderError = Schema.Union([ - ProviderAdapterValidationError, - ProviderAdapterSessionNotFoundError, - ProviderAdapterSessionClosedError, - ProviderAdapterRequestError, - ProviderAdapterProcessError, -]); -export type RunnerProviderError = typeof RunnerProviderError.Type; - -export class RunnerWorkspaceError extends Schema.TaggedErrorClass()( - "RunnerWorkspaceError", - { - operation: Schema.String, - workspaceRoot: Schema.String, - reason: Schema.Literals(["not-exists", "not-directory", "create-failed", "stat-failed"]), - detail: Schema.String, - }, -) { - override get message(): string { - return `Runner workspace ${this.operation} failed for ${this.workspaceRoot}: ${this.detail}`; - } -} - -export const RunnerHello = Schema.Struct({ - protocolVersion: Schema.Number, - /** Stable for the runner's state directory; survives restarts. */ - runnerId: Schema.String, - /** Changes on every runner process start. */ - bootId: Schema.String, - /** Highest sequence ever appended to the outbox (0 when empty). */ - headSequence: NonNegativeInt, - /** Highest sequence the hub has acknowledged. */ - ackedSequence: NonNegativeInt, - instances: Schema.Array(ProviderInstanceId), -}); -export type RunnerHello = typeof RunnerHello.Type; - -export const RunnerEventEnvelope = Schema.Struct({ - sequence: NonNegativeInt, - bootId: Schema.String, - event: ProviderRuntimeEvent, -}); -export type RunnerEventEnvelope = typeof RunnerEventEnvelope.Type; - -const InstanceScoped = { instanceId: ProviderInstanceId } as const; - -export const RunnerThreadSnapshot = Schema.Struct({ - threadId: ThreadId, - turns: Schema.Array(Schema.Struct({ id: TurnId, items: Schema.Array(Schema.Unknown) })), -}); - -// ── Provider adapter group ─────────────────────────────────────────────── - -export const RunnerHelloRpc = Rpc.make("runner.hello", { - payload: Schema.Struct({}), - success: RunnerHello, -}); - -export const RunnerStartSessionRpc = Rpc.make("runner.provider.startSession", { - payload: Schema.Struct({ ...InstanceScoped, input: ProviderSessionStartInput }), - success: ProviderSession, - error: RunnerProviderError, -}); - -export const RunnerSendTurnRpc = Rpc.make("runner.provider.sendTurn", { - payload: Schema.Struct({ ...InstanceScoped, input: ProviderSendTurnInput }), - success: ProviderTurnStartResult, - error: RunnerProviderError, -}); - -export const RunnerInterruptTurnRpc = Rpc.make("runner.provider.interruptTurn", { - payload: Schema.Struct({ - ...InstanceScoped, - threadId: ThreadId, - turnId: Schema.optional(TurnId), - }), - success: Schema.Void, - error: RunnerProviderError, -}); - -export const RunnerRespondToRequestRpc = Rpc.make("runner.provider.respondToRequest", { - payload: Schema.Struct({ - ...InstanceScoped, - threadId: ThreadId, - requestId: ApprovalRequestId, - decision: ProviderApprovalDecision, - }), - success: Schema.Void, - error: RunnerProviderError, -}); - -export const RunnerRespondToUserInputRpc = Rpc.make("runner.provider.respondToUserInput", { - payload: Schema.Struct({ - ...InstanceScoped, - threadId: ThreadId, - requestId: ApprovalRequestId, - answers: ProviderUserInputAnswers, - }), - success: Schema.Void, - error: RunnerProviderError, -}); - -export const RunnerStopSessionRpc = Rpc.make("runner.provider.stopSession", { - payload: Schema.Struct({ ...InstanceScoped, threadId: ThreadId }), - success: Schema.Void, - error: RunnerProviderError, -}); - -export const RunnerListSessionsRpc = Rpc.make("runner.provider.listSessions", { - payload: Schema.Struct({ ...InstanceScoped }), - success: Schema.Array(ProviderSession), - error: RunnerProviderError, -}); - -export const RunnerReadThreadRpc = Rpc.make("runner.provider.readThread", { - payload: Schema.Struct({ ...InstanceScoped, threadId: ThreadId }), - success: RunnerThreadSnapshot, - error: RunnerProviderError, -}); - -export const RunnerRollbackThreadRpc = Rpc.make("runner.provider.rollbackThread", { - payload: Schema.Struct({ ...InstanceScoped, threadId: ThreadId, numTurns: NonNegativeInt }), - success: RunnerThreadSnapshot, - error: RunnerProviderError, -}); - -/** Provider status/models snapshot as the runner's real driver sees it. */ -export const RunnerGetCapabilitiesRpc = Rpc.make("runner.provider.getCapabilities", { - payload: Schema.Struct({ ...InstanceScoped, refresh: Schema.optional(Schema.Boolean) }), - success: Schema.Struct({ - snapshot: ServerProvider, - sessionModelSwitch: Schema.Literals(["in-session", "unsupported"]), - }), - error: RunnerProviderError, -}); - -export const RunnerGenerateThreadTitleRpc = Rpc.make("runner.text.generateThreadTitle", { - payload: Schema.Struct({ - ...InstanceScoped, - cwd: Schema.String, - message: Schema.String, - previousTitle: Schema.optional(Schema.String), - attachments: Schema.optional(Schema.Array(ChatAttachment)), - modelSelection: ModelSelection, - }), - success: Schema.Struct({ title: Schema.String }), - error: TextGenerationError, -}); - -export const RunnerGenerateBranchNameRpc = Rpc.make("runner.text.generateBranchName", { - payload: Schema.Struct({ - ...InstanceScoped, - cwd: Schema.String, - message: Schema.String, - attachments: Schema.optional(Schema.Array(ChatAttachment)), - modelSelection: ModelSelection, - }), - success: Schema.Struct({ branch: Schema.String }), - error: TextGenerationError, -}); - -// ── Event outbox ───────────────────────────────────────────────────────── - -/** - * Replays every outboxed event with `sequence > afterSequence`, then tails - * live events. Idempotent: the hub may resubscribe from any acked point. - */ -export const RunnerSubscribeEventsRpc = Rpc.make("runner.events.subscribe", { - payload: Schema.Struct({ afterSequence: NonNegativeInt }), - success: RunnerEventEnvelope, - stream: true, -}); - -/** Marks every event with `sequence <= throughSequence` durable on the hub. */ -export const RunnerAckEventsRpc = Rpc.make("runner.events.ack", { - payload: Schema.Struct({ throughSequence: NonNegativeInt }), - success: Schema.Struct({ ackedSequence: NonNegativeInt, retained: NonNegativeInt }), -}); - -// ── Checkpoints (hidden git refs in the runner's checkout) ─────────────── - -export const RunnerIsGitRepositoryRpc = Rpc.make("runner.checkpoint.isGitRepository", { - payload: Schema.Struct({ cwd: Schema.String }), - success: Schema.Boolean, - error: VcsError, -}); - -export const RunnerCaptureCheckpointRpc = Rpc.make("runner.checkpoint.capture", { - payload: Schema.Struct({ cwd: Schema.String, checkpointRef: CheckpointRef }), - success: Schema.Void, - error: VcsError, -}); - -export const RunnerHasCheckpointRefRpc = Rpc.make("runner.checkpoint.hasRef", { - payload: Schema.Struct({ cwd: Schema.String, checkpointRef: CheckpointRef }), - success: Schema.Boolean, - error: VcsError, -}); - -export const RunnerRestoreCheckpointRpc = Rpc.make("runner.checkpoint.restore", { - payload: Schema.Struct({ - cwd: Schema.String, - checkpointRef: CheckpointRef, - fallbackToHead: Schema.optional(Schema.Boolean), - }), - success: Schema.Boolean, - error: VcsError, -}); - -export const RunnerDiffCheckpointsRpc = Rpc.make("runner.checkpoint.diff", { - payload: Schema.Struct({ - cwd: Schema.String, - fromCheckpointRef: CheckpointRef, - toCheckpointRef: CheckpointRef, - fallbackFromToHead: Schema.optional(Schema.Boolean), - ignoreWhitespace: Schema.Boolean, - }), - success: Schema.String, - error: VcsError, -}); - -export const RunnerDeleteCheckpointRefsRpc = Rpc.make("runner.checkpoint.deleteRefs", { - payload: Schema.Struct({ cwd: Schema.String, checkpointRefs: Schema.Array(CheckpointRef) }), - success: Schema.Void, - error: VcsError, -}); - -// ── Workspace ──────────────────────────────────────────────────────────── - -export const RunnerNormalizeWorkspaceRootRpc = Rpc.make("runner.workspace.normalizeRoot", { - payload: Schema.Struct({ - workspaceRoot: Schema.String, - createIfMissing: Schema.optional(Schema.Boolean), - }), - success: Schema.String, - error: RunnerWorkspaceError, -}); - -export const RunnerRpcGroup = RpcGroup.make( - RunnerHelloRpc, - RunnerStartSessionRpc, - RunnerSendTurnRpc, - RunnerInterruptTurnRpc, - RunnerRespondToRequestRpc, - RunnerRespondToUserInputRpc, - RunnerStopSessionRpc, - RunnerListSessionsRpc, - RunnerReadThreadRpc, - RunnerRollbackThreadRpc, - RunnerGetCapabilitiesRpc, - RunnerGenerateThreadTitleRpc, - RunnerGenerateBranchNameRpc, - RunnerSubscribeEventsRpc, - RunnerAckEventsRpc, - RunnerIsGitRepositoryRpc, - RunnerCaptureCheckpointRpc, - RunnerHasCheckpointRefRpc, - RunnerRestoreCheckpointRpc, - RunnerDiffCheckpointsRpc, - RunnerDeleteCheckpointRefsRpc, - RunnerNormalizeWorkspaceRootRpc, -); diff --git a/apps/server/src/runner/RunnerServer.ts b/apps/server/src/runner/RunnerServer.ts index a059636fc2e0..593352070bb1 100644 --- a/apps/server/src/runner/RunnerServer.ts +++ b/apps/server/src/runner/RunnerServer.ts @@ -1,21 +1,23 @@ /** - * RunnerServer - the runner build (prototype). + * RunnerServer - the `t3 runner` composition. * * A runner is the half of the T3 server that must live next to a checkout: - * real provider drivers, the checkpoint store, and workspace validation. It - * serves `RunnerRpcGroup` on `RUNNER_WS_PATH` and never runs orchestration, - * projections, or the client API; the hub does. + * the real provider drivers, git and checkpoints, terminals, workspace files, + * review and git actions for exactly one thread. It serves `RunnerRpcGroup` + * on `RUNNER_WS_PATH` and never runs orchestration, projections, auth, or the + * client API; the hub does. * - * Provider runtime events from every hosted instance are stamped with their - * instance id and appended to the durable `RunnerOutbox` before any hub sees - * them. + * Every provider runtime event from every hosted instance is stamped with its + * instance id and committed to the durable `RunnerOutbox` before any hub sees + * it. On graceful shutdown the runner stops its sessions while the event pumps + * still run, so the exit events reach the outbox and the hub settles the turn. * * @module runner/RunnerServer */ import * as NodeCrypto from "node:crypto"; import type { ProviderInstanceId } from "@t3tools/contracts"; -import * as Clock from "effect/Clock"; +import { RUNNER_WS_PATH, RunnerRpcGroup } from "@t3tools/contracts/runner"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Stream from "effect/Stream"; @@ -24,55 +26,65 @@ import { RpcSerialization, RpcServer } from "effect/unstable/rpc"; import * as CheckpointStore from "../checkpointing/CheckpointStore.ts"; import { ServerConfig } from "../config.ts"; -import { isGitRepository } from "../git/Utils.ts"; -import { ClaudeDriver, type ClaudeDriverEnv } from "../provider/Drivers/ClaudeDriver.ts"; -import { CodexDriver, type CodexDriverEnv } from "../provider/Drivers/CodexDriver.ts"; -import { ProviderAdapterValidationError } from "../provider/Errors.ts"; +import * as GitManager from "../git/GitManager.ts"; +import * as GitWorkflowService from "../git/GitWorkflowService.ts"; +import * as ProjectSetupScriptRunner from "../project/ProjectSetupScriptRunner.ts"; +import * as PortScanner from "../preview/PortScanner.ts"; +import * as ProcessRunner from "../processRunner.ts"; +import { resolveBuiltInDrivers } from "../provider/builtInDrivers.ts"; +import type { ProviderAdapterError } from "../provider/Errors.ts"; import * as ModelManifest from "../provider/ModelManifest.ts"; -import type { AnyProviderDriver } from "../provider/ProviderDriver.ts"; +import * as OpenCodeRuntime from "../provider/opencodeRuntime.ts"; import * as ProviderEventLoggers from "../provider/Layers/ProviderEventLoggers.ts"; import { makeProviderInstanceRegistryHydration } from "../provider/Layers/ProviderInstanceRegistryHydration.ts"; +import { ProviderRegistryLive } from "../provider/Layers/ProviderRegistry.ts"; import type { ProviderAdapterShape } from "../provider/Services/ProviderAdapter.ts"; import { ProviderInstanceRegistry } from "../provider/Services/ProviderInstanceRegistry.ts"; -import type { ProviderAdapterError } from "../provider/Errors.ts"; +import * as ReviewService from "../review/ReviewService.ts"; import { ApplicationObservabilityLive, BackgroundLayerLive, HttpServerLive, PlatformServicesLive, + PtyAdapterLive, + SourceControlProviderRegistryLayerLive, VcsDriverRegistryLayerLive, } from "../server.ts"; -import { TextGenerationError } from "@t3tools/contracts"; +import * as TerminalManager from "../terminal/Manager.ts"; +import * as TextGeneration from "../textGeneration/TextGeneration.ts"; +import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as VcsProcess from "../vcs/VcsProcess.ts"; +import * as VcsProvisioningService from "../vcs/VcsProvisioningService.ts"; +import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; +import * as WorkspaceFileSystem from "../workspace/WorkspaceFileSystem.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; +import * as RunnerCheckout from "./RunnerCheckout.ts"; +import { RunnerRpcHandlersLive, resolveRunnerBinding } from "./RunnerHandlers.ts"; import { RunnerOutbox, layer as RunnerOutboxLayer } from "./RunnerOutbox.ts"; -import { - RUNNER_PROTOCOL_VERSION, - RUNNER_WS_PATH, - RunnerRpcGroup, - RunnerWorkspaceError, -} from "./RunnerProtocol.ts"; +import * as RunnerProjectionSnapshotQuery from "./RunnerProjectionSnapshotQuery.ts"; -type RunnerDriversEnv = ClaudeDriverEnv | CodexDriverEnv; - -const RUNNER_DRIVERS: ReadonlyArray> = [ - ClaudeDriver, - CodexDriver, -]; - -/** `T3CODE_RUNNER_DRIVERS=claudeAgent,codex` selects which drivers this runner hosts. */ -const selectRunnerDrivers = () => { - const wanted = new Set( - (process.env.T3CODE_RUNNER_DRIVERS ?? "claudeAgent,codex") - .split(",") - .map((value) => value.trim()) - .filter((value) => value.length > 0), - ); - return RUNNER_DRIVERS.filter((driver) => wanted.has(driver.driverKind)); -}; +/** Time given to session-exit events to reach the outbox on graceful shutdown. */ +const SHUTDOWN_DRAIN = "300 millis"; -const RunnerRegistryLive = Layer.unwrap( - Effect.sync(() => makeProviderInstanceRegistryHydration(selectRunnerDrivers())), +/** + * `T3CODE_RUNNER_DRIVERS=claudeAgent,codex` limits the hosted drivers (for + * development and tests); by default a runner hosts every built-in driver. + */ +const RunnerProviderInstanceRegistryLive = Layer.unwrap( + Effect.gen(function* () { + const config = yield* ServerConfig; + const drivers = resolveBuiltInDrivers({ museCodeEnabled: config.museCodeEnabled }); + const wanted = new Set( + (process.env.T3CODE_RUNNER_DRIVERS ?? "") + .split(",") + .map((value) => value.trim()) + .filter((value) => value.length > 0), + ); + return makeProviderInstanceRegistryHydration( + wanted.size === 0 ? drivers : drivers.filter((driver) => wanted.has(driver.driverKind)), + ); + }), ); /** Appends every hosted adapter's runtime events to the outbox. */ @@ -82,14 +94,9 @@ const RunnerEventPumpLive = Layer.effectDiscard( const outbox = yield* RunnerOutbox; const subscribed = new Map>(); const reconcile = Effect.gen(function* () { - const instances = yield* registry.listInstances; - for (const instance of instances) { + for (const instance of yield* registry.listInstances) { if (subscribed.get(instance.instanceId) === instance.adapter) continue; subscribed.set(instance.instanceId, instance.adapter); - yield* Effect.logInfo("runner hosting provider instance", { - instanceId: instance.instanceId, - driver: instance.driverKind, - }); yield* Stream.runForEach(instance.adapter.streamEvents, (event) => outbox.append({ ...event, providerInstanceId: instance.instanceId }), ).pipe(Effect.forkScoped); @@ -101,17 +108,14 @@ const RunnerEventPumpLive = Layer.effectDiscard( Effect.forkScoped, ); // Registered after the pumps, so it runs first on shutdown: stopping the - // sessions while the pumps still run outboxes their exit events, and the - // hub learns the sessions ended instead of waiting on a silent turn. + // sessions while the pumps still run commits their exit events, and the + // hub learns the turns ended instead of waiting on silent sessions. yield* Effect.addFinalizer(() => Effect.gen(function* () { - const instances = yield* registry.listInstances; - yield* Effect.forEach( - instances, - (instance) => instance.adapter.stopAll().pipe(Effect.ignore), - { discard: true }, - ); - yield* Effect.sleep("300 millis"); + for (const instance of yield* registry.listInstances) { + yield* instance.adapter.stopAll().pipe(Effect.ignore); + } + yield* Effect.sleep(SHUTDOWN_DRAIN); yield* Effect.logInfo("runner drained provider sessions into the outbox", { ...(yield* outbox.stats), }); @@ -120,202 +124,25 @@ const RunnerEventPumpLive = Layer.effectDiscard( }), ); -const RunnerRpcHandlersLive = RunnerRpcGroup.toLayer( - Effect.gen(function* () { - const registry = yield* ProviderInstanceRegistry; - const checkpointStore = yield* CheckpointStore.CheckpointStore; - const workspacePaths = yield* WorkspacePaths.WorkspacePaths; - const outbox = yield* RunnerOutbox; - - const instanceOf = (instanceId: ProviderInstanceId) => - registry.getInstance(instanceId).pipe( - Effect.flatMap((instance) => - instance - ? Effect.succeed(instance) - : Effect.fail( - new ProviderAdapterValidationError({ - provider: String(instanceId), - operation: "runner.resolveInstance", - issue: `This runner does not host provider instance '${instanceId}'.`, - }), - ), - ), - ); - const adapterOf = (instanceId: ProviderInstanceId) => - instanceOf(instanceId).pipe(Effect.map((instance) => instance.adapter)); - /** Logs each state-changing call with its duration, for protocol evidence. */ - const logged = ( - method: string, - attributes: Record, - effect: Effect.Effect, - ) => - Effect.gen(function* () { - const startedAt = yield* Clock.currentTimeMillis; - const exit = yield* Effect.exit(effect); - yield* Effect.logInfo(`runner rpc ${method}`, { - ...attributes, - ok: exit._tag === "Success", - ms: (yield* Clock.currentTimeMillis) - startedAt, - }); - return yield* exit; - }); - const textGenerationOf = (instanceId: ProviderInstanceId, operation: string) => - instanceOf(instanceId).pipe( - Effect.map((instance) => instance.textGeneration), - Effect.mapError((error) => new TextGenerationError({ operation, detail: error.message })), - ); - - return RunnerRpcGroup.of({ - "runner.hello": () => - Effect.gen(function* () { - const stats = yield* outbox.stats; - const instances = yield* registry.listInstances; - return { - protocolVersion: RUNNER_PROTOCOL_VERSION, - runnerId: stats.runnerId, - bootId: stats.bootId, - headSequence: stats.headSequence, - ackedSequence: stats.ackedSequence, - instances: instances.map((instance) => instance.instanceId), - }; - }), - "runner.provider.startSession": ({ instanceId, input }) => - logged( - "startSession", - { - threadId: input.threadId, - cwd: input.cwd, - runtimeMode: input.runtimeMode, - resumeCursor: input.resumeCursor ?? null, - }, - adapterOf(instanceId).pipe(Effect.flatMap((adapter) => adapter.startSession(input))), - ), - "runner.provider.sendTurn": ({ instanceId, input }) => - logged( - "sendTurn", - { threadId: input.threadId }, - adapterOf(instanceId).pipe(Effect.flatMap((adapter) => adapter.sendTurn(input))), - ), - "runner.provider.interruptTurn": ({ instanceId, threadId, turnId }) => - logged( - "interruptTurn", - { threadId, turnId }, - adapterOf(instanceId).pipe( - Effect.flatMap((adapter) => adapter.interruptTurn(threadId, turnId)), - ), - ), - "runner.provider.respondToRequest": ({ instanceId, threadId, requestId, decision }) => - logged( - "respondToRequest", - { threadId, requestId, decision }, - adapterOf(instanceId).pipe( - Effect.flatMap((adapter) => adapter.respondToRequest(threadId, requestId, decision)), - ), - ), - "runner.provider.respondToUserInput": ({ instanceId, threadId, requestId, answers }) => - adapterOf(instanceId).pipe( - Effect.flatMap((adapter) => adapter.respondToUserInput(threadId, requestId, answers)), - ), - "runner.provider.stopSession": ({ instanceId, threadId }) => - logged( - "stopSession", - { threadId }, - adapterOf(instanceId).pipe(Effect.flatMap((adapter) => adapter.stopSession(threadId))), - ), - "runner.provider.listSessions": ({ instanceId }) => - adapterOf(instanceId).pipe(Effect.flatMap((adapter) => adapter.listSessions())), - "runner.provider.readThread": ({ instanceId, threadId }) => - adapterOf(instanceId).pipe(Effect.flatMap((adapter) => adapter.readThread(threadId))), - "runner.provider.rollbackThread": ({ instanceId, threadId, numTurns }) => - adapterOf(instanceId).pipe( - Effect.flatMap((adapter) => adapter.rollbackThread(threadId, numTurns)), - ), - "runner.provider.getCapabilities": ({ instanceId, refresh }) => - instanceOf(instanceId).pipe( - Effect.flatMap((instance) => - (refresh ? instance.snapshot.refresh : instance.snapshot.getSnapshot).pipe( - Effect.map((snapshot) => ({ - snapshot, - sessionModelSwitch: instance.adapter.capabilities.sessionModelSwitch, - })), - ), - ), - ), - "runner.text.generateThreadTitle": ({ instanceId, ...request }) => - logged( - "text.generateThreadTitle", - { cwd: request.cwd, model: request.modelSelection.model }, - textGenerationOf(instanceId, "generateThreadTitle").pipe( - Effect.flatMap((textGeneration) => textGeneration.generateThreadTitle(request)), - ), - ), - "runner.text.generateBranchName": ({ instanceId, ...request }) => - textGenerationOf(instanceId, "generateBranchName").pipe( - Effect.flatMap((textGeneration) => textGeneration.generateBranchName(request)), - ), - "runner.events.subscribe": ({ afterSequence }) => outbox.subscribe(afterSequence), - "runner.events.ack": ({ throughSequence }) => outbox.ack(throughSequence), - // Same `.git` presence check the in-process reactors used, so hub mode - // keeps the exact checkpoint eligibility rule. - "runner.checkpoint.isGitRepository": ({ cwd }) => Effect.sync(() => isGitRepository(cwd)), - "runner.checkpoint.capture": (input) => - logged( - "checkpoint.capture", - { checkpointRef: input.checkpointRef }, - checkpointStore.captureCheckpoint(input), - ), - "runner.checkpoint.hasRef": (input) => checkpointStore.hasCheckpointRef(input), - "runner.checkpoint.restore": ({ cwd, checkpointRef, fallbackToHead }) => - checkpointStore.restoreCheckpoint({ - cwd, - checkpointRef, - ...(fallbackToHead !== undefined ? { fallbackToHead } : {}), - }), - "runner.checkpoint.diff": ({ fallbackFromToHead, ...input }) => - checkpointStore.diffCheckpoints({ - ...input, - ...(fallbackFromToHead !== undefined ? { fallbackFromToHead } : {}), - }), - "runner.checkpoint.deleteRefs": (input) => checkpointStore.deleteCheckpointRefs(input), - "runner.workspace.normalizeRoot": ({ workspaceRoot, createIfMissing }) => - workspacePaths - .normalizeWorkspaceRoot( - workspaceRoot, - createIfMissing !== undefined ? { createIfMissing } : undefined, - ) - .pipe( - Effect.mapError( - (error) => - new RunnerWorkspaceError({ - operation: "normalizeRoot", - workspaceRoot, - reason: - error._tag === "WorkspaceRootNotExistsError" - ? "not-exists" - : error._tag === "WorkspaceRootNotDirectoryError" - ? "not-directory" - : error._tag === "WorkspaceRootCreateFailedError" - ? "create-failed" - : "stat-failed", - detail: error.message, - }), - ), - ), - }); - }), -); - -function tokenMatches(presented: string | null, expected: string): boolean { - if (presented === null) return false; - const left = Buffer.from(presented); - const right = Buffer.from(expected); - return left.length === right.length && NodeCrypto.timingSafeEqual(left, right); -} +const bearerMatches = (header: string | undefined, expected: string): boolean => { + if (header === undefined || !header.startsWith("Bearer ")) return false; + const presented = Buffer.from(header.slice("Bearer ".length)); + const wanted = Buffer.from(expected); + return presented.length === wanted.length && NodeCrypto.timingSafeEqual(presented, wanted); +}; +/** + * The runner protocol endpoint. The hub authenticates with + * `Authorization: Bearer T3CODE_RUNNER_TOKEN`; without a configured token the + * endpoint is open, which is only acceptable on loopback in development. + */ const RunnerRouteLive = Layer.unwrap( Effect.gen(function* () { const config = yield* ServerConfig; const expectedToken = config.runnerToken; + if (!expectedToken) { + yield* Effect.logWarning("runner has no T3CODE_RUNNER_TOKEN; the protocol endpoint is open"); + } const handlers = yield* Layer.build( RunnerRpcHandlersLive.pipe(Layer.provideMerge(RpcSerialization.layerJson)), ); @@ -323,33 +150,82 @@ const RunnerRouteLive = Layer.unwrap( disableTracing: true, }).pipe(Effect.provide(handlers)); return HttpRouter.add("GET", RUNNER_WS_PATH, (request) => - Effect.gen(function* () { - const token = new URL(request.url, "http://runner.local").searchParams.get("token"); - if (expectedToken && !tokenMatches(token, expectedToken)) { - return HttpServerResponse.text("unauthorized", { status: 401 }); - } - return yield* rpcHttpEffect; - }), + expectedToken && !bearerMatches(request.headers.authorization, expectedToken) + ? Effect.succeed(HttpServerResponse.text("unauthorized", { status: 401 })) + : rpcHttpEffect, ); }), ); +const RunnerTerminalLive = TerminalManager.layer.pipe( + Layer.provide(PtyAdapterLive), + Layer.provide(PortScanner.layer.pipe(Layer.provide(ProcessRunner.layer))), +); + +const RunnerWorkspaceLive = Layer.mergeAll( + WorkspaceFileSystem.layer.pipe( + Layer.provide(WorkspaceEntries.layer.pipe(Layer.provide(WorkspacePaths.layer))), + ), + WorkspaceEntries.layer, +).pipe(Layer.provideMerge(WorkspacePaths.layer)); + +const RunnerGitLive = Layer.mergeAll( + GitWorkflowService.layer, + ReviewService.layer, + VcsProvisioningService.layer, + CheckpointStore.layer, + RunnerCheckout.layer, +).pipe( + Layer.provideMerge( + GitManager.layer.pipe( + Layer.provideMerge(ProjectSetupScriptRunner.layer), + Layer.provideMerge(TextGeneration.layer), + Layer.provideMerge(SourceControlProviderRegistryLayerLive), + ), + ), + Layer.provideMerge(GitVcsDriver.layer), + Layer.provideMerge(VcsDriverRegistryLayerLive), +); + +const RunnerCheckoutServicesLive = Layer.mergeAll( + VcsStatusBroadcaster.layer.pipe(Layer.provideMerge(RunnerGitLive)), + RunnerWorkspaceLive, +).pipe( + Layer.provideMerge(RunnerTerminalLive), + Layer.provideMerge(RunnerProjectionSnapshotQuery.layer), +); + const RunnerServicesLive = Layer.mergeAll(RunnerEventPumpLive, RunnerRouteLive).pipe( Layer.provideMerge(RunnerOutboxLayer), - Layer.provideMerge(RunnerRegistryLive), + Layer.provideMerge(RunnerCheckoutServicesLive), + Layer.provideMerge(ProviderRegistryLive), + Layer.provideMerge(RunnerProviderInstanceRegistryLive), Layer.provideMerge( Layer.mergeAll( - CheckpointStore.layer.pipe(Layer.provide(VcsDriverRegistryLayerLive)), - WorkspacePaths.layer, + ProviderEventLoggers.layer, + ModelManifest.layer, + OpenCodeRuntime.OpenCodeRuntimeLive, ), ), - Layer.provideMerge(Layer.mergeAll(ProviderEventLoggers.layer, ModelManifest.layer)), Layer.provideMerge(BackgroundLayerLive), ); +/** Fails fast with a clear message when the runner is not bound to a thread. */ +const RunnerBindingCheckLive = Layer.effectDiscard( + resolveRunnerBinding.pipe( + Effect.tap((binding) => + Effect.logInfo("runner serving thread", { + threadId: binding.threadId, + checkout: binding.checkout, + }), + ), + ), +); + export const makeRunnerLayer = HttpRouter.serve(RunnerServicesLive, { disableLogger: true, }).pipe( + Layer.provideMerge(RunnerBindingCheckLive), Layer.provideMerge(HttpServerLive), Layer.provide(ApplicationObservabilityLive), Layer.provideMerge(FetchHttpClient.layer), diff --git a/apps/server/src/runner/hub/HubLayers.ts b/apps/server/src/runner/hub/HubLayers.ts deleted file mode 100644 index 03a5dfcbe1a3..000000000000 --- a/apps/server/src/runner/hub/HubLayers.ts +++ /dev/null @@ -1,225 +0,0 @@ -/** - * HubLayers - services a checkout-less hub substitutes for host-bound ones. - * - * Each export replaces one service whose local implementation would stat, - * read, or run git against a thread checkout that only exists on the runner: - * - * - delegated: CheckpointStore, CheckoutGitProbe, WorkspacePaths root - * validation (runner RPCs, same cwd strings). - * - stubbed: RepositoryIdentityResolver (null identity), - * WorkspaceEntries (empty index, refresh is a no-op), VcsStatusBroadcaster - * (fails; the runner should push status to a hub cache in production). - * - * @module runner/hub/HubLayers - */ -import { GitManagerError, VcsRepositoryDetectionError, type VcsError } from "@t3tools/contracts"; -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; -import * as Stream from "effect/Stream"; - -import { CheckpointStore } from "../../checkpointing/CheckpointStore.ts"; -import { CheckoutGitProbe } from "../../git/CheckoutGitProbe.ts"; -import { RepositoryIdentityResolver } from "../../project/RepositoryIdentityResolver.ts"; -import { VcsStatusBroadcaster } from "../../vcs/VcsStatusBroadcaster.ts"; -import { WorkspaceEntries } from "../../workspace/WorkspaceEntries.ts"; -import * as WorkspacePaths from "../../workspace/WorkspacePaths.ts"; -import { RunnerClient } from "./RunnerClient.ts"; - -const describe = (cause: unknown): string => - cause && typeof cause === "object" && "message" in cause - ? String((cause as { message: unknown }).message) - : String(cause); - -const VCS_ERROR_TAGS = new Set([ - "VcsProcessSpawnError", - "VcsProcessExitError", - "VcsProcessTimeoutError", - "VcsProcessStdinWriteError", - "VcsProcessOutputReadError", - "VcsProcessOutputLimitError", - "VcsProcessMissingExitCodeError", - "VcsRepositoryDetectionError", - "VcsUnsupportedOperationError", -]); - -const toVcsError = - (operation: string, cwd: string) => - (error: { readonly _tag: string }): VcsError => - VCS_ERROR_TAGS.has(error._tag) - ? (error as VcsError) - : new VcsRepositoryDetectionError({ - operation, - cwd, - detail: `runner call failed: ${describe(error)}`, - }); - -export const remoteCheckpointStoreLayer = Layer.effect( - CheckpointStore, - Effect.gen(function* () { - const runner = yield* RunnerClient; - return CheckpointStore.of({ - isGitRepository: (cwd) => - runner - .use((client) => client["runner.checkpoint.isGitRepository"]({ cwd })) - .pipe(Effect.mapError(toVcsError("CheckpointStore.isGitRepository", cwd))), - captureCheckpoint: (input) => - runner - .use((client) => - client["runner.checkpoint.capture"]({ - cwd: input.cwd, - checkpointRef: input.checkpointRef, - }), - ) - .pipe(Effect.mapError(toVcsError("CheckpointStore.captureCheckpoint", input.cwd))), - hasCheckpointRef: (input) => - runner - .use((client) => - client["runner.checkpoint.hasRef"]({ - cwd: input.cwd, - checkpointRef: input.checkpointRef, - }), - ) - .pipe(Effect.mapError(toVcsError("CheckpointStore.hasCheckpointRef", input.cwd))), - restoreCheckpoint: (input) => - runner - .use((client) => - client["runner.checkpoint.restore"]({ - cwd: input.cwd, - checkpointRef: input.checkpointRef, - ...(input.fallbackToHead !== undefined - ? { fallbackToHead: input.fallbackToHead } - : {}), - }), - ) - .pipe(Effect.mapError(toVcsError("CheckpointStore.restoreCheckpoint", input.cwd))), - diffCheckpoints: (input) => - runner - .use((client) => - client["runner.checkpoint.diff"]({ - cwd: input.cwd, - fromCheckpointRef: input.fromCheckpointRef, - toCheckpointRef: input.toCheckpointRef, - ...(input.fallbackFromToHead !== undefined - ? { fallbackFromToHead: input.fallbackFromToHead } - : {}), - ignoreWhitespace: input.ignoreWhitespace, - }), - ) - .pipe(Effect.mapError(toVcsError("CheckpointStore.diffCheckpoints", input.cwd))), - deleteCheckpointRefs: (input) => - runner - .use((client) => - client["runner.checkpoint.deleteRefs"]({ - cwd: input.cwd, - checkpointRefs: input.checkpointRefs, - }), - ) - .pipe(Effect.mapError(toVcsError("CheckpointStore.deleteCheckpointRefs", input.cwd))), - }); - }), -); - -export const remoteCheckoutGitProbeLayer = Layer.effect( - CheckoutGitProbe, - Effect.gen(function* () { - const runner = yield* RunnerClient; - return (cwd: string) => - runner - .use((client) => client["runner.checkpoint.isGitRepository"]({ cwd })) - .pipe( - Effect.catch((error) => - Effect.logWarning("runner git probe failed; treating checkout as non-git", { - cwd, - detail: describe(error), - }).pipe(Effect.as(false)), - ), - ); - }), -); - -export const remoteWorkspacePathsLayer = Layer.effect( - WorkspacePaths.WorkspacePaths, - Effect.gen(function* () { - const runner = yield* RunnerClient; - // Relative-path resolution is pure; only root validation touches disk. - const local = yield* WorkspacePaths.make; - return WorkspacePaths.WorkspacePaths.of({ - resolveRelativePathWithinRoot: local.resolveRelativePathWithinRoot, - normalizeWorkspaceRoot: (workspaceRoot, options) => - runner - .use((client) => - client["runner.workspace.normalizeRoot"]({ - workspaceRoot, - ...(options?.createIfMissing !== undefined - ? { createIfMissing: options.createIfMissing } - : {}), - }), - ) - .pipe( - Effect.mapError((error) => { - if (error._tag === "RunnerWorkspaceError") { - switch (error.reason) { - case "not-exists": - return new WorkspacePaths.WorkspaceRootNotExistsError({ - workspaceRoot, - normalizedWorkspaceRoot: workspaceRoot, - }); - case "not-directory": - return new WorkspacePaths.WorkspaceRootNotDirectoryError({ - workspaceRoot, - normalizedWorkspaceRoot: workspaceRoot, - }); - case "create-failed": - return new WorkspacePaths.WorkspaceRootCreateFailedError({ - workspaceRoot, - normalizedWorkspaceRoot: workspaceRoot, - cause: error, - }); - default: - break; - } - } - return new WorkspacePaths.WorkspaceRootStatFailedError({ - workspaceRoot, - normalizedWorkspaceRoot: workspaceRoot, - phase: "validate-existing", - cause: error, - }); - }), - ), - }); - }), -); - -export const hubWorkspaceEntriesLayer = Layer.succeed( - WorkspaceEntries, - WorkspaceEntries.of({ - browse: (input) => Effect.succeed({ parentPath: input.partialPath, entries: [] }), - list: () => Effect.succeed({ entries: [], truncated: false }), - search: () => Effect.succeed({ entries: [], truncated: false }), - searchContents: () => Effect.succeed({ matches: [], truncated: false }), - refresh: () => Effect.void, - }), -); - -export const hubRepositoryIdentityResolverLayer = Layer.succeed( - RepositoryIdentityResolver, - RepositoryIdentityResolver.of({ resolve: () => Effect.succeed(null) }), -); - -const vcsStatusUnavailable = (operation: string, cwd: string) => - new GitManagerError({ - operation, - cwd, - detail: "VCS status lives on the thread's runner and is not delegated in prototype 2.", - }); - -export const hubVcsStatusBroadcasterLayer = Layer.succeed( - VcsStatusBroadcaster, - VcsStatusBroadcaster.of({ - getStatus: (input) => Effect.fail(vcsStatusUnavailable("getStatus", input.cwd)), - refreshLocalStatus: (cwd) => Effect.fail(vcsStatusUnavailable("refreshLocalStatus", cwd)), - refreshStatus: (cwd) => Effect.fail(vcsStatusUnavailable("refreshStatus", cwd)), - streamStatus: (input) => Stream.fail(vcsStatusUnavailable("streamStatus", input.cwd)), - }), -); diff --git a/apps/server/src/runner/hub/RemoteProviderDriver.ts b/apps/server/src/runner/hub/RemoteProviderDriver.ts deleted file mode 100644 index 18093491149e..000000000000 --- a/apps/server/src/runner/hub/RemoteProviderDriver.ts +++ /dev/null @@ -1,314 +0,0 @@ -/** - * RemoteProviderDriver - hub-side provider driver that runs nothing locally. - * - * Wraps a built-in driver's identity (driver kind, config schema, defaults) so - * the hub keeps the same instance ids, model selections and session directory - * rows, while the adapter forwards every call to the runner that owns the - * checkout. `ProviderService` above it is unchanged: it still persists resume - * cursors, recovers sessions and serializes per-thread lifecycle work. - * - * `listSessions`/`hasSession` ask the runner so that a hub restart adopts the - * runner's still-running sessions instead of marking them orphaned; when the - * runner is unreachable they report nothing and `ProviderService` recovers - * through the persisted resume cursor on the next turn. - * - * @module runner/hub/RemoteProviderDriver - */ -import { - type ProviderDriverKind, - type ProviderInstanceId, - type ServerProvider, - TextGenerationError, -} from "@t3tools/contracts"; -import * as Effect from "effect/Effect"; -import * as Option from "effect/Option"; -import * as PubSub from "effect/PubSub"; -import * as Ref from "effect/Ref"; -import * as Stream from "effect/Stream"; - -import { - type ProviderAdapterError, - ProviderAdapterRequestError, - ProviderDriverError, -} from "../../provider/Errors.ts"; -import { - defaultProviderContinuationIdentity, - type AnyProviderDriver, - type ProviderInstance, -} from "../../provider/ProviderDriver.ts"; -import type { - ProviderAdapterShape, - ProviderThreadSnapshot, -} from "../../provider/Services/ProviderAdapter.ts"; -import type { ServerProviderShape } from "../../provider/Services/ServerProvider.ts"; -import type { TextGenerationShape } from "../../textGeneration/TextGeneration.ts"; -import { buildUnavailableProviderSnapshot } from "../../provider/unavailableProviderSnapshot.ts"; -import { RunnerClient, type RunnerRpcClient } from "./RunnerClient.ts"; - -const describe = (cause: unknown): string => { - if (cause && typeof cause === "object" && "message" in cause) { - return String((cause as { message: unknown }).message); - } - return String(cause); -}; - -export function makeRemoteProviderDriver( - base: AnyProviderDriver, -): AnyProviderDriver { - const driverKind: ProviderDriverKind = base.driverKind; - return { - driverKind, - metadata: base.metadata, - configSchema: base.configSchema, - defaultConfig: base.defaultConfig as () => Config, - create: ({ instanceId, displayName, accentColor, enabled }) => - Effect.gen(function* () { - const runnerOption = yield* Effect.serviceOption(RunnerClient); - if (Option.isNone(runnerOption) || !runnerOption.value.enabled) { - return yield* new ProviderDriverError({ - driver: driverKind, - instanceId, - detail: "Hub mode requires a configured runner.", - }); - } - const runner = runnerOption.value; - const adapter = makeRemoteAdapter({ runner, driverKind, instanceId }); - const snapshot = yield* makeRemoteSnapshot({ - runner, - driverKind, - instanceId, - displayName, - accentColor, - }); - const textGeneration = makeRemoteTextGeneration({ runner, instanceId }); - return { - instanceId, - driverKind, - continuationIdentity: defaultProviderContinuationIdentity({ driverKind, instanceId }), - displayName, - accentColor, - enabled, - snapshot, - adapter, - textGeneration, - } satisfies ProviderInstance; - }), - }; -} - -function makeRemoteAdapter(input: { - readonly runner: RunnerClient["Service"]; - readonly driverKind: ProviderDriverKind; - readonly instanceId: ProviderInstanceId; -}): ProviderAdapterShape { - const { runner, driverKind, instanceId } = input; - - const call = ( - method: string, - f: (client: RunnerRpcClient) => Effect.Effect, - ): Effect.Effect => - runner.use(f).pipe( - Effect.mapError((error): ProviderAdapterError => { - switch (error._tag) { - case "ProviderAdapterValidationError": - case "ProviderAdapterSessionNotFoundError": - case "ProviderAdapterSessionClosedError": - case "ProviderAdapterRequestError": - case "ProviderAdapterProcessError": - return error as ProviderAdapterError; - default: - return new ProviderAdapterRequestError({ - provider: driverKind, - method, - detail: `runner call failed: ${describe(error)}`, - cause: error, - }); - } - }), - ); - - const listSessions = () => - runner.hello - .pipe( - Effect.map((hello) => hello.instances.includes(instanceId)), - Effect.orElseSucceed(() => true), - Effect.flatMap((hosted) => - hosted - ? call("listSessions", (client) => - client["runner.provider.listSessions"]({ instanceId }), - ) - : Effect.succeed([]), - ), - ) - .pipe( - Effect.catch((error) => - Effect.logWarning("remote listSessions unavailable; reporting none", { - instanceId, - detail: error.message, - }).pipe(Effect.as([])), - ), - ); - - const toThreadSnapshot = (snapshot: { - readonly threadId: ProviderThreadSnapshot["threadId"]; - readonly turns: ReadonlyArray<{ readonly id: string; readonly items: ReadonlyArray }>; - }): ProviderThreadSnapshot => snapshot as ProviderThreadSnapshot; - - return { - provider: driverKind, - // Model switching is a property of the runner's real adapter; the hub - // treats it as in-session and lets the runner reject when unsupported. - capabilities: { sessionModelSwitch: "in-session" }, - startSession: (sessionInput) => - call("startSession", (client) => - client["runner.provider.startSession"]({ instanceId, input: sessionInput }), - ), - sendTurn: (turnInput) => - call("sendTurn", (client) => - client["runner.provider.sendTurn"]({ instanceId, input: turnInput }), - ), - interruptTurn: (threadId, turnId) => - call("interruptTurn", (client) => - client["runner.provider.interruptTurn"]({ - instanceId, - threadId, - ...(turnId !== undefined ? { turnId } : {}), - }), - ), - respondToRequest: (threadId, requestId, decision) => - call("respondToRequest", (client) => - client["runner.provider.respondToRequest"]({ instanceId, threadId, requestId, decision }), - ), - respondToUserInput: (threadId, requestId, answers) => - call("respondToUserInput", (client) => - client["runner.provider.respondToUserInput"]({ instanceId, threadId, requestId, answers }), - ), - stopSession: (threadId) => - call("stopSession", (client) => - client["runner.provider.stopSession"]({ instanceId, threadId }), - ), - listSessions, - hasSession: (threadId) => - listSessions().pipe( - Effect.map((sessions) => sessions.some((session) => session.threadId === threadId)), - ), - readThread: (threadId) => - call("readThread", (client) => - client["runner.provider.readThread"]({ instanceId, threadId }), - ).pipe(Effect.map(toThreadSnapshot)), - rollbackThread: (threadId, numTurns) => - call("rollbackThread", (client) => - client["runner.provider.rollbackThread"]({ instanceId, threadId, numTurns }), - ).pipe(Effect.map(toThreadSnapshot)), - // A hub shutting down must not stop sessions it does not host; the runner - // keeps them running and outboxes their events until the hub returns. - stopAll: () => Effect.void, - streamEvents: runner.eventsFor(instanceId), - }; -} - -const makeRemoteSnapshot = (input: { - readonly runner: RunnerClient["Service"]; - readonly driverKind: ProviderDriverKind; - readonly instanceId: ProviderInstanceId; - readonly displayName: string | undefined; - readonly accentColor: string | undefined; -}) => - Effect.gen(function* () { - const { runner, driverKind, instanceId } = input; - const pending = yield* buildUnavailableProviderSnapshot({ - driverKind, - instanceId, - displayName: input.displayName, - accentColor: input.accentColor, - reason: "Waiting for the thread machine to report this provider.", - }); - const current = yield* Ref.make(pending); - const changes = yield* PubSub.unbounded(); - - const fetch = (refresh: boolean) => - runner - .use((client) => - client["runner.provider.getCapabilities"]({ - instanceId, - ...(refresh ? { refresh } : {}), - }), - ) - .pipe( - Effect.map((result) => result.snapshot), - Effect.catch((error) => - buildUnavailableProviderSnapshot({ - driverKind, - instanceId, - displayName: input.displayName, - accentColor: input.accentColor, - reason: `Runner did not report this provider: ${describe(error)}`, - }), - ), - Effect.tap((snapshot) => Ref.set(current, snapshot)), - Effect.tap((snapshot) => PubSub.publish(changes, snapshot)), - ); - - // Populate in the background so hub startup never waits on a machine. - yield* fetch(false).pipe(Effect.forkScoped); - - return { - maintenanceCapabilities: { provider: driverKind, packageName: null, update: null }, - getSnapshot: Ref.get(current), - refresh: fetch(true), - streamChanges: Stream.fromPubSub(changes), - } satisfies ServerProviderShape; - }); - -function makeRemoteTextGeneration(input: { - readonly runner: RunnerClient["Service"]; - readonly instanceId: ProviderInstanceId; -}): TextGenerationShape { - const { runner, instanceId } = input; - const toTextGenerationError = (operation: string) => (error: { readonly _tag: string }) => - error._tag === "TextGenerationError" - ? (error as TextGenerationError) - : new TextGenerationError({ - operation, - detail: `runner call failed: ${describe(error)}`, - }); - const unsupported = (operation: string) => - Effect.fail( - new TextGenerationError({ - operation, - detail: "Not delegated to the runner in prototype 2.", - }), - ); - return { - generateThreadTitle: (request) => - runner - .use((client) => - client["runner.text.generateThreadTitle"]({ - instanceId, - cwd: request.cwd, - message: request.message, - ...(request.previousTitle !== undefined - ? { previousTitle: request.previousTitle } - : {}), - ...(request.attachments !== undefined ? { attachments: request.attachments } : {}), - modelSelection: request.modelSelection, - }), - ) - .pipe(Effect.mapError(toTextGenerationError("generateThreadTitle"))), - generateBranchName: (request) => - runner - .use((client) => - client["runner.text.generateBranchName"]({ - instanceId, - cwd: request.cwd, - message: request.message, - ...(request.attachments !== undefined ? { attachments: request.attachments } : {}), - modelSelection: request.modelSelection, - }), - ) - .pipe(Effect.mapError(toTextGenerationError("generateBranchName"))), - generateCommitMessage: () => unsupported("generateCommitMessage"), - generatePrContent: () => unsupported("generatePrContent"), - generateStructured: () => unsupported("generateStructured"), - }; -} diff --git a/apps/server/src/runner/hub/RunnerClient.ts b/apps/server/src/runner/hub/RunnerClient.ts deleted file mode 100644 index aeecb7ca79ff..000000000000 --- a/apps/server/src/runner/hub/RunnerClient.ts +++ /dev/null @@ -1,313 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off -- The delivery cursor is a tiny synchronous state file. -/** - * RunnerClient - the hub's connection to a runner (prototype). - * - * Owns one supervised WebSocket RPC connection to the configured runner, - * reconnecting with backoff. Calls wait (bounded) for a live connection. - * - * Event delivery: once the hub's ingestion is running (`startDelivery`), the - * client subscribes to the runner outbox from its durable cursor, drops any - * sequence it already delivered, and fans events out to the remote adapters. - * `ackDelivered` persists the cursor and lets the runner compact; the caller - * invokes it only after ingestion has drained, so an acknowledged event has - * been projected. - * - * In production the hub would route by thread to that thread's machine; the - * prototype has one runner URL for the whole hub. - * - * @module runner/hub/RunnerClient - */ -import * as NodeFS from "node:fs"; -import * as NodePath from "node:path"; - -import type { ProviderInstanceId, ProviderRuntimeEvent } from "@t3tools/contracts"; -import * as NodeSocket from "@effect/platform-node/NodeSocket"; -import * as Context from "effect/Context"; -import * as Deferred from "effect/Deferred"; -import * as Duration from "effect/Duration"; -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; -import * as PubSub from "effect/PubSub"; -import * as Ref from "effect/Ref"; -import * as Schema from "effect/Schema"; -import * as Stream from "effect/Stream"; -import { RpcClient, RpcSerialization } from "effect/unstable/rpc"; -import type { RpcClientError } from "effect/unstable/rpc/RpcClientError"; -import * as Socket from "effect/unstable/socket/Socket"; - -import { ServerConfig } from "../../config.ts"; -import { type RunnerHello, RunnerRpcGroup } from "../RunnerProtocol.ts"; - -export type RunnerRpcClient = RpcClient.FromGroup; - -export class RunnerUnavailableError extends Schema.TaggedErrorClass()( - "RunnerUnavailableError", - { detail: Schema.String }, -) { - override get message(): string { - return `Runner unavailable: ${this.detail}`; - } -} - -export interface RunnerDeliveryState { - readonly runnerId: string | null; - readonly bootId: string | null; - readonly deliveredSequence: number; - readonly ackedSequence: number; - readonly deliveredCount: number; - readonly duplicateCount: number; - readonly connectCount: number; -} - -export class RunnerClient extends Context.Service< - RunnerClient, - { - readonly enabled: boolean; - /** Run one call against the live connection, waiting briefly for one. */ - readonly use: ( - f: (client: RunnerRpcClient) => Effect.Effect, - ) => Effect.Effect; - readonly hello: Effect.Effect; - /** Provider runtime events for one instance, in runner sequence order. */ - readonly eventsFor: (instanceId: ProviderInstanceId) => Stream.Stream; - /** Opens the outbox subscription. Call once hub ingestion is consuming. */ - readonly startDelivery: Effect.Effect; - readonly deliveryState: Effect.Effect; - /** Persist the cursor and acknowledge everything delivered so far. */ - readonly ackDelivered: (throughSequence: number) => Effect.Effect; - } ->()("t3/runner/hub/RunnerClient") {} - -const CONNECT_WAIT = Duration.seconds(20); -const RECONNECT_DELAY = Duration.seconds(1); - -interface Connection { - readonly client: RunnerRpcClient; - readonly hello: RunnerHello; -} - -const PersistedCursorJson = Schema.fromJsonString( - Schema.Struct({ runnerId: Schema.NullOr(Schema.String), ackedSequence: Schema.Number }), -); -type PersistedCursor = typeof PersistedCursorJson.Type; -const decodeCursor = Schema.decodeUnknownSync(PersistedCursorJson); -const encodeCursor = Schema.encodeSync(PersistedCursorJson); - -function readCursor(path: string): PersistedCursor { - try { - return decodeCursor(NodeFS.readFileSync(path, "utf8")); - } catch { - return { runnerId: null, ackedSequence: 0 }; - } -} - -function writeCursor(path: string, cursor: PersistedCursor): void { - const tmp = `${path}.tmp`; - NodeFS.writeFileSync(tmp, encodeCursor(cursor)); - NodeFS.renameSync(tmp, path); -} - -const makeLive = (runnerUrl: string) => - Effect.gen(function* () { - const config = yield* ServerConfig; - const cursorPath = NodePath.join(config.stateDir, "runner-client-cursor.json"); - const persisted = readCursor(cursorPath); - const url = new URL(runnerUrl); - if (config.runnerToken) url.searchParams.set("token", config.runnerToken); - const socketUrl = url.toString(); - - const events = yield* PubSub.unbounded(); - const readyRef = yield* Ref.make(yield* Deferred.make()); - const deliveryEnabled = yield* Deferred.make(); - const state = yield* Ref.make({ - runnerId: persisted.runnerId, - bootId: null, - deliveredSequence: persisted.ackedSequence, - ackedSequence: persisted.ackedSequence, - deliveredCount: 0, - duplicateCount: 0, - connectCount: 0, - }); - - const awaitConnection = Ref.get(readyRef).pipe( - Effect.flatMap(Deferred.await), - Effect.timeoutOrElse({ - duration: CONNECT_WAIT, - orElse: () => - Effect.fail( - new RunnerUnavailableError({ - detail: `no connection to ${runnerUrl} within ${Duration.format(CONNECT_WAIT)}`, - }), - ), - }), - ); - - const pumpEvents = (client: RunnerRpcClient) => - Effect.gen(function* () { - yield* Deferred.await(deliveryEnabled); - const { deliveredSequence } = yield* Ref.get(state); - yield* Effect.logInfo("runner event subscription opened", { - afterSequence: deliveredSequence, - }); - yield* client["runner.events.subscribe"]({ afterSequence: deliveredSequence }).pipe( - Stream.runForEach((envelope) => - Ref.modify(state, (current) => { - if (envelope.sequence <= current.deliveredSequence) { - return [false, { ...current, duplicateCount: current.duplicateCount + 1 }] as const; - } - return [ - true, - { - ...current, - deliveredSequence: envelope.sequence, - deliveredCount: current.deliveredCount + 1, - }, - ] as const; - }).pipe( - Effect.flatMap((fresh) => - fresh ? PubSub.publish(events, envelope.event).pipe(Effect.asVoid) : Effect.void, - ), - ), - ), - ); - }); - - const connectOnce = Effect.scoped( - Effect.gen(function* () { - const disconnected = yield* Deferred.make(); - const hooks = RpcClient.ConnectionHooks.of({ - onConnect: Effect.void, - onDisconnect: Deferred.succeed(disconnected, undefined).pipe(Effect.asVoid), - }); - const protocol = yield* Layer.build( - Layer.effect( - RpcClient.Protocol, - RpcClient.makeProtocolSocket({ retryTransientErrors: false }), - ).pipe( - Layer.provide( - Layer.mergeAll( - Socket.layerWebSocket(socketUrl, { openTimeout: Duration.seconds(5) }).pipe( - Layer.provide(NodeSocket.layerWebSocketConstructor), - ), - RpcSerialization.layerJson, - Layer.succeed(RpcClient.ConnectionHooks, hooks), - ), - ), - ), - ); - const client = yield* RpcClient.make(RunnerRpcGroup, { disableTracing: true }).pipe( - Effect.provide(protocol), - ); - const hello = yield* client["runner.hello"]({}).pipe(Effect.timeout("10 seconds")); - yield* Ref.update(state, (current) => { - // A different runner identity means a different outbox; its - // sequences restart, so the cursor restarts with it. - const sameRunner = current.runnerId === null || current.runnerId === hello.runnerId; - return { - ...current, - runnerId: hello.runnerId, - bootId: hello.bootId, - connectCount: current.connectCount + 1, - ...(sameRunner ? {} : { deliveredSequence: 0, ackedSequence: 0 }), - }; - }); - yield* Effect.logInfo("runner connected", { - runnerId: hello.runnerId, - bootId: hello.bootId, - headSequence: hello.headSequence, - runnerAckedSequence: hello.ackedSequence, - instances: hello.instances, - }); - const ready = yield* Ref.get(readyRef); - yield* Deferred.succeed(ready, { client, hello }); - yield* Effect.raceFirst(Deferred.await(disconnected), pumpEvents(client)); - }), - ).pipe( - Effect.ensuring( - Effect.gen(function* () { - const current = yield* Ref.get(readyRef); - if (yield* Deferred.isDone(current)) { - yield* Ref.set(readyRef, yield* Deferred.make()); - } - }), - ), - ); - - yield* connectOnce.pipe( - Effect.catchCause((cause) => - Effect.logWarning("runner connection ended", { cause: String(cause).slice(0, 400) }), - ), - Effect.andThen(Effect.sleep(RECONNECT_DELAY)), - Effect.forever, - Effect.forkScoped, - ); - - const use = (f: (client: RunnerRpcClient) => Effect.Effect) => - awaitConnection.pipe(Effect.flatMap((connection) => f(connection.client))); - - const ackDelivered = (throughSequence: number) => - Effect.gen(function* () { - const current = yield* Ref.get(state); - if (throughSequence <= current.ackedSequence) return; - writeCursor(cursorPath, { runnerId: current.runnerId, ackedSequence: throughSequence }); - const latest = yield* Ref.updateAndGet(state, (value) => ({ - ...value, - ackedSequence: throughSequence, - })); - yield* Effect.logInfo("runner events acknowledged", { - throughSequence, - deliveredCount: latest.deliveredCount, - duplicateCount: latest.duplicateCount, - connectCount: latest.connectCount, - }); - yield* use((client) => client["runner.events.ack"]({ throughSequence })).pipe( - Effect.catchCause((cause) => - Effect.logWarning("runner ack failed; it is retried with the next ack", { - throughSequence, - cause: String(cause).slice(0, 200), - }), - ), - ); - }); - - return RunnerClient.of({ - enabled: true, - use, - hello: awaitConnection.pipe(Effect.map((connection) => connection.hello)), - eventsFor: (instanceId) => - Stream.fromPubSub(events).pipe( - Stream.filter((event) => event.providerInstanceId === instanceId), - ), - startDelivery: Deferred.succeed(deliveryEnabled, undefined).pipe(Effect.asVoid), - deliveryState: Ref.get(state), - ackDelivered, - }); - }); - -const disabled = RunnerClient.of({ - enabled: false, - use: () => Effect.fail(new RunnerUnavailableError({ detail: "no runner configured" })), - hello: Effect.fail(new RunnerUnavailableError({ detail: "no runner configured" })), - eventsFor: () => Stream.empty, - startDelivery: Effect.void, - deliveryState: Effect.succeed({ - runnerId: null, - bootId: null, - deliveredSequence: 0, - ackedSequence: 0, - deliveredCount: 0, - duplicateCount: 0, - connectCount: 0, - }), - ackDelivered: () => Effect.void, -}); - -/** Live client in hub mode (`T3CODE_RUNNER_URL`), an inert one otherwise. */ -export const layer = Layer.unwrap( - Effect.gen(function* () { - const config = yield* ServerConfig; - return config.runnerUrl - ? Layer.effect(RunnerClient, makeLive(config.runnerUrl)) - : Layer.succeed(RunnerClient, disabled); - }), -); diff --git a/apps/server/src/runner/hub/RunnerDelivery.ts b/apps/server/src/runner/hub/RunnerDelivery.ts deleted file mode 100644 index 0216364a8e0c..000000000000 --- a/apps/server/src/runner/hub/RunnerDelivery.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * RunnerDelivery - starts runner event delivery once hub ingestion runs, and - * acknowledges delivered events only after ingestion has drained them. - * - * Prototype semantics: at-least-once across hub restarts with a window of one - * ack interval (an event ingested but not yet acked is re-delivered after a - * crash). Production should persist the runner cursor in the same database - * transaction as the projection writes the event produced. - * - * @module runner/hub/RunnerDelivery - */ -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; - -import { ProviderRuntimeIngestionService } from "../../orchestration/Services/ProviderRuntimeIngestion.ts"; -import { forkParked } from "../../serverActivation.ts"; -import { RunnerClient } from "./RunnerClient.ts"; - -const ACK_INTERVAL = "500 millis"; - -export const layer = Layer.effectDiscard( - Effect.gen(function* () { - const runner = yield* RunnerClient; - if (!runner.enabled) return; - const ingestion = yield* ProviderRuntimeIngestionService; - yield* forkParked( - Effect.gen(function* () { - // Ingestion's stream consumers are parked at the same activation - // gate; give them a moment to subscribe before replay starts. - yield* Effect.sleep("500 millis"); - yield* runner.startDelivery; - return yield* Effect.gen(function* () { - const { deliveredSequence, ackedSequence } = yield* runner.deliveryState; - if (deliveredSequence <= ackedSequence) return; - // Two async hops separate delivery from the ingestion queue - // (adapter stream -> ProviderService PubSub -> ingestion worker). - yield* Effect.sleep("50 millis"); - yield* ingestion.drain; - yield* runner.ackDelivered(deliveredSequence); - }).pipe( - Effect.catchCause((cause) => - Effect.logWarning("runner delivery ack failed", { cause: String(cause) }), - ), - Effect.andThen(Effect.sleep(ACK_INTERVAL)), - Effect.forever, - ); - }), - ); - }), -); diff --git a/apps/server/src/runner/remoteErrors.ts b/apps/server/src/runner/remoteErrors.ts new file mode 100644 index 000000000000..35219d8aff26 --- /dev/null +++ b/apps/server/src/runner/remoteErrors.ts @@ -0,0 +1,67 @@ +/** + * Server-internal error unions across the runner protocol. + * + * Contract errors (VCS, git, terminal, review, text generation) cross the wire + * as themselves. Error unions that only exist in the server (provider adapter + * and workspace errors) are encoded with their own schema into + * `RunnerRemoteError` on the runner and decoded back on the hub, so hub-side + * callers see the exact error class the in-process service would raise. + * + * @module runner/remoteErrors + */ +import { RunnerRemoteError } from "@t3tools/contracts/runner"; +import * as Schema from "effect/Schema"; + +import { + ProviderAdapterProcessError, + ProviderAdapterRequestError, + ProviderAdapterSessionClosedError, + ProviderAdapterSessionNotFoundError, + ProviderAdapterValidationError, +} from "../provider/Errors.ts"; + +export const ProviderAdapterErrorSchema = Schema.Union([ + ProviderAdapterValidationError, + ProviderAdapterSessionNotFoundError, + ProviderAdapterSessionClosedError, + ProviderAdapterRequestError, + ProviderAdapterProcessError, +]); + +type Codec = Schema.ConstraintDecoder & Schema.ConstraintEncoder; + +const describe = (error: unknown): string => { + if (error && typeof error === "object" && "message" in error) { + return String((error as { readonly message: unknown }).message); + } + return String(error); +}; + +const tagOf = (error: unknown): string => + error && typeof error === "object" && "_tag" in error + ? String((error as { readonly _tag: unknown })._tag) + : "UnknownError"; + +/** Runner side: wraps an error of `schema`'s union for the wire. */ +export const toRunnerRemoteError = + (schema: S) => + (error: S["Type"]): RunnerRemoteError => { + let encoded: unknown = null; + try { + encoded = Schema.encodeUnknownSync(schema)(error); + } catch { + // Keep the readable tag and message; the hub falls back to them. + } + return new RunnerRemoteError({ errorTag: tagOf(error), detail: describe(error), encoded }); + }; + +/** Hub side: restores the original error, or `fallback` when it cannot be decoded. */ +export const fromRunnerRemoteError = + (schema: S, fallback: (remote: RunnerRemoteError) => F) => + (remote: RunnerRemoteError): S["Type"] | F => { + try { + return Schema.decodeUnknownSync(schema)(remote.encoded); + } catch { + return fallback(remote); + } + }; diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 19b4a480682e..d6a342471b9a 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -134,10 +134,7 @@ import * as NetService from "@t3tools/shared/Net"; import * as RelayClient from "@t3tools/shared/relayClient"; import { disableTailscaleServe, ensureTailscaleServe } from "@t3tools/tailscale"; import { forkParked, ServerActivation } from "./serverActivation.ts"; -import { CheckoutGitProbe, localCheckoutGitProbe } from "./git/CheckoutGitProbe.ts"; -import * as HubLayers from "./runner/hub/HubLayers.ts"; -import * as RunnerClient from "./runner/hub/RunnerClient.ts"; -import * as RunnerDelivery from "./runner/hub/RunnerDelivery.ts"; +import * as HubLayers from "./hub/HubLayers.ts"; // MCP handoff thread IDs include escaped provenance and can exceed find-my-way's // 100-character default for one path segment. @@ -155,7 +152,7 @@ export const ApplicationObservabilityLive = ObservabilityLive.pipe( Layer.provideMerge(ResourceAttributionLayerLive), ); -const PtyAdapterLive = Layer.unwrap( +export const PtyAdapterLive = Layer.unwrap( Effect.gen(function* () { if (typeof Bun !== "undefined") { const BunPtyAdapter = yield* Effect.promise(() => import("./terminal/BunPtyAdapter.ts")); @@ -168,17 +165,35 @@ const PtyAdapterLive = Layer.unwrap( ); /** - * Picks the checkout-bound implementation (`local`) or the hub substitute - * (`hub`) depending on whether this server delegates to a runner. + * Picks the standalone or the hub implementation of a mode-dependent layer + * group (see `ServerModeLayers`). Consumers see only the services both provide. */ -const byServerMode = ( - local: Layer.Layer, - hub: Layer.Layer, -): Layer.Layer => +const byServerMode = ( + standalone: Layer.Layer, + hub: Layer.Layer, +): Layer.Layer, E1 | E2, R1 | R2 | ServerConfig.ServerConfig> => Layer.unwrap( Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; - return (config.serverMode === "hub" ? hub : local) as Layer.Layer; + return (ServerConfig.serverModeOf(config) === "hub" + ? hub + : standalone) as unknown as Layer.Layer, E1 | E2, R1 | R2>; + }), + ); + +/** + * A layer only a hub runs. Standalone servers build nothing in its place; the + * services it provides are only required by the other hub groups. + */ +const hubOnly = ( + hub: Layer.Layer, +): Layer.Layer => + Layer.unwrap( + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + return ServerConfig.serverModeOf(config) === "hub" + ? hub + : (Layer.empty as unknown as Layer.Layer); }), ); @@ -291,13 +306,7 @@ export const PlatformServicesLive = Layer.unwrap( }), ); -const CheckoutGitProbeLayerLive = byServerMode( - Layer.succeed(CheckoutGitProbe, localCheckoutGitProbe), - HubLayers.remoteCheckoutGitProbeLayer, -); - const ReactorLayerLive = Layer.empty.pipe( - Layer.provideMerge(RunnerDelivery.layer), Layer.provideMerge(OrchestrationReactorLive), Layer.provideMerge(ProviderRuntimeIngestionLive), Layer.provideMerge(ProviderCommandReactorLive), @@ -306,7 +315,6 @@ const ReactorLayerLive = Layer.empty.pipe( Layer.provideMerge(ThreadSettlementReactor.layer), Layer.provideMerge(AgentAwarenessRelayLayerLive), Layer.provideMerge(RuntimeReceiptBusLive), - Layer.provideMerge(CheckoutGitProbeLayerLive), ); const ProviderSessionDirectoryLayerLive = ProviderSessionDirectoryLive.pipe( @@ -330,7 +338,7 @@ export const VcsDriverRegistryLayerLive = VcsDriverRegistry.layer.pipe( Layer.provide(VcsProjectConfig.layer), ); -const SourceControlProviderRegistryLayerLive = SourceControlProviderRegistry.layer.pipe( +export const SourceControlProviderRegistryLayerLive = SourceControlProviderRegistry.layer.pipe( Layer.provide( Layer.mergeAll(AzureDevOpsCli.layer, BitbucketApi.layer, GitHubCli.layer, GitLabCli.layer), ), @@ -382,22 +390,12 @@ const VcsLayerLive = Layer.empty.pipe( Layer.provideMerge(ReviewLayerLive), Layer.provideMerge(GrokReviewLayerLive), Layer.provideMerge(SourceControlRepositoryServiceLayerLive), - Layer.provideMerge( - byServerMode( - VcsStatusBroadcaster.layer.pipe(Layer.provide(GitWorkflowLayerLive)), - HubLayers.hubVcsStatusBroadcasterLayer, - ), - ), + Layer.provideMerge(VcsStatusBroadcaster.layer.pipe(Layer.provide(GitWorkflowLayerLive))), ); const CheckpointingLayerLive = Layer.empty.pipe( Layer.provideMerge(CheckpointDiffQuery.layer), - Layer.provideMerge( - byServerMode( - CheckpointStore.layer.pipe(Layer.provide(VcsDriverRegistryLayerLive)), - HubLayers.remoteCheckpointStoreLayer, - ), - ), + Layer.provideMerge(CheckpointStore.layer.pipe(Layer.provide(VcsDriverRegistryLayerLive))), ); const PortScannerLayerLive = PortScanner.layer.pipe(Layer.provide(ProcessRunner.layer)); @@ -420,8 +418,8 @@ const WorkspaceFileSystemLayerLive = WorkspaceFileSystem.layer.pipe( ); const WorkspaceLayerLive = Layer.mergeAll( - byServerMode(WorkspacePaths.layer, HubLayers.remoteWorkspacePathsLayer), - byServerMode(WorkspaceEntriesLayerLive, HubLayers.hubWorkspaceEntriesLayer), + WorkspacePaths.layer, + WorkspaceEntriesLayerLive, WorkspaceFileSystemLayerLive, ); @@ -448,22 +446,92 @@ const CloudManagedEndpointRuntimeLive = Layer.mergeAll( ), ); +/** + * Layer groups whose implementation depends on the server mode. Standalone + * uses the local implementations above; a hub (`T3CODE_SERVER_MODE=hub`) + * replaces every checkout-bound group with `hub/HubLayers.ts`, so the hub never + * reads a checkout, runs git, opens a PTY, or spawns a provider CLI. The rest + * of the runtime is identical in both modes. See + * docs/internals/thread-machines.md. + */ +const ServerModeLayers = { + /** Background work that needs the reactors (hub: runner delivery and machine release). */ + background: hubOnly(HubLayers.hubRuntimeBackgroundLayer), + /** Hooks read by reactors and command dispatch; standalone relies on their defaults. */ + reactorHooks: hubOnly(HubLayers.hubReactorHooksLayer), + checkpointing: byServerMode( + CheckpointingLayerLive, + CheckpointDiffQuery.layer.pipe(Layer.provideMerge(HubLayers.hubCheckpointStoreLayer)), + ), + git: byServerMode( + GitLayerLive, + HubLayers.hubGitManagerLayer.pipe( + Layer.provideMerge(ProjectSetupScriptRunner.layer), + Layer.provideMerge(GitVcsDriver.layer), + Layer.provideMerge(SourceControlProviderRegistryLayerLive), + Layer.provideMerge(TextGeneration.layer), + ), + ), + vcs: byServerMode( + VcsLayerLive, + Layer.mergeAll( + HubLayers.hubVcsStatusBroadcasterLayer, + HubLayers.hubVcsProvisioningServiceLayer, + HubLayers.hubSourceControlRepositoryServiceLayer.pipe( + Layer.provide(SourceControlRepositoryServiceLayerLive), + ), + GrokReviewService.layer.pipe(Layer.provide(HubLayers.hubReviewServiceLayer)), + ).pipe( + Layer.provideMerge(HubLayers.hubReviewServiceLayer), + Layer.provideMerge(HubLayers.hubGitWorkflowServiceLayer), + Layer.provideMerge(VcsDriverRegistryLayerLive), + Layer.provideMerge(VcsProjectConfig.layer), + ), + ), + terminal: byServerMode(TerminalLayerLive, HubLayers.hubTerminalManagerLayer), + workspace: byServerMode( + WorkspaceLayerLive, + Layer.mergeAll( + HubLayers.hubWorkspacePathsLayer, + HubLayers.hubWorkspaceEntriesLayer, + HubLayers.hubWorkspaceFileSystemLayer, + ), + ), + repositoryIdentity: byServerMode( + RepositoryIdentityResolver.layer, + HubLayers.hubRepositoryIdentityResolverLayer, + ), + providerInstances: byServerMode( + ProviderInstanceRegistryHydrationLive, + HubLayers.hubProviderInstanceRegistryLayer, + ), + /** Hub-only services under the whole runtime (machine directory, runner pool, caches). */ + infrastructure: hubOnly( + HubLayers.makeHubInfrastructureLayer({ sqlitePersistence: SqlitePersistenceLayerLive }), + ), +}; + const ProviderRuntimeLayerLive = Layer.mergeAll( ProviderSessionReaperLive, TurnLivenessWatchdogLive, ).pipe(Layer.provideMerge(ProviderLayerLive), Layer.provideMerge(OrchestrationLayerLive)); -const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( +const RuntimeReactorsLive = ServerModeLayers.background.pipe( + Layer.provideMerge(ReactorLayerLive), + Layer.provideMerge(ServerModeLayers.reactorHooks), +); + +const RuntimeCoreDependenciesLive = RuntimeReactorsLive.pipe( // Core Services Layer.provideMerge(ServerSettingsLayerLive), - Layer.provideMerge(CheckpointingLayerLive), + Layer.provideMerge(ServerModeLayers.checkpointing), Layer.provideMerge( Layer.mergeAll(SourceControlProviderRegistryLayerLive, PullRequestServiceLive), ), - Layer.provideMerge(GitLayerLive), - Layer.provideMerge(VcsLayerLive), + Layer.provideMerge(ServerModeLayers.git), + Layer.provideMerge(ServerModeLayers.vcs), Layer.provideMerge(ProviderRuntimeLayerLive), - Layer.provideMerge(Layer.mergeAll(TerminalLayerLive, PreviewLayerLive)), + Layer.provideMerge(Layer.mergeAll(ServerModeLayers.terminal, PreviewLayerLive)), Layer.provideMerge(PersistenceLayerLive), // Both read a user-owned file out of the state directory and stream changes // to clients; neither depends on the other. @@ -474,7 +542,7 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( // through this layer. Built-in drivers come from `BUILT_IN_DRIVERS`; // `providerInstances` hydration merges `settings.providers.` // with explicit `providerInstances` entries on boot. - Layer.provideMerge(ProviderInstanceRegistryHydrationLive), + Layer.provideMerge(ServerModeLayers.providerInstances), // Shared native/canonical NDJSON writers used by both the per-instance // drivers (native stream, written from inside each `Adapter`) and // `ProviderService` (canonical stream, written after event normalization). @@ -490,11 +558,9 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( // no longer transitively provides it. Exposing it at the runtime level // keeps a single Live for all opencode consumers. Layer.provideMerge(OpenCodeRuntime.OpenCodeRuntimeLive), - Layer.provideMerge(WorkspaceLayerLive), + Layer.provideMerge(ServerModeLayers.workspace), Layer.provideMerge(ProjectFaviconResolverLayerLive), - Layer.provideMerge( - byServerMode(RepositoryIdentityResolver.layer, HubLayers.hubRepositoryIdentityResolverLayer), - ), + Layer.provideMerge(ServerModeLayers.repositoryIdentity), Layer.provideMerge(ServerEnvironmentLayerLive), Layer.provideMerge(AuthLayerLive), Layer.provideMerge(ServerSecretStore.layer), @@ -773,7 +839,7 @@ export const makeServerLayer = Layer.unwrap( return serverApplicationLayer.pipe( Layer.provideMerge(runtimeServicesLive), - Layer.provideMerge(RunnerClient.layer), + Layer.provideMerge(ServerModeLayers.infrastructure), Layer.provide(activationLayer), Layer.provideMerge(serverRelayBrokerTracingLayer), Layer.provideMerge(HttpServerLive), From f4bb1804282631612ed7310d544dfa9af68495d7 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 23:00:57 +0000 Subject: [PATCH 14/58] feat(server): import a standalone state directory into a hub tenant t3 hub import copies a standalone server's state.sqlite, settings, keybindings, environment id, anonymous id, attachments, and provider environment secrets into the tenant named by the T3CODE_HUB_* environment, in one Postgres transaction. Tables are paged by rowid so a large history streams with bounded memory, NUL escapes in JSON become U+FFFD, the tenant's event counter continues after the imported history, and projects whose checkouts exist on this machine get their repository identity recorded. A tenant with existing rows is refused unless --replace is given. The source is opened read-only and must be at the current standalone migration. Only counts are printed. The test builds a standalone history through the standalone layers, imports it, and requires every snapshot read to match; it also checks refusal, replace, re-encrypted secrets, attachments, and continued sequencing. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/bin.ts | 2 + apps/server/src/cli/hub.ts | 124 +++++++ .../persistence/Postgres/HubImport.test.ts | 257 +++++++++++++ .../src/persistence/Postgres/HubImport.ts | 349 ++++++++++++++++++ .../Postgres/HubRepositories.test.ts | 193 +--------- .../persistence/Postgres/hubTestScenario.ts | 219 +++++++++++ 6 files changed, 952 insertions(+), 192 deletions(-) create mode 100644 apps/server/src/cli/hub.ts create mode 100644 apps/server/src/persistence/Postgres/HubImport.test.ts create mode 100644 apps/server/src/persistence/Postgres/HubImport.ts create mode 100644 apps/server/src/persistence/Postgres/hubTestScenario.ts diff --git a/apps/server/src/bin.ts b/apps/server/src/bin.ts index 101df9f3dc0c..5b7f04c48ebb 100644 --- a/apps/server/src/bin.ts +++ b/apps/server/src/bin.ts @@ -17,6 +17,7 @@ import { isEntrypoint } from "./entrypoint.ts"; import { projectCommand } from "./cli/project.ts"; import { runServerCommand, serveCommand, startCommand } from "./cli/server.ts"; import { runnerCommand } from "./cli/runner.ts"; +import { hubCommand } from "./cli/hub.ts"; import { serviceCommand } from "./cli/service.ts"; import { servicePreflightCommand } from "./cli/servicePreflight.ts"; import { themeCommand } from "./cli/theme.ts"; @@ -64,6 +65,7 @@ export const makeCli = ({ cloudEnabled = hasCloudPublicConfig } = {}) => themeCommand, triageCommand, runnerCommand, + hubCommand, cloudEnabled ? connectCommand : connectUnavailableCommand, ]), ); diff --git a/apps/server/src/cli/hub.ts b/apps/server/src/cli/hub.ts new file mode 100644 index 000000000000..560d9e0271ab --- /dev/null +++ b/apps/server/src/cli/hub.ts @@ -0,0 +1,124 @@ +import * as Config from "effect/Config"; +import * as Console from "effect/Console"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Redacted from "effect/Redacted"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import type { SqlError } from "effect/unstable/sql/SqlError"; +import { Argument, Command, Flag } from "effect/unstable/cli"; + +import { expandHomePath } from "../os-jank.ts"; +import { makeHubDatabase } from "../persistence/Postgres/HubDatabaseLive.ts"; +import { + HubImportError, + importStandaloneState, + type HubImportReport, +} from "../persistence/Postgres/HubImport.ts"; +import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; + +const HubImportEnv = Config.all({ + databaseUrl: Config.redacted("T3CODE_HUB_DATABASE_URL"), + databaseAdminUrl: Config.redacted("T3CODE_HUB_DATABASE_ADMIN_URL").pipe(Config.option), + tenantId: Config.string("T3CODE_HUB_TENANT_ID"), + secretKey: Config.redacted("T3CODE_HUB_SECRET_KEY"), +}); + +type ReadonlySqliteLoader = { + readonly layer: (config: { + readonly filename: string; + readonly readonly?: boolean; + }) => Layer.Layer; +}; + +/** Opens `state.sqlite` read-only; the import never migrates or writes the source. */ +const openReadonlySqlite = (filename: string) => + Effect.gen(function* () { + const loader = yield* Effect.promise(() => + process.versions.bun !== undefined + ? import("@effect/sql-sqlite-bun/SqliteClient") + : import("../persistence/NodeSqliteClient.ts"), + ); + const context = yield* Layer.build(loader.layer({ filename, readonly: true })); + return Context.get(context, SqlClient.SqlClient); + }); + +const formatReport = (report: HubImportReport) => + [ + report.replacedExisting + ? "Replaced the tenant's hub data with the standalone state." + : "Imported the standalone state into the hub tenant.", + ...Object.entries(report.rows).map(([table, count]) => ` ${table}: ${count} rows`), + ` documents: ${report.documents}`, + ` attachments: ${report.attachments} (skipped ${report.attachmentsSkipped})`, + ` provider secrets: ${report.secrets}`, + ` repository identities: ${report.repositoryIdentities}`, + ` next event sequence: ${report.lastEventSequence + 1}`, + "", + ].join("\n"); + +const importCommand = Command.make("import", { + stateDir: Argument.string("state-dir").pipe( + Argument.withDescription( + "Standalone state directory holding state.sqlite, for example ~/.t3/userdata.", + ), + ), + replace: Flag.boolean("replace").pipe( + Flag.withDescription("Delete the tenant's existing hub data before importing."), + ), + skipRepositoryIdentity: Flag.boolean("skip-repository-identity").pipe( + Flag.withDescription( + "Do not record repository identities from project checkouts on this machine.", + ), + ), +}).pipe( + Command.withDescription( + "Import a standalone T3 state directory into a hub tenant. Reads the hub database, tenant, and secret key from the T3CODE_HUB_* environment. Prints counts only.", + ), + Command.withHandler((flags) => + Effect.gen(function* () { + const env = yield* HubImportEnv; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = path.resolve(yield* expandHomePath(flags.stateDir)); + const databasePath = path.join(stateDir, "state.sqlite"); + if (!(yield* fileSystem.exists(databasePath))) { + return yield* new HubImportError({ + reason: "source-missing", + detail: `No state.sqlite in ${stateDir}.`, + }); + } + + const hub = yield* makeHubDatabase({ + databaseUrl: Redacted.value(env.databaseUrl), + databaseAdminUrl: Option.getOrUndefined(Option.map(env.databaseAdminUrl, Redacted.value)), + tenantId: env.tenantId, + }); + const source = yield* openReadonlySqlite(databasePath); + const resolver = flags.skipRepositoryIdentity + ? undefined + : Context.get( + yield* Layer.build(RepositoryIdentityResolver.layer), + RepositoryIdentityResolver.RepositoryIdentityResolver, + ); + + const report = yield* importStandaloneState({ + stateDir, + source, + hub, + secretKey: Redacted.value(env.secretKey), + replace: flags.replace, + ...(resolver ? { resolveRepositoryIdentity: resolver.resolve } : {}), + }); + yield* Console.log(formatReport(report)); + }).pipe(Effect.scoped), + ), +); + +export const hubCommand = Command.make("hub").pipe( + Command.withDescription("Manage thread-machine hub state."), + Command.withSubcommands([importCommand]), +); diff --git a/apps/server/src/persistence/Postgres/HubImport.test.ts b/apps/server/src/persistence/Postgres/HubImport.test.ts new file mode 100644 index 000000000000..6a7d97b31e8c --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubImport.test.ts @@ -0,0 +1,257 @@ +/** + * `t3 hub import`: a synthetic standalone state directory copied into a hub + * tenant must read back exactly as it did standalone. Runs only with + * T3_HUB_TEST_DATABASE_URL (see `hubTestDatabase.ts`); the NUL-escape rule is + * checked without a database. + */ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { CommandId } from "@t3tools/contracts"; +import { assert, describe, it } from "@effect/vitest"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Reactivity from "effect/unstable/reactivity/Reactivity"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { createAttachmentId } from "../../attachmentStore.ts"; +import * as ServerSecretStore from "../../auth/ServerSecretStore.ts"; +import * as ServerConfig from "../../config.ts"; +import { OrchestrationEngineService } from "../../orchestration/Services/OrchestrationEngine.ts"; +import { OrchestrationLayerLive } from "../../orchestration/runtimeLayer.ts"; +import { RepositoryIdentityResolver } from "../../project/RepositoryIdentityResolver.ts"; +import { + layerConfig as ServerPersistenceLive, + makeSqlitePersistenceLive, +} from "../Layers/Sqlite.ts"; +import * as NodeSqliteClient from "../NodeSqliteClient.ts"; +import { HubDatabase } from "./HubDatabase.ts"; +import { + importStandaloneState, + neutralizeJsonNulEscapes, + type HubImportReport, +} from "./HubImport.ts"; +import { + HUB_TEST_SECRET_KEY, + hubTestDatabaseLayer, + hubTestDatabaseUrl, + hubTestServerConfigLayer, + makeHubTestSchema, + type HubTestSchema, +} from "./hubTestDatabase.ts"; +import { orchestrationScenario, readScenario, threadId } from "./hubTestScenario.ts"; + +const TENANT = "tenant-import"; + +const withSchema = (use: (schema: HubTestSchema) => Effect.Effect) => + Effect.scoped(Effect.flatMap(makeHubTestSchema(hubTestDatabaseUrl!), use)).pipe( + Effect.provide(Reactivity.layer), + ); + +const platform = Layer.mergeAll( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-import-" }), + Layer.succeed(RepositoryIdentityResolver, { resolve: () => Effect.succeed(null) }), +).pipe(Layer.provideMerge(NodeServices.layer)); + +const hub = (schema: HubTestSchema) => + ServerPersistenceLive.pipe( + Layer.provideMerge(hubTestDatabaseLayer(schema, TENANT)), + Layer.provide(platform), + ); + +/** Standalone strings with a NUL read back from the hub with U+FFFD instead (plain data only). */ +const withNulReplaced = (value: unknown): unknown => { + if (typeof value === "string") return value.replaceAll("\u0000", "�"); + if (Array.isArray(value)) return value.map(withNulReplaced); + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([key, inner]) => [key, withNulReplaced(inner)]), + ); + } + return value; +}; + +const repositoryIdentity = { + canonicalKey: "github.com/aldo/app", + locator: { + source: "git-remote" as const, + remoteName: "origin", + remoteUrl: "git@github.com:aldo/app.git", + }, + provider: "github", + owner: "aldo", + name: "app", +}; + +describe("neutralizeJsonNulEscapes", () => { + it("replaces NUL escapes but not escaped backslashes followed by u0000", () => { + assert.strictEqual(neutralizeJsonNulEscapes('{"a":"x\\u0000y"}'), '{"a":"x\\ufffdy"}'); + assert.strictEqual( + neutralizeJsonNulEscapes('{"a":"\\u0000\\u0000"}'), + '{"a":"\\ufffd\\ufffd"}', + ); + assert.strictEqual(neutralizeJsonNulEscapes('{"a":"\\\\u0000"}'), '{"a":"\\\\u0000"}'); + assert.strictEqual(neutralizeJsonNulEscapes('{"a":"\\\\\\u0000"}'), '{"a":"\\\\\\ufffd"}'); + assert.strictEqual(neutralizeJsonNulEscapes('{"a":1}'), '{"a":1}'); + }); +}); + +describe.skipIf(hubTestDatabaseUrl === undefined)("hub import", () => { + it.effect("copies a standalone state directory into a tenant, once", () => + withSchema((schema) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-standalone-" }); + const checkout = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkout-" }); + const databasePath = path.join(stateDir, "state.sqlite"); + + // A standalone server's history, written through the standalone layers. + const standalone = yield* orchestrationScenario("alice", checkout).pipe( + Effect.provide( + OrchestrationLayerLive.pipe( + Layer.provideMerge(makeSqlitePersistenceLive(databasePath)), + Layer.provideMerge(platform), + ), + ), + ); + yield* fileSystem.writeFileString( + path.join(stateDir, "settings.json"), + '{ "enableProviderUpdateChecks": false }\n', + ); + yield* fileSystem.writeFileString(path.join(stateDir, "keybindings.json"), "[]\n"); + yield* fileSystem.writeFileString( + path.join(stateDir, "environment-id"), + "4f2d8c3e-1d0b-4b8e-9b53-2c0c6c0d5e21\n", + ); + yield* fileSystem.makeDirectory(path.join(stateDir, "secrets")); + yield* fileSystem.writeFile( + path.join(stateDir, "secrets", "provider-env-Y29kZXg-T1BFTlJPVVRFUl9BUElfS0VZ.bin"), + new TextEncoder().encode("sk-or-secret"), + ); + yield* fileSystem.writeFile( + path.join(stateDir, "secrets", "session-signing-key.bin"), + new Uint8Array(32), + ); + const attachmentsDir = path.join(stateDir, "attachments"); + yield* fileSystem.makeDirectory(attachmentsDir); + const attachmentFile = `${createAttachmentId(threadId)}.png`; + const attachmentBytes = new Uint8Array([137, 80, 78, 71, 0, 1, 2]); + yield* fileSystem.writeFile(path.join(attachmentsDir, attachmentFile), attachmentBytes); + yield* fileSystem.writeFile( + path.join(attachmentsDir, "upload.png.abc.part"), + new Uint8Array(3), + ); + + const source = yield* Layer.build( + NodeSqliteClient.layer({ filename: databasePath, readonly: true }), + ).pipe(Effect.map((context) => Context.get(context, SqlClient.SqlClient))); + const sourceCounts = yield* Effect.forEach( + ["orchestration_events", "projection_thread_activities", "projection_turns"], + (table) => + source.unsafe<{ readonly count: number }>(`SELECT count(*) AS count FROM ${table}`), + ); + + const runImport = (options: { + readonly replace: boolean; + readonly withIdentity: boolean; + }) => + Effect.flatMap(HubDatabase, (database) => + importStandaloneState({ + stateDir, + source, + hub: database!, + secretKey: HUB_TEST_SECRET_KEY, + replace: options.replace, + ...(options.withIdentity + ? { + resolveRepositoryIdentity: (root: string) => + Effect.succeed(root === checkout ? repositoryIdentity : null), + } + : {}), + }), + ).pipe(Effect.provide(hubTestDatabaseLayer(schema, TENANT))); + + const report: HubImportReport = yield* runImport({ replace: false, withIdentity: false }); + assert.strictEqual(report.rows.orchestration_events, sourceCounts[0]?.[0]?.count); + assert.strictEqual(report.rows.projection_thread_activities, sourceCounts[1]?.[0]?.count); + assert.strictEqual(report.rows.projection_turns, sourceCounts[2]?.[0]?.count); + assert.strictEqual(report.lastEventSequence, standalone.events.at(-1)?.sequence); + assert.deepStrictEqual( + [report.documents, report.secrets, report.attachments, report.attachmentsSkipped], + [3, 1, 1, 1], + ); + assert.isFalse(report.replacedExisting); + + // Everything reads back as it did standalone. + const imported = yield* readScenario(checkout).pipe( + Effect.provide( + OrchestrationLayerLive.pipe( + Layer.provideMerge(hub(schema)), + Layer.provideMerge(platform), + ), + ), + ); + assert.deepStrictEqual(withNulReplaced(imported), withNulReplaced(standalone)); + assert.strictEqual(imported.shell.threads.length, 1); + assert.isTrue(Option.isSome(imported.detailSnapshot)); + + // A second import is refused; --replace starts over and records identities. + const refused = yield* runImport({ replace: false, withIdentity: false }).pipe(Effect.flip); + assert.strictEqual(refused._tag === "HubImportError" && refused.reason, "tenant-not-empty"); + const replaced = yield* runImport({ replace: true, withIdentity: true }); + assert.isTrue(replaced.replacedExisting); + assert.strictEqual(replaced.repositoryIdentities, 1); + assert.deepStrictEqual(replaced.rows, report.rows); + + yield* Effect.gen(function* () { + const engine = yield* OrchestrationEngineService; + const secrets = yield* ServerSecretStore.ServerSecretStore; + const database = (yield* HubDatabase)!; + + // New commands continue the imported sequence. + const result = yield* engine.dispatch({ + type: "thread.meta.update", + commandId: CommandId.make("cmd-after-import"), + threadId, + title: "After import", + }); + assert.strictEqual(result.sequence, report.lastEventSequence + 1); + + const secret = yield* secrets.get("provider-env-Y29kZXg-T1BFTlJPVVRFUl9BUElfS0VZ"); + assert.strictEqual(new TextDecoder().decode(Option.getOrThrow(secret)), "sk-or-secret"); + assert.isTrue(Option.isNone(yield* secrets.get("session-signing-key"))); + + const [attachment] = yield* database.sql<{ readonly content: Uint8Array }>` + SELECT content FROM hub_attachments WHERE user_id = ${TENANT} + `; + assert.deepStrictEqual(new Uint8Array(attachment!.content), attachmentBytes); + const [project] = yield* database.sql<{ readonly identity: string }>` + SELECT repository_identity_json AS identity FROM projection_projects + WHERE user_id = ${TENANT} + `; + assert.include(project!.identity, "github.com/aldo/app"); + const payloads = yield* database.sql<{ readonly payload: string }>` + SELECT payload_json AS payload FROM projection_thread_activities + WHERE user_id = ${TENANT} + `; + assert.isTrue(payloads.every((row) => !row.payload.includes("\\u0000"))); + }).pipe( + Effect.provide( + Layer.mergeAll(OrchestrationLayerLive, ServerSecretStore.layer).pipe( + Layer.provideMerge(ServerPersistenceLive), + Layer.provideMerge(hubTestDatabaseLayer(schema, TENANT)), + Layer.provideMerge(hubTestServerConfigLayer(schema, TENANT)), + Layer.provideMerge( + Layer.succeed(RepositoryIdentityResolver, { resolve: () => Effect.succeed(null) }), + ), + Layer.provideMerge(NodeServices.layer), + ), + ), + ); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); +}); diff --git a/apps/server/src/persistence/Postgres/HubImport.ts b/apps/server/src/persistence/Postgres/HubImport.ts new file mode 100644 index 000000000000..db1a5b90d32b --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubImport.ts @@ -0,0 +1,349 @@ +/** + * HubImport - copies a standalone server's state into a hub tenant. + * + * Reads a standalone state directory (the one holding `state.sqlite`) and + * writes the tenant's rows in one Postgres transaction, so an import either + * lands completely or not at all: + * + * - orchestration events, receipts, projections, checkpoint diffs, and + * provider session runtime rows, paged by rowid so memory stays bounded; + * - settings.json, keybindings.json, environment-id, and anonymous-id as + * hub documents (the environment id is kept, so clients see the same + * environment); + * - attachment files as hub attachments, within the upload size limit; + * - provider environment secrets, re-encrypted with the hub secret key. + * + * Auth sessions, pairing links, and other secrets (signing keys, T3 Connect + * credentials) stay behind: they belong to the old server's origin. + * + * A tenant that already has rows is refused unless `replace` is set, which + * deletes them first in the same transaction. JSON text keeps its content + * except `\u0000` escapes, which become `�` (Postgres JSON operators + * reject them). Only counts are reported. + * + * @module HubImport + */ +import { PROVIDER_SEND_TURN_MAX_FILE_BYTES, RepositoryIdentity } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import type * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { + parseAttachmentIdFromRelativePath, + parseThreadSegmentFromAttachmentId, +} from "../../attachmentStore.ts"; +import { migrationEntries } from "../Migrations.ts"; +import type { HubDatabaseShape } from "./HubDatabase.ts"; +import { encryptHubSecret, parseHubSecretKey } from "./HubSecretCipher.ts"; +import { HUB_BASELINE_TENANT_TABLES } from "./migrations/001_HubBaseline.ts"; + +export class HubImportError extends Schema.TaggedErrorClass()("HubImportError", { + reason: Schema.Literals([ + "source-missing", + "source-version", + "schema-mismatch", + "tenant-not-empty", + ]), + detail: Schema.String, +}) { + override get message(): string { + return this.detail; + } +} + +/** Standalone tables copied into the hub, in dependency-free order. */ +const IMPORTED_TABLES: ReadonlyArray<{ readonly table: string; readonly pageRows: number }> = [ + { table: "projection_projects", pageRows: 500 }, + { table: "projection_threads", pageRows: 500 }, + { table: "projection_thread_sessions", pageRows: 500 }, + { table: "projection_turns", pageRows: 500 }, + { table: "projection_thread_proposed_plans", pageRows: 200 }, + { table: "projection_pending_approvals", pageRows: 500 }, + { table: "projection_state", pageRows: 500 }, + { table: "projection_thread_messages", pageRows: 200 }, + { table: "projection_thread_activities", pageRows: 200 }, + { table: "orchestration_events", pageRows: 200 }, + { table: "orchestration_command_receipts", pageRows: 1000 }, + { table: "checkpoint_diff_blobs", pageRows: 50 }, + { table: "provider_session_runtime", pageRows: 500 }, +]; + +// Hub-generated columns with no standalone counterpart. +const HUB_ONLY_COLUMNS = new Set(["user_id", "row_id", "repository_identity_json"]); + +const IMPORTED_DOCUMENTS = ["settings.json", "keybindings.json", "environment-id", "anonymous-id"]; + +// ServerSettings keeps sensitive provider environment values in these secrets. +const IMPORTED_SECRET_PREFIX = "provider-env-"; + +// Postgres caps a statement at 65535 bind parameters. +const MAX_PARAMETERS_PER_INSERT = 30_000; + +/** Replaces `\u0000` JSON escapes (not escaped backslashes before "u0000"). */ +export const neutralizeJsonNulEscapes = (json: string): string => + json.includes("\\u0000") ? json.replace(/(?>; + readonly lastEventSequence: number; + readonly documents: number; + readonly attachments: number; + readonly attachmentsSkipped: number; + readonly secrets: number; + readonly repositoryIdentities: number; + readonly replacedExisting: boolean; +} + +export interface HubImportInput { + /** Standalone state directory containing `state.sqlite`. */ + readonly stateDir: string; + /** Read-only client over the standalone `state.sqlite`. */ + readonly source: SqlClient.SqlClient; + readonly hub: HubDatabaseShape; + readonly secretKey: string; + readonly replace: boolean; + /** Resolves a project's repository identity from its checkout, when present. */ + readonly resolveRepositoryIdentity?: ( + workspaceRoot: string, + ) => Effect.Effect; +} + +const encodeRepositoryIdentityJson = Schema.encodeSync(Schema.fromJsonString(RepositoryIdentity)); + +const quoteIdentifier = (identifier: string) => `"${identifier.replaceAll('"', '""')}"`; + +const verifySourceVersion = (source: SqlClient.SqlClient) => + Effect.gen(function* () { + const expected = Math.max(...migrationEntries.map(([id]) => id)); + const rows = yield* source<{ readonly id: number | null }>` + SELECT max(migration_id) AS id FROM effect_sql_migrations + `.pipe(Effect.orElseSucceed(() => [{ id: null }])); + const actual = rows[0]?.id ?? null; + if (actual !== expected) { + return yield* new HubImportError({ + reason: "source-version", + detail: + actual === null || actual < expected + ? `The standalone database is at migration ${actual ?? "none"}; start this T3 version on it once so it migrates to ${expected}, then import.` + : `The standalone database is at migration ${actual}, newer than this T3 version (${expected}).`, + }); + } + }); + +const importTable = ( + source: SqlClient.SqlClient, + hub: HubDatabaseShape, + table: string, + pageRows: number, +) => + Effect.gen(function* () { + const sourceColumns = (yield* source<{ readonly name: string }>` + SELECT name FROM pragma_table_info(${table}) + `) + .map((row) => row.name) + .filter((name) => !HUB_ONLY_COLUMNS.has(name)); + const hubColumns = new Set( + (yield* hub.sql<{ readonly name: string }>` + SELECT column_name AS name + FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = ${table} + `).map((row) => row.name), + ); + const missing = sourceColumns.filter((name) => !hubColumns.has(name)); + if (sourceColumns.length === 0 || missing.length > 0) { + return yield* new HubImportError({ + reason: "schema-mismatch", + detail: `Table ${table} has columns the hub schema lacks: ${missing.join(", ") || "(table missing)"}.`, + }); + } + + const columnList = sourceColumns.map(quoteIdentifier).join(", "); + const jsonColumns = sourceColumns.map((name) => name.endsWith("_json")); + const rowsPerInsert = Math.max( + 1, + Math.min(pageRows, Math.floor(MAX_PARAMETERS_PER_INSERT / (sourceColumns.length + 1))), + ); + let lastRowId = -1; + let copied = 0; + while (true) { + const page = yield* source.unsafe>( + `SELECT rowid AS "__rowid", ${columnList} FROM ${quoteIdentifier(table)} + WHERE rowid > ? ORDER BY rowid LIMIT ?`, + [lastRowId, rowsPerInsert], + ); + if (page.length === 0) { + return copied; + } + const params: Array = []; + const tuples = page.map((row) => { + const placeholders = [`$${params.push(hub.tenantId)}`]; + sourceColumns.forEach((column, index) => { + const value = row[column]; + placeholders.push( + `$${params.push( + jsonColumns[index] && typeof value === "string" + ? neutralizeJsonNulEscapes(value) + : typeof value === "bigint" + ? Number(value) + : (value ?? null), + )}`, + ); + }); + return `(${placeholders.join(", ")})`; + }); + yield* hub.sql.unsafe( + `INSERT INTO ${quoteIdentifier(table)} (user_id, ${columnList}) VALUES ${tuples.join(", ")}`, + params, + ); + copied += page.length; + lastRowId = Number(page[page.length - 1]!["__rowid"]); + } + }); + +/** Copies the standalone state into the hub tenant in one transaction. */ +export const importStandaloneState = (input: HubImportInput) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { hub, source } = input; + const key = yield* Effect.sync(() => parseHubSecretKey(input.secretKey)); + yield* verifySourceVersion(source); + + return yield* hub.sql.withTransaction( + Effect.gen(function* () { + const occupied: Array = []; + for (const table of HUB_BASELINE_TENANT_TABLES) { + const [row] = yield* hub.sql<{ readonly present: boolean }>` + SELECT EXISTS ( + SELECT 1 FROM ${hub.sql(table)} WHERE user_id = ${hub.tenantId} + ) AS present + `; + if (row?.present) occupied.push(table); + } + if (occupied.length > 0 && !input.replace) { + return yield* new HubImportError({ + reason: "tenant-not-empty", + detail: `Tenant already has hub data (${occupied.length} tables); rerun with --replace to overwrite it.`, + }); + } + for (const table of occupied) { + yield* hub.sql`DELETE FROM ${hub.sql(table)} WHERE user_id = ${hub.tenantId}`; + } + + const rows: Record = {}; + for (const { table, pageRows } of IMPORTED_TABLES) { + rows[table] = yield* importTable(source, hub, table, pageRows); + } + + const [sequence] = yield* source<{ readonly last: number | null }>` + SELECT max(sequence) AS last FROM orchestration_events + `; + const lastEventSequence = Number(sequence?.last ?? 0); + yield* hub.sql` + INSERT INTO hub_users (user_id, last_event_sequence) + VALUES (${hub.tenantId}, ${lastEventSequence}) + ON CONFLICT (user_id) DO UPDATE SET last_event_sequence = excluded.last_event_sequence + `; + + let documents = 0; + for (const name of IMPORTED_DOCUMENTS) { + const filePath = path.join(input.stateDir, name); + if (!(yield* fileSystem.exists(filePath))) continue; + const contents = yield* fileSystem.readFileString(filePath); + yield* hub.sql` + INSERT INTO hub_documents (user_id, name, contents) + VALUES (${hub.tenantId}, ${name}, ${contents}) + `; + documents += 1; + } + + let secrets = 0; + const secretsDir = path.join(input.stateDir, "secrets"); + if (yield* fileSystem.exists(secretsDir)) { + for (const entry of (yield* fileSystem.readDirectory(secretsDir)).toSorted()) { + if (!entry.startsWith(IMPORTED_SECRET_PREFIX) || !entry.endsWith(".bin")) continue; + const name = entry.slice(0, -".bin".length); + const plaintext = yield* fileSystem.readFile(path.join(secretsDir, entry)); + const stored = encryptHubSecret({ key, tenantId: hub.tenantId, name, plaintext }); + yield* hub.sql` + INSERT INTO hub_secrets (user_id, name, format, nonce, ciphertext) + VALUES (${hub.tenantId}, ${name}, ${stored.format}, ${stored.nonce}, ${stored.ciphertext}) + `; + secrets += 1; + } + } + + let attachments = 0; + let attachmentsSkipped = 0; + const attachmentsDir = path.join(input.stateDir, "attachments"); + if (yield* fileSystem.exists(attachmentsDir)) { + for (const entry of (yield* fileSystem.readDirectory(attachmentsDir)).toSorted()) { + const attachmentId = entry.endsWith(".part") + ? null + : parseAttachmentIdFromRelativePath(entry); + const threadSegment = attachmentId + ? parseThreadSegmentFromAttachmentId(attachmentId) + : null; + const filePath = path.join(attachmentsDir, entry); + const info = yield* fileSystem.stat(filePath); + if ( + !attachmentId || + !threadSegment || + info.type !== "File" || + Number(info.size) > PROVIDER_SEND_TURN_MAX_FILE_BYTES + ) { + attachmentsSkipped += 1; + continue; + } + const content = yield* fileSystem.readFile(filePath); + yield* hub.sql` + INSERT INTO hub_attachments ( + user_id, relative_path, attachment_id, thread_segment, size_bytes, content + ) + VALUES ( + ${hub.tenantId}, ${entry}, ${attachmentId}, ${threadSegment}, + ${content.byteLength}, ${content} + ) + `; + attachments += 1; + } + } + + let repositoryIdentities = 0; + if (input.resolveRepositoryIdentity !== undefined) { + const projects = yield* hub.sql<{ + readonly projectId: string; + readonly workspaceRoot: string; + }>` + SELECT project_id AS "projectId", workspace_root AS "workspaceRoot" + FROM projection_projects + WHERE user_id = ${hub.tenantId} AND deleted_at IS NULL + `; + for (const project of projects) { + if (!(yield* fileSystem.exists(project.workspaceRoot))) continue; + const identity = yield* input.resolveRepositoryIdentity(project.workspaceRoot); + if (identity === null) continue; + yield* hub.sql` + UPDATE projection_projects + SET repository_identity_json = ${encodeRepositoryIdentityJson(identity)} + WHERE user_id = ${hub.tenantId} AND project_id = ${project.projectId} + `; + repositoryIdentities += 1; + } + } + + return { + rows, + lastEventSequence, + documents, + attachments, + attachmentsSkipped, + secrets, + repositoryIdentities, + replacedExisting: occupied.length > 0, + } satisfies HubImportReport; + }), + ); + }); diff --git a/apps/server/src/persistence/Postgres/HubRepositories.test.ts b/apps/server/src/persistence/Postgres/HubRepositories.test.ts index 77bb4781b6e3..6eb29eeb06a8 100644 --- a/apps/server/src/persistence/Postgres/HubRepositories.test.ts +++ b/apps/server/src/persistence/Postgres/HubRepositories.test.ts @@ -9,27 +9,18 @@ import { AuthStandardClientScopes, AuthSessionId, CheckpointRef, - CommandId, - DEFAULT_PROVIDER_INTERACTION_MODE, - EventId, - MessageId, - ProjectId, ProviderInstanceId, ThreadId, - TurnId, - type OrchestrationCommand, } from "@t3tools/contracts"; import { assert, describe, it } from "@effect/vitest"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as Stream from "effect/Stream"; import * as Reactivity from "effect/unstable/reactivity/Reactivity"; import * as SqlClient from "effect/unstable/sql/SqlClient"; import * as ServerConfig from "../../config.ts"; -import { OrchestrationEngineService } from "../../orchestration/Services/OrchestrationEngine.ts"; import { ProjectionSnapshotQuery } from "../../orchestration/Services/ProjectionSnapshotQuery.ts"; import { OrchestrationLayerLive } from "../../orchestration/runtimeLayer.ts"; import { RepositoryIdentityResolver } from "../../project/RepositoryIdentityResolver.ts"; @@ -45,6 +36,7 @@ import { makeHubTestSchema, type HubTestSchema, } from "./hubTestDatabase.ts"; +import { at, orchestrationScenario, threadId, turnId } from "./hubTestScenario.ts"; const withSchema = (use: (schema: HubTestSchema) => Effect.Effect) => Effect.scoped(Effect.flatMap(makeHubTestSchema(hubTestDatabaseUrl!), use)).pipe( @@ -66,189 +58,6 @@ const hub = (schema: HubTestSchema, tenantId: string) => Layer.provide(platform), ); -const at = (second: number) => `2026-09-25T00:00:${String(second).padStart(2, "0")}.000Z`; -const modelSelection = { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5-codex" }; -const projectId = ProjectId.make("project-shared"); -const threadId = ThreadId.make("thread-shared"); -const otherThreadId = ThreadId.make("thread-archived"); -const turnId = TurnId.make("turn-1"); - -// Both tenants reuse the same ids: isolation must come from the tenant key. -const commandsFor = (label: string): ReadonlyArray => [ - { - type: "project.create", - commandId: CommandId.make("cmd-project-create"), - projectId, - title: `Project of ${label}`, - workspaceRoot: `/workspace/p/${projectId}`, - defaultModelSelection: modelSelection, - createdAt: at(1), - }, - { - type: "thread.create", - commandId: CommandId.make("cmd-thread-create"), - threadId, - projectId, - title: `Thread of ${label}`, - modelSelection, - interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, - runtimeMode: "approval-required", - branch: null, - worktreePath: `/workspace/t/${threadId}`, - createdAt: at(2), - }, - { - type: "thread.turn.start", - commandId: CommandId.make("cmd-turn-start"), - threadId, - message: { - messageId: MessageId.make("msg-user-1"), - role: "user", - text: `hello from ${label}`, - attachments: [], - }, - interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, - runtimeMode: "approval-required", - createdAt: at(3), - }, - { - type: "thread.session.set", - commandId: CommandId.make("cmd-session-set"), - threadId, - session: { - threadId, - status: "running", - providerName: "codex", - providerInstanceId: ProviderInstanceId.make("codex"), - runtimeMode: "approval-required", - activeTurnId: turnId, - lastError: null, - updatedAt: at(4), - }, - createdAt: at(4), - }, - { - type: "thread.activity.append", - commandId: CommandId.make("cmd-activity-1"), - threadId, - activity: { - id: EventId.make("activity-1"), - tone: "tool", - kind: "tool.completed", - summary: `tool ran for ${label}`, - // JSON payloads may carry escaped NULs; Postgres keeps them as text. - payload: { owner: label, output: "binary\u0000tail" }, - turnId, - createdAt: at(5), - }, - createdAt: at(5), - }, - { - type: "thread.message.assistant.delta", - commandId: CommandId.make("cmd-assistant-delta-1"), - threadId, - messageId: MessageId.make("msg-assistant-1"), - delta: `reply to ${label}`, - turnId, - createdAt: at(6), - }, - { - type: "thread.message.assistant.complete", - commandId: CommandId.make("cmd-assistant-complete-1"), - threadId, - messageId: MessageId.make("msg-assistant-1"), - turnId, - createdAt: at(7), - }, - { - type: "thread.proposed-plan.upsert", - commandId: CommandId.make("cmd-plan-1"), - threadId, - proposedPlan: { - id: "plan-1", - turnId, - planMarkdown: `# Plan for ${label}`, - implementedAt: null, - implementationThreadId: null, - createdAt: at(8), - updatedAt: at(8), - }, - createdAt: at(8), - }, - { - type: "thread.turn.diff.complete", - commandId: CommandId.make("cmd-diff-1"), - threadId, - turnId, - completedAt: at(9), - checkpointRef: CheckpointRef.make("refs/t3/checkpoints/thread-shared/turn/1"), - status: "ready", - files: [{ path: "notes.txt", kind: "modified", additions: 1, deletions: 0 }], - assistantMessageId: MessageId.make("msg-assistant-1"), - checkpointTurnCount: 1, - createdAt: at(9), - }, - { - type: "thread.meta.update", - commandId: CommandId.make("cmd-thread-rename"), - threadId, - title: `Renamed thread of ${label}`, - }, - { - type: "thread.create", - commandId: CommandId.make("cmd-thread-archived-create"), - threadId: otherThreadId, - projectId, - title: `Archived thread of ${label}`, - modelSelection, - interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, - runtimeMode: "full-access", - branch: null, - worktreePath: null, - createdAt: at(10), - }, - { - type: "thread.archive", - commandId: CommandId.make("cmd-thread-archive"), - threadId: otherThreadId, - }, -]; - -/** Dispatches the scenario and reads back through every snapshot query method. */ -const orchestrationScenario = (label: string) => - Effect.gen(function* () { - const engine = yield* OrchestrationEngineService; - const query = yield* ProjectionSnapshotQuery; - for (const command of commandsFor(label)) { - yield* engine.dispatch(command); - } - const events = yield* Stream.runCollect(engine.readEvents(0)); - return { - events: Array.from(events, (event) => ({ - sequence: event.sequence, - type: event.type, - aggregateId: event.aggregateId, - payload: event.payload, - })), - commandReadModel: yield* query.getCommandReadModel(), - snapshot: yield* query.getSnapshot(), - shell: yield* query.getShellSnapshot(), - archivedShell: yield* query.getArchivedShellSnapshot(), - counts: yield* query.getCounts(), - snapshotSequence: yield* query.getSnapshotSequence(), - projectByRoot: yield* query.getActiveProjectByWorkspaceRoot(`/workspace/p/${projectId}`), - projectShell: yield* query.getProjectShellById(projectId), - firstThread: yield* query.getFirstActiveThreadIdByProjectId(projectId), - checkpointContext: yield* query.getThreadCheckpointContext(threadId), - fullDiffContext: yield* query.getFullThreadDiffContext(threadId, 1), - threadShell: yield* query.getThreadShellById(threadId), - threadDetail: yield* query.getThreadDetailById(threadId), - detailSnapshot: yield* query.getThreadDetailSnapshot(threadId), - windowedDetail: yield* query.getThreadDetailSnapshot(threadId, { turnLimit: 1 }), - search: yield* query.searchThreads({ query: "RENAMED" }), - }; - }); - const runOrchestration = (persistence: Layer.Layer, label: string) => orchestrationScenario(label).pipe( Effect.provide( diff --git a/apps/server/src/persistence/Postgres/hubTestScenario.ts b/apps/server/src/persistence/Postgres/hubTestScenario.ts new file mode 100644 index 000000000000..27244b7b1a9a --- /dev/null +++ b/apps/server/src/persistence/Postgres/hubTestScenario.ts @@ -0,0 +1,219 @@ +/** + * A shared orchestration scenario for hub tests: commands that touch every + * projection, and a read of every snapshot query method. + */ +import { + CheckpointRef, + CommandId, + DEFAULT_PROVIDER_INTERACTION_MODE, + EventId, + MessageId, + ProjectId, + ProviderInstanceId, + ThreadId, + TurnId, + type OrchestrationCommand, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; + +import { OrchestrationEngineService } from "../../orchestration/Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../../orchestration/Services/ProjectionSnapshotQuery.ts"; + +export const at = (second: number) => `2026-09-25T00:00:${String(second).padStart(2, "0")}.000Z`; +export const modelSelection = { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", +}; +export const projectId = ProjectId.make("project-shared"); +export const threadId = ThreadId.make("thread-shared"); +export const otherThreadId = ThreadId.make("thread-archived"); +export const turnId = TurnId.make("turn-1"); + +// Both tenants reuse the same ids: isolation must come from the tenant key. +export const commandsFor = ( + label: string, + workspaceRoot = `/workspace/p/${projectId}`, +): ReadonlyArray => [ + { + type: "project.create", + commandId: CommandId.make("cmd-project-create"), + projectId, + title: `Project of ${label}`, + workspaceRoot, + defaultModelSelection: modelSelection, + createdAt: at(1), + }, + { + type: "thread.create", + commandId: CommandId.make("cmd-thread-create"), + threadId, + projectId, + title: `Thread of ${label}`, + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: `/workspace/t/${threadId}`, + createdAt: at(2), + }, + { + type: "thread.turn.start", + commandId: CommandId.make("cmd-turn-start"), + threadId, + message: { + messageId: MessageId.make("msg-user-1"), + role: "user", + text: `hello from ${label}`, + attachments: [], + }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: at(3), + }, + { + type: "thread.session.set", + commandId: CommandId.make("cmd-session-set"), + threadId, + session: { + threadId, + status: "running", + providerName: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "approval-required", + activeTurnId: turnId, + lastError: null, + updatedAt: at(4), + }, + createdAt: at(4), + }, + { + type: "thread.activity.append", + commandId: CommandId.make("cmd-activity-1"), + threadId, + activity: { + id: EventId.make("activity-1"), + tone: "tool", + kind: "tool.completed", + summary: `tool ran for ${label}`, + // JSON payloads may carry escaped NULs; Postgres keeps them as text. + payload: { owner: label, output: "binary\u0000tail" }, + turnId, + createdAt: at(5), + }, + createdAt: at(5), + }, + { + type: "thread.message.assistant.delta", + commandId: CommandId.make("cmd-assistant-delta-1"), + threadId, + messageId: MessageId.make("msg-assistant-1"), + delta: `reply to ${label}`, + turnId, + createdAt: at(6), + }, + { + type: "thread.message.assistant.complete", + commandId: CommandId.make("cmd-assistant-complete-1"), + threadId, + messageId: MessageId.make("msg-assistant-1"), + turnId, + createdAt: at(7), + }, + { + type: "thread.proposed-plan.upsert", + commandId: CommandId.make("cmd-plan-1"), + threadId, + proposedPlan: { + id: "plan-1", + turnId, + planMarkdown: `# Plan for ${label}`, + implementedAt: null, + implementationThreadId: null, + createdAt: at(8), + updatedAt: at(8), + }, + createdAt: at(8), + }, + { + type: "thread.turn.diff.complete", + commandId: CommandId.make("cmd-diff-1"), + threadId, + turnId, + completedAt: at(9), + checkpointRef: CheckpointRef.make("refs/t3/checkpoints/thread-shared/turn/1"), + status: "ready", + files: [{ path: "notes.txt", kind: "modified", additions: 1, deletions: 0 }], + assistantMessageId: MessageId.make("msg-assistant-1"), + checkpointTurnCount: 1, + createdAt: at(9), + }, + { + type: "thread.meta.update", + commandId: CommandId.make("cmd-thread-rename"), + threadId, + title: `Renamed thread of ${label}`, + }, + { + type: "thread.create", + commandId: CommandId.make("cmd-thread-archived-create"), + threadId: otherThreadId, + projectId, + title: `Archived thread of ${label}`, + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt: at(10), + }, + { + type: "thread.archive", + commandId: CommandId.make("cmd-thread-archive"), + threadId: otherThreadId, + }, +]; + +/** Dispatches the scenario and reads back through every snapshot query method. */ +export const orchestrationScenario = (label: string, workspaceRoot?: string) => + Effect.gen(function* () { + const engine = yield* OrchestrationEngineService; + for (const command of commandsFor(label, workspaceRoot)) { + yield* engine.dispatch(command); + } + return yield* readScenario(workspaceRoot); + }); + +/** Reads the scenario back through the event stream and every snapshot query. */ +export const readScenario = (workspaceRoot?: string) => + Effect.gen(function* () { + const engine = yield* OrchestrationEngineService; + const query = yield* ProjectionSnapshotQuery; + const events = yield* Stream.runCollect(engine.readEvents(0)); + return { + events: Array.from(events, (event) => ({ + sequence: event.sequence, + type: event.type, + aggregateId: event.aggregateId, + payload: event.payload, + })), + commandReadModel: yield* query.getCommandReadModel(), + snapshot: yield* query.getSnapshot(), + shell: yield* query.getShellSnapshot(), + archivedShell: yield* query.getArchivedShellSnapshot(), + counts: yield* query.getCounts(), + snapshotSequence: yield* query.getSnapshotSequence(), + projectByRoot: yield* query.getActiveProjectByWorkspaceRoot( + workspaceRoot ?? `/workspace/p/${projectId}`, + ), + projectShell: yield* query.getProjectShellById(projectId), + firstThread: yield* query.getFirstActiveThreadIdByProjectId(projectId), + checkpointContext: yield* query.getThreadCheckpointContext(threadId), + fullDiffContext: yield* query.getFullThreadDiffContext(threadId, 1), + threadShell: yield* query.getThreadShellById(threadId), + threadDetail: yield* query.getThreadDetailById(threadId), + detailSnapshot: yield* query.getThreadDetailSnapshot(threadId), + windowedDetail: yield* query.getThreadDetailSnapshot(threadId, { turnLimit: 1 }), + search: yield* query.searchThreads({ query: "RENAMED" }), + }; + }); From 0fe9292259182ae0f65ee52e11cb45c1dfb9fb23 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 23:08:59 +0000 Subject: [PATCH 15/58] fix(server): accept libpq sslrootcert=system in hub database URLs node-postgres reads sslrootcert as a file path, so a PlanetScale-style URL with sslrootcert=system failed to connect. Drop that parameter; Node verifies against its bundled CA store. Hub tests now create roles only in a loopback cluster and skip the RLS role check when the only remote role bypasses RLS, and the boot test allowlists the base directory: only logs, caches, and the live runtime state file may exist. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/persistence/Postgres/HubClient.ts | 16 ++++++++++- .../persistence/Postgres/HubDatabase.test.ts | 8 +++++- .../persistence/Postgres/hubTestDatabase.ts | 13 ++++++++- apps/server/src/server.hub.test.ts | 28 +++++++++++-------- 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/apps/server/src/persistence/Postgres/HubClient.ts b/apps/server/src/persistence/Postgres/HubClient.ts index af02d34516aa..9412d27bba7d 100644 --- a/apps/server/src/persistence/Postgres/HubClient.ts +++ b/apps/server/src/persistence/Postgres/HubClient.ts @@ -80,10 +80,24 @@ export interface HubPoolOptions { readonly applicationName?: string; } +/** + * libpq accepts `sslrootcert=system` (use the platform's CA store); node-postgres + * would try to read a file named "system". Node verifies against its bundled + * CA store by default, so the parameter is dropped. + */ +export const normalizeHubDatabaseUrl = (url: string): string => { + const parsed = new URL(url); + if (parsed.searchParams.get("sslrootcert") !== "system") { + return url; + } + parsed.searchParams.delete("sslrootcert"); + return parsed.toString(); +}; + /** A plain pooled client: migrations and administration only (no tenant). */ export const makeHubPool = (options: HubPoolOptions) => PgClient.make({ - url: Redacted.make(options.url), + url: Redacted.make(normalizeHubDatabaseUrl(options.url)), // Many hub processes share one database, and PlanetScale's direct port // allows few connections. One user's engine is a single writer, so a // handful of connections is enough. diff --git a/apps/server/src/persistence/Postgres/HubDatabase.test.ts b/apps/server/src/persistence/Postgres/HubDatabase.test.ts index 53d89b258c33..aa859a8e488d 100644 --- a/apps/server/src/persistence/Postgres/HubDatabase.test.ts +++ b/apps/server/src/persistence/Postgres/HubDatabase.test.ts @@ -127,9 +127,15 @@ describe.skipIf(hubTestDatabaseUrl === undefined)("hub database", () => { ), ); - it.effect("fails closed for a role without BYPASSRLS", () => + it.effect("fails closed for a role without BYPASSRLS", (context) => withSchema((schema) => Effect.gen(function* () { + if (!schema.runtimeSubjectToRls) { + // A remote database whose only role bypasses RLS; roles are never + // created outside a local cluster. + context.skip(); + return; + } const database = yield* HubDatabase; const tenantA = database!.sql; const runtime = yield* makeHubPool({ url: schema.runtimeUrl, maxConnections: 2 }); diff --git a/apps/server/src/persistence/Postgres/hubTestDatabase.ts b/apps/server/src/persistence/Postgres/hubTestDatabase.ts index 5d2af544de62..8a4d2298e779 100644 --- a/apps/server/src/persistence/Postgres/hubTestDatabase.ts +++ b/apps/server/src/persistence/Postgres/hubTestDatabase.ts @@ -36,8 +36,15 @@ export interface HubTestSchema { readonly runtimeUrl: string; /** Whether a separate role without BYPASSRLS was created for the runtime. */ readonly separateRuntimeRole: boolean; + /** Whether the runtime role is subject to row-level security. */ + readonly runtimeSubjectToRls: boolean; } +const isLoopbackDatabase = (url: string) => { + const host = new URL(url).hostname; + return host === "127.0.0.1" || host === "localhost" || host === "[::1]" || host === "::1"; +}; + const withSearchPath = (url: string, schema: string, credentials?: [string, string]) => { const parsed = new URL(url); parsed.searchParams.set("options", `-c search_path=${schema}`); @@ -62,7 +69,10 @@ export const makeHubTestSchema = (baseUrl: string) => readonly rolsuper: boolean; readonly rolbypassrls: boolean; }>`SELECT rolsuper, rolbypassrls FROM pg_roles WHERE rolname = current_user`; - const needsRuntimeRole = attributes?.rolsuper === true || attributes?.rolbypassrls === true; + const bypassesRls = attributes?.rolsuper === true || attributes?.rolbypassrls === true; + // Only ever create roles in a private local cluster; elsewhere a test may + // touch nothing but its own schema. + const needsRuntimeRole = bypassesRls && isLoopbackDatabase(baseUrl); const runtimeRole = `${schema}_app`; const runtimePassword = NodeCrypto.randomBytes(12).toString("hex"); @@ -98,6 +108,7 @@ export const makeHubTestSchema = (baseUrl: string) => ? withSearchPath(baseUrl, schema, [runtimeRole, runtimePassword]) : adminUrl, separateRuntimeRole: needsRuntimeRole, + runtimeSubjectToRls: needsRuntimeRole || !bypassesRls, } satisfies HubTestSchema; }).pipe(Effect.provide(Reactivity.layer)); diff --git a/apps/server/src/server.hub.test.ts b/apps/server/src/server.hub.test.ts index 9e227c7de2c7..ac7b4fe2e259 100644 --- a/apps/server/src/server.hub.test.ts +++ b/apps/server/src/server.hub.test.ts @@ -24,6 +24,7 @@ import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; import * as Reactivity from "effect/unstable/reactivity/Reactivity"; @@ -110,22 +111,25 @@ const startHub = (schema: HubTestSchema, baseDir: string) => }; }); +/** + * Files a hub leaves in its base directory that are not disposable: anything + * but logs, caches, and the runtime state file of the live process. + */ const durableStateFiles = (config: ServerConfig.ServerConfig["Service"]) => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; - const candidates = [ - config.dbPath, - config.settingsPath, - config.keybindingsConfigPath, - config.environmentIdPath, - config.anonymousIdPath, - config.secretsDir, - ]; - const present: Array = []; - for (const candidate of candidates) { - if (yield* fileSystem.exists(candidate)) present.push(candidate); + const path = yield* Path.Path; + const disposable = (absolute: string) => + absolute === config.serverRuntimeStatePath || + absolute.startsWith(`${config.logsDir}${path.sep}`) || + absolute.startsWith(`${config.providerStatusCacheDir}${path.sep}`); + const durable: Array = []; + for (const entry of yield* fileSystem.readDirectory(config.baseDir, { recursive: true })) { + const absolute = path.join(config.baseDir, entry); + const info = yield* fileSystem.stat(absolute); + if (info.type !== "Directory" && !disposable(absolute)) durable.push(entry); } - return present; + return durable; }); describe.skipIf(hubTestDatabaseUrl === undefined)("hub server", () => { From 3bbf4afbf328be260f296ce62934f93fa3f100cd Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 23:12:46 +0000 Subject: [PATCH 16/58] docs: describe what the hub persists and how to operate it Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/internals/thread-machines.md | 55 ++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/docs/internals/thread-machines.md b/docs/internals/thread-machines.md index 04ada5089536..e531299431b2 100644 --- a/docs/internals/thread-machines.md +++ b/docs/internals/thread-machines.md @@ -108,8 +108,53 @@ deduplicates by `(threadId, runner sequence)`. ## What the hub persists -Everything a standalone server keeps in its state directory moves to Postgres, scoped by -`user_id` with row-level security as a backstop: orchestration events, projections, command -receipts, provider session runtime, auth sessions and pairing links, settings, keybindings, -secrets (encrypted with `T3CODE_HUB_SECRET_KEY`), the environment ID, attachments, per-turn -diffs, repository identity, and the last reported VCS status per thread. +A hub process serves exactly one tenant (`T3CODE_HUB_TENANT_ID`) and its base directory is +disposable: it holds only logs, caches, and the live process's runtime state file. Everything +a standalone server keeps in its state directory lives in Postgres instead, in tables whose +keys and indexes all start with `user_id`: + +| Standalone | Hub | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| `state.sqlite`: events, receipts, projections, provider session runtime | The same tables in Postgres; event sequences are per tenant (`hub_users.last_event_sequence`) | +| `state.sqlite`: auth sessions and pairing links | `auth_sessions`, `auth_pairing_links`; T3's pairing, browser sessions, and bearer tokens work unchanged | +| `settings.json`, `keybindings.json`, `environment-id`, `anonymous-id` | `hub_documents` rows named after the files; in-process notifications replace the file watcher | +| `secrets/*.bin` | `hub_secrets`, AES-256-GCM with `T3CODE_HUB_SECRET_KEY`; the authenticated data binds tenant and secret name | +| `attachments/` | `hub_attachments` (`bytea`); the local directory is a cache filled on lookup | +| Repository identity from `git remote` on every read | `projection_projects.repository_identity_json`, recorded by `project.create` / `project.meta.update` | +| `checkpoint_diff_blobs` (unused standalone) | Created for the per-turn diff cache | + +Machine-published themes stay local (a hub has no desktop to publish them). The last +reported VCS status per thread and the runner cursors belong to the thread-machine +migrations. + +In code, hub mode is a `HubDatabase` reference provided at the server and auth CLI roots +(`apps/server/src/persistence/Postgres/`). The SQLite persistence layer hands out its client, +each repository layer selects its Postgres port through `localOrHub`, and file-backed stores +branch on it. Standalone layers, types, and behavior are unchanged. + +## Operations + +**Migrations.** Hub migrations are numbered entries in +`apps/server/src/persistence/Postgres/migrations/index.ts`: 001–049 for hub persistence, +050–099 for thread-machine state. A hub applies pending ones at startup with +`T3CODE_HUB_DATABASE_ADMIN_URL` (or the runtime URL), each in its own transaction under a +transaction-scoped advisory lock, recorded in `hub_schema_migrations`. They are applied by id, +not list position, so separately owned ranges can land in any order. Keep them +expand/contract compatible: an older hub may still be running. + +**Roles and row-level security.** Every tenant table has a forced policy comparing `user_id` +with the transaction-local `hub.user_id`. The runtime client sets it with `SET LOCAL` at the +start of every transaction and wraps statements outside one in their own transaction (two +extra round trips), so it works behind a transaction pooler and never leaks between clients. +Without the setting a query sees nothing and cannot write. Use a runtime role without +`BYPASSRLS` (PlanetScale's default role has it) and pass the owner as the admin URL; after +migrating, the hub grants the runtime role DML on the schema's tables. The explicit +`user_id` predicates remain the primary isolation; RLS is the backstop. + +**Import.** `t3 hub import ` copies a standalone state directory into the tenant +named by the `T3CODE_HUB_*` environment in one transaction: the database (read-only, paged, +at the current standalone migration), the documents, attachments within the upload limit, +and provider environment secrets re-encrypted with the hub key. Auth sessions, pairing +links, and other secrets stay behind. It refuses a tenant with data unless `--replace`, +records repository identities for projects whose checkouts exist on the machine, and prints +counts only. From 6e4e80205259b879e26ad106f3dac8c7169c22a1 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 23:15:26 +0000 Subject: [PATCH 17/58] perf(server): load the Postgres driver only when a hub command needs it Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/cli/hub.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/server/src/cli/hub.ts b/apps/server/src/cli/hub.ts index 560d9e0271ab..ebe0a873d8eb 100644 --- a/apps/server/src/cli/hub.ts +++ b/apps/server/src/cli/hub.ts @@ -12,7 +12,6 @@ import type { SqlError } from "effect/unstable/sql/SqlError"; import { Argument, Command, Flag } from "effect/unstable/cli"; import { expandHomePath } from "../os-jank.ts"; -import { makeHubDatabase } from "../persistence/Postgres/HubDatabaseLive.ts"; import { HubImportError, importStandaloneState, @@ -92,6 +91,10 @@ const importCommand = Command.make("import", { }); } + // Loaded on use so other `t3` commands never load the Postgres driver. + const { makeHubDatabase } = yield* Effect.promise( + () => import("../persistence/Postgres/HubDatabaseLive.ts"), + ); const hub = yield* makeHubDatabase({ databaseUrl: Redacted.value(env.databaseUrl), databaseAdminUrl: Option.getOrUndefined(Option.map(env.databaseAdminUrl, Redacted.value)), From 6454e002e019eafe05d4898c3e6719fec1dbe48b Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 23:17:23 +0000 Subject: [PATCH 18/58] fix(server): cover later tenant tables in hub import and protect migration history Import discovers tenant tables by their user_id column, so --replace and the non-empty check also cover tables added by later migrations. The runtime role keeps read access to hub_schema_migrations but can no longer rewrite it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/persistence/Postgres/HubDatabase.test.ts | 10 ++++++++++ .../src/persistence/Postgres/HubDatabaseLive.ts | 4 ++++ apps/server/src/persistence/Postgres/HubImport.ts | 11 +++++++++-- .../src/persistence/Postgres/migrations/index.ts | 4 ++++ docs/internals/thread-machines.md | 8 +++++--- 5 files changed, 32 insertions(+), 5 deletions(-) diff --git a/apps/server/src/persistence/Postgres/HubDatabase.test.ts b/apps/server/src/persistence/Postgres/HubDatabase.test.ts index aa859a8e488d..e7a18fdf1a42 100644 --- a/apps/server/src/persistence/Postgres/HubDatabase.test.ts +++ b/apps/server/src/persistence/Postgres/HubDatabase.test.ts @@ -185,6 +185,16 @@ describe.skipIf(hubTestDatabaseUrl === undefined)("hub database", () => { SELECT contents FROM hub_documents WHERE user_id = 'tenant-a' `; assert.strictEqual(row?.contents, "{}"); + + if (schema.separateRuntimeRole) { + // The runtime role reads the migration history but cannot rewrite it. + const history = yield* runtime<{ + readonly id: number; + }>`SELECT id FROM hub_schema_migrations`; + assert.isAbove(history.length, 0); + const rewrite = yield* runtime`DELETE FROM hub_schema_migrations`.pipe(Effect.exit); + assert.isTrue(Exit.isFailure(rewrite)); + } }).pipe(Effect.provide(hubTestDatabaseLayer(schema, "tenant-a"))), ), ); diff --git a/apps/server/src/persistence/Postgres/HubDatabaseLive.ts b/apps/server/src/persistence/Postgres/HubDatabaseLive.ts index d5d6d44d08e6..73c88b6e9b63 100644 --- a/apps/server/src/persistence/Postgres/HubDatabaseLive.ts +++ b/apps/server/src/persistence/Postgres/HubDatabaseLive.ts @@ -48,6 +48,10 @@ const grantRuntimeRole = (admin: SqlClient.SqlClient, runtimeRole: string) => `GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA ${schema} TO ${role}`, ); yield* admin.unsafe(`GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA ${schema} TO ${role}`); + // The runtime role may read the migration history but not rewrite it. + yield* admin.unsafe( + `REVOKE INSERT, UPDATE, DELETE ON ${schema}.hub_schema_migrations FROM ${role}`, + ); }).pipe( Effect.catchCause((cause) => Effect.logWarning("Could not grant the hub runtime role access to hub tables.", { diff --git a/apps/server/src/persistence/Postgres/HubImport.ts b/apps/server/src/persistence/Postgres/HubImport.ts index db1a5b90d32b..7fc47a96863b 100644 --- a/apps/server/src/persistence/Postgres/HubImport.ts +++ b/apps/server/src/persistence/Postgres/HubImport.ts @@ -37,7 +37,6 @@ import { import { migrationEntries } from "../Migrations.ts"; import type { HubDatabaseShape } from "./HubDatabase.ts"; import { encryptHubSecret, parseHubSecretKey } from "./HubSecretCipher.ts"; -import { HUB_BASELINE_TENANT_TABLES } from "./migrations/001_HubBaseline.ts"; export class HubImportError extends Schema.TaggedErrorClass()("HubImportError", { reason: Schema.Literals([ @@ -213,8 +212,16 @@ export const importStandaloneState = (input: HubImportInput) => return yield* hub.sql.withTransaction( Effect.gen(function* () { + // Every tenant table leads with user_id, including ones added by later + // migrations. + const tenantTables = (yield* hub.sql<{ readonly table: string }>` + SELECT DISTINCT table_name AS "table" + FROM information_schema.columns + WHERE table_schema = current_schema() AND column_name = 'user_id' + ORDER BY table_name + `).map((row) => row.table); const occupied: Array = []; - for (const table of HUB_BASELINE_TENANT_TABLES) { + for (const table of tenantTables) { const [row] = yield* hub.sql<{ readonly present: boolean }>` SELECT EXISTS ( SELECT 1 FROM ${hub.sql(table)} WHERE user_id = ${hub.tenantId} diff --git a/apps/server/src/persistence/Postgres/migrations/index.ts b/apps/server/src/persistence/Postgres/migrations/index.ts index d7397ba95136..bf3a98230fb1 100644 --- a/apps/server/src/persistence/Postgres/migrations/index.ts +++ b/apps/server/src/persistence/Postgres/migrations/index.ts @@ -2,6 +2,10 @@ * Hub migration registry. Add a numbered file under this directory and list it * in the range that owns it; `runHubMigrations` orders entries by id, so the * position in these lists does not matter. + * + * Tenant tables lead every key and index with `user_id` and enable the tenant + * policy (`hubTenantPolicyStatements` in 001). Migrations must stay compatible + * with the previous hub release, which may still be running. */ import { orderHubMigrations, type HubMigrationEntry } from "../HubMigrator.ts"; import Migration001 from "./001_HubBaseline.ts"; diff --git a/docs/internals/thread-machines.md b/docs/internals/thread-machines.md index e531299431b2..eb349fb3150c 100644 --- a/docs/internals/thread-machines.md +++ b/docs/internals/thread-machines.md @@ -148,13 +148,15 @@ start of every transaction and wraps statements outside one in their own transac extra round trips), so it works behind a transaction pooler and never leaks between clients. Without the setting a query sees nothing and cannot write. Use a runtime role without `BYPASSRLS` (PlanetScale's default role has it) and pass the owner as the admin URL; after -migrating, the hub grants the runtime role DML on the schema's tables. The explicit -`user_id` predicates remain the primary isolation; RLS is the backstop. +migrating, the hub grants the runtime role DML on the schema's tables (read-only on the +migration history). The explicit `user_id` predicates remain the primary isolation; RLS is +the backstop. Each hub keeps a pool of at most four connections. **Import.** `t3 hub import ` copies a standalone state directory into the tenant named by the `T3CODE_HUB_*` environment in one transaction: the database (read-only, paged, at the current standalone migration), the documents, attachments within the upload limit, and provider environment secrets re-encrypted with the hub key. Auth sessions, pairing links, and other secrets stay behind. It refuses a tenant with data unless `--replace`, -records repository identities for projects whose checkouts exist on the machine, and prints +which first deletes every row the tenant has (sessions and hub secrets included), records +repository identities for projects whose checkouts exist on the machine, and prints counts only. From 492d81b8864b9fde91cf90d65a73acb86fe1f883 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 23:24:18 +0000 Subject: [PATCH 19/58] fix(server): resubscribe runner delivery and sweep idle runners on demand Runner event delivery resubscribes from the last delivered sequence when its stream fails on a live connection, instead of stalling until the connection drops. The connection pool exposes its idle sweep, and provider capability reports tolerate runner defects. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/hub/RemoteProviderDriver.ts | 5 +++-- apps/server/src/hub/RunnerConnectionPool.ts | 15 +++++++++++++-- apps/server/src/hub/RunnerEventDelivery.ts | 21 ++++++++++++++++++++- apps/server/src/runner/RunnerServer.ts | 2 +- 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/apps/server/src/hub/RemoteProviderDriver.ts b/apps/server/src/hub/RemoteProviderDriver.ts index c788601f2b5e..05104df6bc09 100644 --- a/apps/server/src/hub/RemoteProviderDriver.ts +++ b/apps/server/src/hub/RemoteProviderDriver.ts @@ -44,6 +44,7 @@ import { type RunnerMcpSession, ThreadMachineUnavailableError, } from "@t3tools/contracts/runner"; +import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; @@ -428,10 +429,10 @@ const makeRemoteSnapshot = (input: { Effect.tap((snapshot) => Ref.set(current, snapshot)), Effect.tap((snapshot) => PubSub.publish(changes, snapshot)), Effect.asVoid, - Effect.catch((error) => + Effect.catchCause((cause) => Effect.logWarning("runner did not report provider capabilities", { instanceId: context.instanceId, - detail: describe(error), + detail: Cause.pretty(cause).slice(0, 400), }), ), ) diff --git a/apps/server/src/hub/RunnerConnectionPool.ts b/apps/server/src/hub/RunnerConnectionPool.ts index 5c47075652c6..bee3dcc62d86 100644 --- a/apps/server/src/hub/RunnerConnectionPool.ts +++ b/apps/server/src/hub/RunnerConnectionPool.ts @@ -129,6 +129,11 @@ export interface RunnerConnectionPoolShape { readonly setContextResolver: (resolver: ThreadMachineContextResolver) => Effect.Effect; /** Closes the connection and retires the machine. */ readonly release: (threadId: ThreadId) => Effect.Effect; + /** + * Closes connections idle for `idleTimeout` and notifies the directory. + * Runs every `idleCheckInterval`; exposed so callers can sweep on demand. + */ + readonly sweepIdle: Effect.Effect; } export class RunnerConnectionPool extends Context.Service< @@ -607,7 +612,7 @@ export const make = (options: RunnerConnectionPoolOptions = {}) => ); // Idle connections close and tell the platform the machine may sleep. - yield* Effect.gen(function* () { + const sweepIdle = Effect.gen(function* () { const now = yield* Clock.currentTimeMillis; for (const [threadId, slot] of slots) { if (!slot.connection || slot.closing) continue; @@ -625,7 +630,12 @@ export const make = (options: RunnerConnectionPoolOptions = {}) => ), ); } - }).pipe(Effect.delay(idleCheckInterval), Effect.forever, Effect.forkIn(poolScope)); + }); + yield* sweepIdle.pipe( + Effect.delay(idleCheckInterval), + Effect.forever, + Effect.forkIn(poolScope), + ); yield* Effect.addFinalizer(() => Effect.forEach([...slots], ([threadId, slot]) => closeSlot(threadId, slot, true), { @@ -634,6 +644,7 @@ export const make = (options: RunnerConnectionPoolOptions = {}) => ); return RunnerConnectionPool.of({ + sweepIdle, checkoutRoot, checkoutFor, use, diff --git a/apps/server/src/hub/RunnerEventDelivery.ts b/apps/server/src/hub/RunnerEventDelivery.ts index 97611d122456..8a5879ac62e1 100644 --- a/apps/server/src/hub/RunnerEventDelivery.ts +++ b/apps/server/src/hub/RunnerEventDelivery.ts @@ -37,10 +37,12 @@ import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; +import * as Schedule from "effect/Schedule"; import * as Stream from "effect/Stream"; import { RunnerCursorStore } from "../persistence/Services/HubThreadMachineState.ts"; @@ -106,6 +108,11 @@ const closesTurn = (event: ProviderRuntimeEvent) => const MACHINE_RESTARTED = "The thread machine restarted; the turn was interrupted."; const RECONCILE_WAIT = "30 seconds"; +const SUBSCRIPTION_RETRY = Schedule.exponential("250 millis").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(5))), + ), +); export const make = Effect.gen(function* () { const pool = yield* RunnerConnectionPool; @@ -276,7 +283,11 @@ export const make = Effect.gen(function* () { }); yield* reconcileWhenCaughtUp(state.delivered); - yield* client["runner.events.subscribe"]({ afterSequence: state.delivered }).pipe( + // Resubscribes from the last delivered sequence if the stream fails while + // the connection stays open; the connection's scope ends the loop. + yield* Stream.suspend(() => + client["runner.events.subscribe"]({ afterSequence: state.delivered }), + ).pipe( Stream.runForEach((envelope) => Effect.gen(function* () { if (envelope.sequence <= state.delivered) { @@ -288,6 +299,14 @@ export const make = Effect.gen(function* () { yield* reconcileWhenCaughtUp(envelope.sequence); }), ), + Effect.tapCause((cause) => + Effect.logWarning("runner event subscription failed; resubscribing", { + threadId, + delivered: state.delivered, + cause: Cause.pretty(cause).slice(0, 400), + }), + ), + Effect.retry(SUBSCRIPTION_RETRY), ); }); diff --git a/apps/server/src/runner/RunnerServer.ts b/apps/server/src/runner/RunnerServer.ts index 593352070bb1..3214f54310d3 100644 --- a/apps/server/src/runner/RunnerServer.ts +++ b/apps/server/src/runner/RunnerServer.ts @@ -88,7 +88,7 @@ const RunnerProviderInstanceRegistryLive = Layer.unwrap( ); /** Appends every hosted adapter's runtime events to the outbox. */ -const RunnerEventPumpLive = Layer.effectDiscard( +export const RunnerEventPumpLive = Layer.effectDiscard( Effect.gen(function* () { const registry = yield* ProviderInstanceRegistry; const outbox = yield* RunnerOutbox; From c6855e19cb2e7d19f73145dba5fa7fc8091bf835 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 23:24:19 +0000 Subject: [PATCH 20/58] test(server): cover hub routing, wake semantics, delivery and a loopback runner - RunnerOutbox: replay across restarts, monotonic sequences, unique event ids, retention gaps. - RunnerCheckout: clone and branch from the base ref, continue a pushed branch, blank projects, refuse to clobber a non-repository. - Hub state stores on SQLite (Postgres variant runs with T3_HUB_TEST_DATABASE_URL). - RunnerConnectionPool with a fake machine directory: reads never wake, actions wake and wait through starting states, wake timeout, failed and incompatible machines, shared connection attempts, idle close. - RunnerEventDelivery: exactly-once publication, cursors only at turn-safe points, replay after a hub restart, boot reconciliation and lost machines. - Hub checkout services: cached diffs and git status without waking, reads vs writes, paths that are not thread checkouts, terminal wake rules. - Loopback integration: ProviderService and the reactors on a hub against the real runner handlers over WebSocket: a turn with checkpoint and diff served after the runner stops, an approval round trip, a runner restart that settles the open turn and resumes, and a hub restart that replays an open turn without duplicates. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../HubRunnerLoopbackHarness.integration.ts | 517 ++++++++++++++++++ .../hubRunnerLoopback.integration.test.ts | 331 +++++++++++ .../src/hub/HubCheckoutServices.test.ts | 310 +++++++++++ .../src/hub/RunnerConnectionPool.test.ts | 254 +++++++++ .../src/hub/RunnerEventDelivery.test.ts | 316 +++++++++++ apps/server/src/hub/testUtils/runnerServer.ts | 102 ++++ .../Layers/HubThreadMachineState.test.ts | 117 ++++ .../Postgres/HubThreadMachineState.test.ts | 111 ++++ apps/server/src/runner/RunnerCheckout.test.ts | 157 ++++++ apps/server/src/runner/RunnerOutbox.test.ts | 143 +++-- 10 files changed, 2316 insertions(+), 42 deletions(-) create mode 100644 apps/server/integration/HubRunnerLoopbackHarness.integration.ts create mode 100644 apps/server/integration/hubRunnerLoopback.integration.test.ts create mode 100644 apps/server/src/hub/HubCheckoutServices.test.ts create mode 100644 apps/server/src/hub/RunnerConnectionPool.test.ts create mode 100644 apps/server/src/hub/RunnerEventDelivery.test.ts create mode 100644 apps/server/src/hub/testUtils/runnerServer.ts create mode 100644 apps/server/src/persistence/Layers/HubThreadMachineState.test.ts create mode 100644 apps/server/src/persistence/Postgres/HubThreadMachineState.test.ts create mode 100644 apps/server/src/runner/RunnerCheckout.test.ts diff --git a/apps/server/integration/HubRunnerLoopbackHarness.integration.ts b/apps/server/integration/HubRunnerLoopbackHarness.integration.ts new file mode 100644 index 000000000000..00855c949ab8 --- /dev/null +++ b/apps/server/integration/HubRunnerLoopbackHarness.integration.ts @@ -0,0 +1,517 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * A hub and a runner in one process, talking the real runner protocol over a + * loopback WebSocket. + * + * Runner: the production `RunnerRpcHandlersLive` and event pump, a real + * SQLite outbox, real git checkpoints and checkout preparation in + * `/t/`, hosting the scripted `TestProviderAdapter`. + * + * Hub: the orchestration engine, `ProviderService` and reactors on SQLite, + * with the hub layer set for everything checkout-bound: the remote provider + * driver, routed checkpoint store with the diff cache, git status cache, + * workspace routing, reactor hooks, event delivery and its ack loop, and a + * fake machine directory pointing at the runner. + * + * Either side can be restarted: the runner with a new boot (optionally a + * fresh provider adapter that lost its sessions), and the hub on the same + * database. + */ +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { + ApprovalRequestId, + defaultInstanceIdForDriver, + ProviderDriverKind, + ThreadId, + type OrchestrationThread, + type ProviderApprovalDecision, + type ServerProvider, +} from "@t3tools/contracts"; +import { threadCheckoutPath } from "@t3tools/contracts/runner"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Layer from "effect/Layer"; +import * as ManagedRuntime from "effect/ManagedRuntime"; +import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; +import * as Schedule from "effect/Schedule"; +import * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; + +import * as CheckpointStore from "../src/checkpointing/CheckpointStore.ts"; +import { ServerConfig } from "../src/config.ts"; +import * as GitWorkflowService from "../src/git/GitWorkflowService.ts"; +import * as HubLayers from "../src/hub/HubLayers.ts"; +import { hubVcsStatusCacheLayer } from "../src/hub/HubVcs.ts"; +import { MachineDirectory, makeFakeMachineDirectory } from "../src/hub/MachineDirectory.ts"; +import { makeRemoteProviderDriver } from "../src/hub/RemoteProviderDriver.ts"; +import * as RemoteSessionRegistry from "../src/hub/RemoteSessionRegistry.ts"; +import { make as makePool, RunnerConnectionPool } from "../src/hub/RunnerConnectionPool.ts"; +import * as RunnerEventDelivery from "../src/hub/RunnerEventDelivery.ts"; +import { serveRunner } from "../src/hub/testUtils/runnerServer.ts"; +import { CheckpointReactorLive } from "../src/orchestration/Layers/CheckpointReactor.ts"; +import { OrchestrationEngineLive } from "../src/orchestration/Layers/OrchestrationEngine.ts"; +import { OrchestrationReactorLive } from "../src/orchestration/Layers/OrchestrationReactor.ts"; +import { OrchestrationProjectionPipelineLive } from "../src/orchestration/Layers/ProjectionPipeline.ts"; +import { OrchestrationProjectionSnapshotQueryLive } from "../src/orchestration/Layers/ProjectionSnapshotQuery.ts"; +import { ProviderCommandReactorLive } from "../src/orchestration/Layers/ProviderCommandReactor.ts"; +import { ProviderRuntimeIngestionLive } from "../src/orchestration/Layers/ProviderRuntimeIngestion.ts"; +import { RuntimeReceiptBusTest } from "../src/orchestration/Layers/RuntimeReceiptBus.ts"; +import { + OrchestrationEngineService, + type OrchestrationEngineShape, +} from "../src/orchestration/Services/OrchestrationEngine.ts"; +import { OrchestrationReactor } from "../src/orchestration/Services/OrchestrationReactor.ts"; +import { ProjectionSnapshotQuery } from "../src/orchestration/Services/ProjectionSnapshotQuery.ts"; +import { ThreadDeletionReactor } from "../src/orchestration/Services/ThreadDeletionReactor.ts"; +import * as ThreadBackgroundLiveness from "../src/orchestration/ThreadBackgroundLiveness.ts"; +import * as ThreadPlanProgress from "../src/orchestration/ThreadPlanProgress.ts"; +import * as ThreadSettlementReactor from "../src/orchestration/ThreadSettlementReactor.ts"; +import { HubThreadMachineStateSqliteLive } from "../src/persistence/Layers/HubThreadMachineState.ts"; +import { OrchestrationCommandReceiptRepositoryLive } from "../src/persistence/Layers/OrchestrationCommandReceipts.ts"; +import { OrchestrationEventStoreLive } from "../src/persistence/Layers/OrchestrationEventStore.ts"; +import { ProjectionCheckpointRepositoryLive } from "../src/persistence/Layers/ProjectionCheckpoints.ts"; +import { ProjectionPendingApprovalRepositoryLive } from "../src/persistence/Layers/ProjectionPendingApprovals.ts"; +import { makeSqlitePersistenceLive } from "../src/persistence/Layers/Sqlite.ts"; +import * as ProviderSessionRuntime from "../src/persistence/ProviderSessionRuntime.ts"; +import { CheckpointTurnDiffStore } from "../src/persistence/Services/HubThreadMachineState.ts"; +import { ProjectionPendingApprovalRepository } from "../src/persistence/Services/ProjectionPendingApprovals.ts"; +import { CodexDriver } from "../src/provider/Drivers/CodexDriver.ts"; +import { + NoOpProviderEventLoggers, + ProviderEventLoggers, +} from "../src/provider/Layers/ProviderEventLoggers.ts"; +import { makeProviderServiceLive } from "../src/provider/Layers/ProviderService.ts"; +import { ProviderSessionDirectoryLive } from "../src/provider/Layers/ProviderSessionDirectory.ts"; +import type { ProviderInstance } from "../src/provider/ProviderDriver.ts"; +import { ProviderAdapterRegistry } from "../src/provider/Services/ProviderAdapterRegistry.ts"; +import { ProviderInstanceRegistry } from "../src/provider/Services/ProviderInstanceRegistry.ts"; +import { makeAdapterRegistryMock } from "../src/provider/testUtils/providerAdapterRegistryMock.ts"; +import { makeProviderRegistryLayer } from "../src/provider/testUtils/providerRegistryMock.ts"; +import * as AgentAwarenessRelay from "../src/relay/AgentAwarenessRelay.ts"; +import * as ReviewService from "../src/review/ReviewService.ts"; +import * as RunnerCheckout from "../src/runner/RunnerCheckout.ts"; +import { RunnerRpcHandlersLive } from "../src/runner/RunnerHandlers.ts"; +import { + makeWithOptions as makeOutbox, + RunnerOutbox, + type RunnerOutboxShape, +} from "../src/runner/RunnerOutbox.ts"; +import { RunnerEventPumpLive } from "../src/runner/RunnerServer.ts"; +import { ServerSettingsService } from "../src/serverSettings.ts"; +import { AnalyticsService } from "../src/telemetry/AnalyticsService.ts"; +import * as TerminalManager from "../src/terminal/Manager.ts"; +import { TextGeneration, type TextGenerationShape } from "../src/textGeneration/TextGeneration.ts"; +import * as GitVcsDriver from "../src/vcs/GitVcsDriver.ts"; +import * as VcsDriverRegistry from "../src/vcs/VcsDriverRegistry.ts"; +import * as VcsProcess from "../src/vcs/VcsProcess.ts"; +import * as VcsProvisioningService from "../src/vcs/VcsProvisioningService.ts"; +import * as VcsStatusBroadcaster from "../src/vcs/VcsStatusBroadcaster.ts"; +import * as WorkspaceEntries from "../src/workspace/WorkspaceEntries.ts"; +import * as WorkspaceFileSystem from "../src/workspace/WorkspaceFileSystem.ts"; +import * as WorkspacePaths from "../src/workspace/WorkspacePaths.ts"; +import { + makeTestProviderAdapterHarness, + type TestProviderAdapterHarness, +} from "./TestProviderAdapter.integration.ts"; + +export const LOOPBACK_PROVIDER = ProviderDriverKind.make("codex"); +export const LOOPBACK_INSTANCE_ID = defaultInstanceIdForDriver(LOOPBACK_PROVIDER); + +const git = (cwd: string, args: ReadonlyArray) => + NodeChildProcess.execFileSync( + "git", + ["-c", "user.email=test@example.com", "-c", "user.name=Test", ...args], + { cwd, stdio: ["ignore", "pipe", "pipe"], encoding: "utf8" }, + ); + +const waitFor = ( + read: Effect.Effect, + predicate: (value: A) => boolean, + description: string, + timeoutMs = 20_000, +): Effect.Effect => + read.pipe( + Effect.filterOrFail(predicate, () => "retry" as const), + Effect.retry({ + schedule: Schedule.spaced("10 millis"), + times: Math.floor(timeoutMs / 10), + while: (error) => error === "retry", + }), + Effect.mapError((error) => + error === "retry" ? new Error(`timed out waiting for ${description}`) : error, + ), + Effect.orDie, + ); + +const instanceRegistryLayer = (adapter: TestProviderAdapterHarness["adapter"]) => { + const instance: ProviderInstance = { + instanceId: LOOPBACK_INSTANCE_ID, + driverKind: LOOPBACK_PROVIDER, + continuationIdentity: { + driverKind: LOOPBACK_PROVIDER, + continuationKey: `${LOOPBACK_PROVIDER}:instance:${LOOPBACK_INSTANCE_ID}`, + }, + displayName: undefined, + enabled: true, + snapshot: { + maintenanceCapabilities: { provider: LOOPBACK_PROVIDER, packageName: null, update: null }, + getSnapshot: Effect.succeed({} as ServerProvider), + refresh: Effect.succeed({} as ServerProvider), + streamChanges: Stream.empty, + }, + adapter, + textGeneration: {} as TextGenerationShape, + }; + return Layer.succeed(ProviderInstanceRegistry, { + getInstance: (instanceId) => + Effect.succeed(instanceId === LOOPBACK_INSTANCE_ID ? instance : undefined), + listInstances: Effect.succeed([instance]), + listUnavailable: Effect.succeed([]), + streamChanges: Stream.empty, + subscribeChanges: Effect.flatMap(PubSub.unbounded(), (pubsub) => + PubSub.subscribe(pubsub), + ), + }); +}; + +export interface LoopbackRunner { + readonly url: string; + readonly adapterHarness: TestProviderAdapterHarness; + readonly outbox: RunnerOutboxShape; +} + +export interface HubRunnerLoopbackHarness { + readonly rootDir: string; + readonly checkoutRoot: string; + readonly checkout: string; + readonly threadId: ThreadId; + readonly runner: () => LoopbackRunner; + /** Stops the runner process; with `freshAdapter` the next boot has lost every session. */ + readonly restartRunner: (options?: { readonly freshAdapter?: boolean }) => Effect.Effect; + readonly engine: () => OrchestrationEngineShape; + readonly snapshotQuery: () => ProjectionSnapshotQuery["Service"]; + readonly checkpointStore: () => CheckpointStore.CheckpointStore["Service"]; + readonly diffStore: () => CheckpointTurnDiffStore["Service"]; + readonly delivery: () => RunnerEventDelivery.RunnerEventDelivery["Service"]; + /** Disposes the hub runtime and starts a new one on the same database. */ + readonly restartHub: Effect.Effect; + readonly waitForThread: ( + predicate: (thread: OrchestrationThread) => boolean, + description: string, + ) => Effect.Effect; + readonly waitForPendingApproval: ( + requestId: string, + predicate: (row: { + readonly status: string; + readonly decision: ProviderApprovalDecision | null; + }) => boolean, + ) => Effect.Effect; + readonly dispose: Effect.Effect; +} + +export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") => + Effect.gen(function* () { + const threadId = ThreadId.make(threadIdValue); + const rootDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-hub-runner-loopback-")); + const checkoutRoot = NodePath.join(rootDir, "t"); + const checkout = threadCheckoutPath(threadId, checkoutRoot); + NodeFS.mkdirSync(checkout, { recursive: true }); + git(checkout, ["init", "--initial-branch=main"]); + NodeFS.writeFileSync(NodePath.join(checkout, "README.md"), "v1\n"); + git(checkout, ["add", "."]); + git(checkout, ["commit", "-m", "Initial"]); + + // ── Runner ─────────────────────────────────────────────────────────── + const runnerStateDir = NodePath.join(rootDir, "runner"); + const runnerConfigLayer = Layer.effect( + ServerConfig, + Effect.gen(function* () { + const base = yield* ServerConfig; + return { + ...base, + cwd: checkout, + serverMode: "runner" as const, + runnerThreadId: threadId, + runnerCheckout: checkout, + }; + }), + ).pipe(Layer.provide(ServerConfig.layerTest(checkout, runnerStateDir))); + + let runnerScope: Scope.Closeable | null = null; + let runner: LoopbackRunner | null = null; + + const startRunner = (adapterHarness: TestProviderAdapterHarness) => + Effect.gen(function* () { + const scope = yield* Scope.make(); + const outbox = yield* makeOutbox({ + databasePath: NodePath.join(runnerStateDir, "runner", "outbox.sqlite"), + }).pipe(Scope.provide(scope)); + const runnerLayer = Layer.mergeAll(RunnerEventPumpLive, RunnerRpcHandlersLive).pipe( + Layer.provideMerge(Layer.succeed(RunnerOutbox, outbox)), + Layer.provideMerge(instanceRegistryLayer(adapterHarness.adapter)), + Layer.provideMerge( + Layer.mergeAll( + RunnerCheckout.layer.pipe(Layer.provide(GitVcsDriver.layer)), + CheckpointStore.layer.pipe(Layer.provide(VcsDriverRegistry.layer)), + WorkspaceFileSystem.layer.pipe( + Layer.provide(WorkspaceEntries.layer.pipe(Layer.provide(WorkspacePaths.layer))), + Layer.provide(WorkspacePaths.layer), + ), + WorkspaceEntries.layer.pipe(Layer.provide(WorkspacePaths.layer)), + Layer.mock(GitWorkflowService.GitWorkflowService)({}), + Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({ + streamStatus: () => Stream.never, + refreshLocalStatus: () => + Effect.succeed({ + isRepo: true, + hasPrimaryRemote: false, + isDefaultRef: true, + refName: "main", + hasWorkingTreeChanges: false, + workingTree: { files: [], insertions: 0, deletions: 0 }, + }), + }), + Layer.mock(VcsProvisioningService.VcsProvisioningService)({}), + Layer.mock(ReviewService.ReviewService)({}), + Layer.mock(TerminalManager.TerminalManager)({ + subscribe: () => Effect.succeed(() => undefined), + subscribeMetadata: (listener) => + listener({ type: "snapshot", terminals: [] }).pipe(Effect.as(() => undefined)), + }), + ServerSettingsService.layerTest(), + ), + ), + Layer.provideMerge(VcsDriverRegistry.layer), + Layer.provideMerge(runnerConfigLayer), + Layer.provideMerge(VcsProcess.layer), + Layer.provideMerge(NodeServices.layer), + ); + const served = yield* serveRunner(runnerLayer).pipe(Scope.provide(scope)); + runnerScope = scope; + runner = { url: served.url, adapterHarness, outbox }; + return runner; + }); + + const firstRunner = yield* startRunner( + yield* makeTestProviderAdapterHarness({ provider: LOOPBACK_PROVIDER }), + ); + const directory = yield* makeFakeMachineDirectory({ + initial: [[threadId, { state: "running", runnerUrl: firstRunner.url, bootId: "boot" }]], + }); + + const restartRunner: HubRunnerLoopbackHarness["restartRunner"] = (options) => + Effect.gen(function* () { + const previous = runner!; + if (runnerScope) yield* Scope.close(runnerScope, Exit.void); + const next = yield* startRunner( + options?.freshAdapter + ? yield* makeTestProviderAdapterHarness({ provider: LOOPBACK_PROVIDER }) + : previous.adapterHarness, + ); + yield* directory.set(threadId, { state: "running", runnerUrl: next.url }); + }).pipe(Effect.orDie); + + // ── Hub ────────────────────────────────────────────────────────────── + const hubBaseDir = NodePath.join(rootDir, "hub"); + const hubConfigLayer = Layer.effect( + ServerConfig, + Effect.gen(function* () { + const base = yield* ServerConfig; + return { ...base, serverMode: "hub" as const, hub: { checkoutRoot } }; + }), + ).pipe(Layer.provide(ServerConfig.layerTest(rootDir, hubBaseDir))); + + const makeHubLayer = () => { + const persistence = makeSqlitePersistenceLive(NodePath.join(hubBaseDir, "hub.sqlite")); + const infrastructure = Layer.mergeAll(RunnerEventDelivery.layer, hubVcsStatusCacheLayer).pipe( + Layer.provideMerge( + RemoteSessionRegistry.layer.pipe(Layer.provide(ProviderSessionRuntime.layer)), + ), + Layer.provideMerge( + Layer.effect( + RunnerConnectionPool, + makePool({ checkoutRoot, wakePollInterval: "5 millis", idleCheckInterval: "1 hour" }), + ), + ), + Layer.provideMerge(Layer.succeed(MachineDirectory, directory.directory)), + Layer.provideMerge(HubThreadMachineStateSqliteLive), + ); + const remoteAdapterRegistry = Layer.effect( + ProviderAdapterRegistry, + Effect.gen(function* () { + const instance = yield* makeRemoteProviderDriver(CodexDriver).create({ + instanceId: LOOPBACK_INSTANCE_ID, + displayName: undefined, + environment: [], + enabled: true, + config: CodexDriver.defaultConfig(), + }); + return makeAdapterRegistryMock({ [LOOPBACK_PROVIDER]: instance.adapter }); + }), + ); + const providerLayer = makeProviderServiceLive().pipe( + Layer.provide( + ProviderSessionDirectoryLive.pipe(Layer.provide(ProviderSessionRuntime.layer)), + ), + Layer.provide(remoteAdapterRegistry), + Layer.provide(AnalyticsService.layerTest), + Layer.provide(Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers)), + ); + const projectionSnapshotQueryLayer = OrchestrationProjectionSnapshotQueryLive; + const runtimeServices = Layer.mergeAll( + projectionSnapshotQueryLayer, + OrchestrationEngineLive.pipe( + Layer.provide(OrchestrationProjectionPipelineLive), + Layer.provide(OrchestrationEventStoreLive), + Layer.provide(OrchestrationCommandReceiptRepositoryLive), + Layer.provide(projectionSnapshotQueryLayer), + ), + ProjectionCheckpointRepositoryLive, + ProjectionPendingApprovalRepositoryLive, + HubLayers.hubCheckpointStoreLayer, + providerLayer, + RuntimeReceiptBusTest, + ).pipe( + Layer.provideMerge(ThreadBackgroundLiveness.layer), + Layer.provideMerge(ThreadPlanProgress.layer), + ); + const checkoutServices = Layer.mergeAll( + HubLayers.hubVcsStatusBroadcasterLayer, + HubLayers.hubWorkspaceEntriesLayer, + HubLayers.hubGitWorkflowServiceLayer, + Layer.succeed(TextGeneration, { + generateBranchName: () => Effect.succeed({ branch: "update" }), + generateThreadTitle: () => Effect.succeed({ title: "New thread" }), + } as unknown as TextGenerationShape), + HubLayers.hubReactorHooksLayer, + ); + const reactors = OrchestrationReactorLive.pipe( + Layer.provideMerge(ProviderRuntimeIngestionLive), + Layer.provideMerge(ProviderCommandReactorLive), + Layer.provideMerge(CheckpointReactorLive), + Layer.provideMerge( + Layer.succeed(ThreadDeletionReactor, { + start: () => Effect.void, + drainThrough: () => Effect.void, + }), + ), + Layer.provideMerge( + Layer.succeed(ThreadSettlementReactor.ThreadSettlementReactor, { + start: () => Effect.void, + drain: Effect.void, + }), + ), + Layer.provideMerge( + Layer.succeed(AgentAwarenessRelay.AgentAwarenessRelay, { + publishThread: () => Effect.void, + start: () => Effect.void, + }), + ), + ); + return HubLayers.hubRuntimeBackgroundLayer.pipe( + Layer.provideMerge(reactors), + Layer.provideMerge(checkoutServices), + Layer.provideMerge(runtimeServices), + Layer.provideMerge(makeProviderRegistryLayer()), + Layer.provideMerge(HubLayers.hubRepositoryIdentityResolverLayer), + Layer.provideMerge(infrastructure), + Layer.provideMerge(persistence), + Layer.provideMerge(ServerSettingsService.layerTest()), + Layer.provideMerge(hubConfigLayer), + Layer.provideMerge(NodeServices.layer), + ); + }; + + interface HubInstance { + readonly runtime: ManagedRuntime.ManagedRuntime; + readonly scope: Scope.Closeable; + readonly engine: OrchestrationEngineShape; + readonly snapshotQuery: ProjectionSnapshotQuery["Service"]; + readonly checkpointStore: CheckpointStore.CheckpointStore["Service"]; + readonly diffStore: CheckpointTurnDiffStore["Service"]; + readonly delivery: RunnerEventDelivery.RunnerEventDelivery["Service"]; + readonly approvals: ProjectionPendingApprovalRepository["Service"]; + } + + const startHub = Effect.gen(function* () { + const runtime = ManagedRuntime.make(makeHubLayer()); + const services = yield* Effect.promise(() => + runtime.runPromise( + Effect.all({ + engine: Effect.service(OrchestrationEngineService), + snapshotQuery: Effect.service(ProjectionSnapshotQuery), + checkpointStore: Effect.service(CheckpointStore.CheckpointStore), + diffStore: Effect.service(CheckpointTurnDiffStore), + delivery: Effect.service(RunnerEventDelivery.RunnerEventDelivery), + approvals: Effect.service(ProjectionPendingApprovalRepository), + reactor: Effect.service(OrchestrationReactor), + }), + ), + ); + const { reactor, ...rest } = services; + const instance = { + runtime, + scope: yield* Scope.make("sequential"), + ...rest, + } satisfies HubInstance; + yield* Effect.promise(() => + runtime.runPromise(reactor.start().pipe(Scope.provide(instance.scope))), + ); + return instance; + }); + + const stopHub = (instance: HubInstance) => + Effect.gen(function* () { + yield* Scope.close(instance.scope, Exit.void); + yield* Effect.promise(() => instance.runtime.dispose()); + }); + + let hub = yield* startHub; + + const waitForThread: HubRunnerLoopbackHarness["waitForThread"] = (predicate, description) => + waitFor( + Effect.suspend(() => hub.snapshotQuery.getThreadDetailById(threadId)).pipe( + Effect.map(Option.getOrNull), + ), + (thread) => thread !== null && predicate(thread), + description, + ) as Effect.Effect; + + return { + rootDir, + checkoutRoot, + checkout, + threadId, + runner: () => runner!, + restartRunner, + engine: () => hub.engine, + snapshotQuery: () => hub.snapshotQuery, + checkpointStore: () => hub.checkpointStore, + diffStore: () => hub.diffStore, + delivery: () => hub.delivery, + restartHub: Effect.gen(function* () { + yield* stopHub(hub); + hub = yield* startHub; + }), + waitForThread, + waitForPendingApproval: (requestId, predicate) => + waitFor( + Effect.suspend(() => + hub.approvals.getByRequestId({ requestId: ApprovalRequestId.make(requestId) }), + ).pipe(Effect.map(Option.getOrNull)), + (row) => row !== null && predicate(row), + `pending approval ${requestId}`, + ), + dispose: Effect.gen(function* () { + yield* stopHub(hub); + if (runnerScope) yield* Scope.close(runnerScope, Exit.void); + NodeFS.rmSync(rootDir, { recursive: true, force: true }); + }), + } satisfies HubRunnerLoopbackHarness; + }); diff --git a/apps/server/integration/hubRunnerLoopback.integration.test.ts b/apps/server/integration/hubRunnerLoopback.integration.test.ts new file mode 100644 index 000000000000..602e1239f4d1 --- /dev/null +++ b/apps/server/integration/hubRunnerLoopback.integration.test.ts @@ -0,0 +1,331 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * The remote provider driver and the orchestration reactors on a hub, against + * a real runner over the runner protocol (loopback WebSocket, one process). + */ +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; + +import { assert, it } from "@effect/vitest"; +import { + CommandId, + DEFAULT_PROVIDER_INTERACTION_MODE, + EventId, + MessageId, + ProjectId, + type ProviderRuntimeEvent, + TurnId, +} from "@t3tools/contracts"; +import { projectVirtualRoot } from "@t3tools/contracts/runner"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import { checkpointRefForThreadTurn } from "../src/checkpointing/Utils.ts"; +import { + type HubRunnerLoopbackHarness, + LOOPBACK_INSTANCE_ID, + LOOPBACK_PROVIDER, + makeHubRunnerLoopbackHarness, +} from "./HubRunnerLoopbackHarness.integration.ts"; +import type { TestTurnResponse } from "./TestProviderAdapter.integration.ts"; + +const PROJECT_ID = ProjectId.make("project-loopback"); +const MODEL = { instanceId: LOOPBACK_INSTANCE_ID, model: "gpt-5-codex" }; +const at = (second: number) => `2026-09-25T00:00:${String(second).padStart(2, "0")}.000Z`; + +const withHarness = (use: (harness: HubRunnerLoopbackHarness) => Effect.Effect) => + Effect.acquireUseRelease(makeHubRunnerLoopbackHarness(), use, (harness) => harness.dispose); + +const seed = (harness: HubRunnerLoopbackHarness) => + Effect.gen(function* () { + const engine = harness.engine(); + yield* engine.dispatch({ + type: "project.create", + commandId: CommandId.make("cmd-project-create"), + projectId: PROJECT_ID, + title: "Loopback", + workspaceRoot: projectVirtualRoot(PROJECT_ID), + defaultModelSelection: MODEL, + createdAt: at(0), + }); + yield* engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-thread-create"), + threadId: harness.threadId, + projectId: PROJECT_ID, + title: "Loopback thread", + modelSelection: MODEL, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: harness.checkout, + createdAt: at(1), + }); + }); + +const startTurn = (harness: HubRunnerLoopbackHarness, n: number, text: string) => + harness.engine().dispatch({ + type: "thread.turn.start", + commandId: CommandId.make(`cmd-turn-start-${n}`), + threadId: harness.threadId, + message: { messageId: MessageId.make(`msg-user-${n}`), role: "user", text, attachments: [] }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: at(10 + n), + }); + +const fixture = (harness: HubRunnerLoopbackHarness, id: string) => ({ + eventId: EventId.make(id), + provider: LOOPBACK_PROVIDER, + createdAt: at(30), + threadId: harness.threadId, + turnId: "turn-fixture", +}); + +/** A runtime event appended straight to the runner outbox, as a provider would emit it. */ +const outboxEvent = ( + harness: HubRunnerLoopbackHarness, + id: string, + turnId: string, + body: + | { readonly type: "turn.started"; readonly payload: Record } + | { + readonly type: "content.delta"; + readonly payload: { readonly streamKind: "assistant_text"; readonly delta: string }; + } + | { readonly type: "turn.completed"; readonly payload: { readonly state: "completed" } }, +): ProviderRuntimeEvent => + ({ + eventId: EventId.make(id), + provider: LOOPBACK_PROVIDER, + providerInstanceId: LOOPBACK_INSTANCE_ID, + threadId: harness.threadId, + turnId: TurnId.make(turnId), + createdAt: at(40), + ...body, + }) as ProviderRuntimeEvent; + +const runFirstTurn = (harness: HubRunnerLoopbackHarness) => + Effect.gen(function* () { + const response: TestTurnResponse = { + events: [ + { type: "turn.started", ...fixture(harness, "evt-1") }, + { type: "message.delta", ...fixture(harness, "evt-2"), delta: "Hello from the runner.\n" }, + { type: "turn.completed", ...fixture(harness, "evt-3"), status: "completed" }, + ], + mutateWorkspace: ({ cwd }) => + Effect.sync(() => + NodeFS.writeFileSync(NodePath.join(cwd, "notes.txt"), "edited remotely\n"), + ), + }; + yield* harness.runner().adapterHarness.queueTurnResponseForNextSession(response); + // The scripted provider edits the checkout synchronously inside sendTurn, so + // capture the pre-turn baseline up front instead of racing the reactor. + yield* harness.checkpointStore().captureCheckpoint({ + cwd: harness.checkout, + checkpointRef: checkpointRefForThreadTurn(harness.threadId, 0), + }); + yield* startTurn(harness, 1, "Edit notes.txt"); + return yield* harness.waitForThread( + (thread) => + thread.latestTurn?.state === "completed" && + thread.checkpoints.length === 1 && + thread.messages.some((message) => message.role === "assistant" && !message.streaming), + "first turn settled with a checkpoint", + ); + }); + +it.live( + "runs a turn on the runner, projects it on the hub, and serves its diff without the machine", + () => + withHarness((harness) => + Effect.gen(function* () { + yield* seed(harness); + const thread = yield* runFirstTurn(harness); + + assert.deepEqual( + thread.messages + .filter((message) => message.role === "assistant") + .map((message) => message.text), + ["Hello from the runner.\n"], + ); + assert.deepEqual( + thread.checkpoints[0]?.files.map((file) => file.path), + ["notes.txt"], + ); + assert.equal(harness.runner().adapterHarness.getStartCount(), 1); + + // The capture stored the patch on the hub; stop the runner and read it back. + const diffs = harness.diffStore(); + + yield* harness + .waitForThread(() => true, "noop") + .pipe( + Effect.andThen( + Effect.repeat( + diffs.get({ + threadId: harness.threadId, + fromTurnCount: 0, + toTurnCount: 1, + ignoreWhitespace: true, + }), + { until: Option.isSome }, + ), + ), + ); + yield* harness.restartRunner().pipe(Effect.andThen(Effect.void)); + const patch = yield* harness.checkpointStore().diffCheckpoints({ + cwd: harness.checkout, + fromCheckpointRef: checkpointRefForThreadTurn(harness.threadId, 0), + toCheckpointRef: checkpointRefForThreadTurn(harness.threadId, 1), + ignoreWhitespace: true, + }); + assert.include(patch, "notes.txt"); + assert.include(patch, "+edited remotely"); + }), + ), +); + +it.live("routes an approval from the runner to the hub and the decision back", () => + withHarness((harness) => + Effect.gen(function* () { + yield* seed(harness); + yield* harness.runner().adapterHarness.queueTurnResponseForNextSession({ + events: [ + { type: "turn.started", ...fixture(harness, "evt-a1") }, + { + type: "approval.requested", + ...fixture(harness, "evt-a2"), + requestId: "req-loopback-1", + requestKind: "command", + detail: "Run tests", + }, + { type: "turn.completed", ...fixture(harness, "evt-a3"), status: "completed" }, + ], + }); + yield* startTurn(harness, 1, "Needs approval"); + yield* harness.waitForPendingApproval("req-loopback-1", (row) => row.status === "pending"); + yield* harness.engine().dispatch({ + type: "thread.approval.respond", + commandId: CommandId.make("cmd-approval-respond"), + threadId: harness.threadId, + requestId: "req-loopback-1" as never, + decision: "accept", + createdAt: at(20), + }); + yield* harness.waitForPendingApproval( + "req-loopback-1", + (row) => row.status === "resolved" && row.decision === "accept", + ); + const responses = yield* Effect.repeat( + Effect.sync(() => harness.runner().adapterHarness.getApprovalResponses(harness.threadId)), + { until: (value) => value.length === 1 }, + ); + assert.equal(responses[0]?.decision, "accept"); + }), + ), +); + +it.live("settles a running turn when the runner restarts without its session, then resumes", () => + withHarness((harness) => + Effect.gen(function* () { + yield* seed(harness); + yield* runFirstTurn(harness); + + // The provider starts another turn, then the machine restarts and loses it. + yield* harness + .runner() + .outbox.append( + outboxEvent(harness, "evt-open-1", "turn-open", { type: "turn.started", payload: {} }), + ); + yield* harness.waitForThread( + (thread) => thread.session?.activeTurnId === "turn-open", + "the provider-started turn is active", + ); + yield* harness.restartRunner({ freshAdapter: true }); + const settled = yield* harness.waitForThread( + (thread) => thread.session?.activeTurnId === null && thread.session?.status !== "running", + "the interrupted turn settled after the runner restart", + ); + assert.notEqual(settled.session?.status, "running"); + + // The next turn resumes the session from the persisted resume cursor. + yield* harness.runner().adapterHarness.queueTurnResponseForNextSession({ + events: [ + { type: "turn.started", ...fixture(harness, "evt-r1") }, + { type: "message.delta", ...fixture(harness, "evt-r2"), delta: "Back again.\n" }, + ], + }); + yield* startTurn(harness, 2, "Continue"); + // The scripted provider numbers turns per session, so the resumed session's + // first turn reuses the id `turn-1`; assert on the text, not the turn id. + const resumed = yield* harness.waitForThread( + (thread) => + thread.latestTurn?.state === "completed" && + thread.messages.some( + (message) => message.role === "assistant" && message.text.includes("Back again."), + ), + "resumed turn", + ); + assert.equal(harness.runner().adapterHarness.getStartCount(), 1); + assert.equal(resumed.session?.status, "ready"); + }), + ), +); + +it.live("replays an open turn after a hub restart without duplicating anything", () => + withHarness((harness) => + Effect.gen(function* () { + yield* seed(harness); + yield* runFirstTurn(harness); + const outbox = () => harness.runner().outbox; + + yield* outbox().append( + outboxEvent(harness, "evt-h1", "turn-h", { type: "turn.started", payload: {} }), + ); + yield* outbox().append( + outboxEvent(harness, "evt-h2", "turn-h", { + type: "content.delta", + payload: { streamKind: "assistant_text", delta: "part A, " }, + }), + ); + yield* harness.waitForThread( + (thread) => thread.session?.activeTurnId === "turn-h", + "open turn delivered", + ); + yield* Effect.repeat(harness.delivery().state, { + until: (state) => + state.get(harness.threadId)?.delivered === (state.get(harness.threadId)?.safe ?? 0) + 2, + }); + + yield* harness.restartHub; + // The provider keeps going while the hub is down. + yield* outbox().append( + outboxEvent(harness, "evt-h3", "turn-h", { + type: "content.delta", + payload: { streamKind: "assistant_text", delta: "part B" }, + }), + ); + yield* outbox().append( + outboxEvent(harness, "evt-h4", "turn-h", { + type: "turn.completed", + payload: { state: "completed" }, + }), + ); + + const thread = yield* harness.waitForThread( + (value) => + value.session?.activeTurnId === null && + value.messages.some( + (message) => message.role === "assistant" && message.text === "part A, part B", + ), + "replayed turn completed with its full text", + ); + const assistantTexts = thread.messages + .filter((message) => message.role === "assistant") + .map((message) => message.text); + assert.deepEqual(assistantTexts, ["Hello from the runner.\n", "part A, part B"]); + assert.equal(thread.checkpoints.length, 1); + }), + ), +); diff --git a/apps/server/src/hub/HubCheckoutServices.test.ts b/apps/server/src/hub/HubCheckoutServices.test.ts new file mode 100644 index 000000000000..23cbf34082c9 --- /dev/null +++ b/apps/server/src/hub/HubCheckoutServices.test.ts @@ -0,0 +1,310 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { it } from "@effect/vitest"; +import { ThreadId, type VcsStatusStreamEvent } from "@t3tools/contracts"; +import { threadCheckoutPath } from "@t3tools/contracts/runner"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; +import * as Stream from "effect/Stream"; +import { describe, expect } from "vite-plus/test"; + +import { CheckpointStore } from "../checkpointing/CheckpointStore.ts"; +import { checkpointRefForThreadTurn } from "../checkpointing/Utils.ts"; +import { HubThreadMachineStateSqliteLive } from "../persistence/Layers/HubThreadMachineState.ts"; +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; +import { + CheckpointTurnDiffStore, + ThreadVcsStatusStore, +} from "../persistence/Services/HubThreadMachineState.ts"; +import * as TerminalManager from "../terminal/Manager.ts"; +import { VcsStatusBroadcaster } from "../vcs/VcsStatusBroadcaster.ts"; +import { WorkspaceEntries } from "../workspace/WorkspaceEntries.ts"; +import { WorkspaceFileSystem } from "../workspace/WorkspaceFileSystem.ts"; +import { hubTerminalManagerLayer } from "./HubTerminals.ts"; +import { hubVcsStatusBroadcasterLayer, hubVcsStatusCacheLayer } from "./HubVcs.ts"; +import { + hubCheckpointStoreLayer, + hubWorkspaceEntriesLayer, + hubWorkspaceFileSystemLayer, +} from "./HubWorkspace.ts"; +import { + type FakeMachine, + MachineDirectory, + makeFakeMachineDirectory, +} from "./MachineDirectory.ts"; +import { make as makePool, RunnerConnectionPool } from "./RunnerConnectionPool.ts"; +import { fakeRunner, fakeRunnerHello, serveRunner } from "./testUtils/runnerServer.ts"; + +type RunnerOverrides = Parameters[0]; + +const threadId = ThreadId.make("thread-services"); +const cwd = threadCheckoutPath(threadId); +const ref = (turn: number) => checkpointRefForThreadTurn(threadId, turn); + +const local = { + isRepo: true, + hasPrimaryRemote: true, + isDefaultRef: false, + refName: "t3/work", + hasWorkingTreeChanges: false, + workingTree: { files: [], insertions: 0, deletions: 0 }, +}; + +/** + * Hub checkout services over a fake directory and a fake runner. The machine + * wakes to the runner; `calls` records every runner RPC by name. + */ +const setup = (machine: Omit, overrides: RunnerOverrides = {}) => + Effect.gen(function* () { + const calls: Array = []; + const record = + , R>(name: string, f: (...args: A) => R) => + (...args: A) => { + calls.push(name); + return f(...args); + }; + const wrapped = Object.fromEntries( + Object.entries(overrides).map(([name, handler]) => [ + name, + record(name, handler as (...args: ReadonlyArray) => unknown), + ]), + ) as RunnerOverrides; + const runner = yield* serveRunner( + fakeRunner({ + "runner.hello": (input) => Effect.succeed(fakeRunnerHello({ threadId: input.threadId })), + ...wrapped, + }), + ); + const fake = yield* makeFakeMachineDirectory({ + initial: [ + [threadId, machine.state === "running" ? { ...machine, runnerUrl: runner.url } : machine], + ], + onWake: () => ({ state: "running", runnerUrl: runner.url }), + }); + const pool = yield* makePool({ + wakePollInterval: "5 millis", + idleCheckInterval: "1 hour", + }).pipe(Effect.provideService(MachineDirectory, fake.directory)); + const services = yield* Layer.build( + Layer.mergeAll( + hubCheckpointStoreLayer, + hubWorkspaceEntriesLayer, + hubWorkspaceFileSystemLayer, + hubTerminalManagerLayer, + hubVcsStatusBroadcasterLayer.pipe(Layer.provideMerge(hubVcsStatusCacheLayer)), + ).pipe(Layer.provide(Layer.succeed(RunnerConnectionPool, pool))), + ); + const directoryCalls = fake.calls.pipe( + Effect.map((entries) => + entries.map((call) => (call.wake ? `${call.method}+wake` : call.method)), + ), + ); + return { services, pool, calls, directoryCalls }; + }); + +const TestLayer = HubThreadMachineStateSqliteLive.pipe( + Layer.provideMerge(SqlitePersistenceMemory), + Layer.provideMerge(NodeServices.layer), +); + +describe("hub checkout services", () => { + it.live("serves captured diffs without waking and fetches a missing diff once", () => + Effect.scoped( + Effect.gen(function* () { + const { services, calls, directoryCalls } = yield* setup( + { state: "paused" }, + { "runner.checkpoint.diff": () => Effect.succeed("diff --git a/x b/x\n") }, + ); + const store = yield* CheckpointTurnDiffStore; + yield* store.put({ + threadId, + fromTurnCount: 0, + toTurnCount: 1, + ignoreWhitespace: true, + diff: "cached patch", + createdAt: "t", + }); + const checkpointStore = Context.get(services, CheckpointStore); + const diff = (from: number, to: number) => + checkpointStore.diffCheckpoints({ + cwd, + fromCheckpointRef: ref(from), + toCheckpointRef: ref(to), + ignoreWhitespace: true, + }); + + expect(yield* diff(0, 1)).toBe("cached patch"); + expect(yield* directoryCalls).toEqual([]); + + expect(yield* diff(1, 2)).toBe("diff --git a/x b/x\n"); + expect(yield* diff(1, 2)).toBe("diff --git a/x b/x\n"); + expect(calls).toEqual(["runner.checkpoint.diff"]); + expect(yield* directoryCalls).toEqual(["ensure+wake"]); + }), + ).pipe(Effect.provide(TestLayer)), + ); + + it.live("recapturing a checkpoint invalidates its diffs and precomputes new ones", () => + Effect.scoped( + Effect.gen(function* () { + const { services, calls } = yield* setup( + { state: "running" }, + { + "runner.checkpoint.capture": () => Effect.void, + "runner.checkpoint.diff": (input) => + Effect.succeed( + `new ${input.fromCheckpointRef.slice(-1)}-${input.toCheckpointRef.slice(-1)}`, + ), + }, + ); + const store = yield* CheckpointTurnDiffStore; + yield* store.put({ + threadId, + fromTurnCount: 1, + toTurnCount: 2, + ignoreWhitespace: true, + diff: "stale", + createdAt: "t", + }); + const checkpointStore = Context.get(services, CheckpointStore); + yield* checkpointStore.captureCheckpoint({ cwd, checkpointRef: ref(2) }); + const key = { threadId, fromTurnCount: 1, toTurnCount: 2, ignoreWhitespace: true }; + const refreshed = yield* Effect.repeat(store.get(key), { + until: (value) => Option.isSome(value) && value.value !== "stale", + }); + expect(Option.getOrThrow(refreshed)).toBe("new 1-2"); + yield* Effect.repeat( + Effect.sync(() => calls.length), + { until: (count) => count >= 5 }, + ); + expect(calls.filter((name) => name === "runner.checkpoint.diff")).toHaveLength(4); + }), + ).pipe(Effect.provide(TestLayer)), + ); + + it.live("streams cached git status for a sleeping machine without waking it", () => + Effect.scoped( + Effect.gen(function* () { + const { services, directoryCalls } = yield* setup({ state: "paused" }); + yield* (yield* ThreadVcsStatusStore).put({ threadId, local, remote: null, updatedAt: "t" }); + const broadcaster = Context.get(services, VcsStatusBroadcaster); + const [first] = yield* broadcaster + .streamStatus({ cwd }) + .pipe(Stream.take(1), Stream.runCollect); + expect(first).toEqual({ _tag: "snapshot", local, remote: null }); + expect((yield* broadcaster.refreshStatus(cwd)).refName).toBe("t3/work"); + expect(yield* directoryCalls).toEqual(["status"]); + }), + ).pipe(Effect.provide(TestLayer)), + ); + + it.live("forwards status a connected runner pushes and persists it", () => + Effect.scoped( + Effect.gen(function* () { + const pushed = yield* PubSub.unbounded(); + const { services, pool } = yield* setup( + { state: "running" }, + { "runner.vcs.streamStatus": () => Stream.fromPubSub(pushed) }, + ); + const broadcaster = Context.get(services, VcsStatusBroadcaster); + const received = yield* broadcaster + .streamStatus({ cwd }) + .pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped); + yield* Effect.yieldNow; + yield* pool.use(threadId, { wake: false, operation: "test.connect" }, () => Effect.void); + yield* Effect.repeat(PubSub.publish(pushed, { _tag: "snapshot", local, remote: null }), { + until: () => received.pollUnsafe() !== undefined, + }); + expect(Array.from(yield* Fiber.join(received))).toEqual([ + { _tag: "snapshot", local, remote: null }, + ]); + const persisted = yield* (yield* ThreadVcsStatusStore).list(); + expect(persisted.map((row) => row.local.refName)).toEqual(["t3/work"]); + }), + ).pipe(Effect.provide(TestLayer)), + ); + + it.live("reads never wake a machine; writes do", () => + Effect.scoped( + Effect.gen(function* () { + const { services, calls, directoryCalls } = yield* setup( + { state: "paused" }, + { + "runner.workspace.writeFile": (input) => + Effect.succeed({ relativePath: input.relativePath }), + }, + ); + const files = Context.get(services, WorkspaceFileSystem); + const readError = yield* files + .readFile({ cwd, relativePath: "README.md" }) + .pipe(Effect.flip); + expect(readError).toMatchObject({ + _tag: "WorkspaceFileSystemOperationError", + cause: { _tag: "ThreadMachineUnavailableError", reason: "asleep" }, + }); + expect(yield* directoryCalls).toEqual(["status"]); + + yield* files.writeFile({ cwd, relativePath: "notes.txt", contents: "hi" }); + expect(calls).toEqual(["runner.workspace.writeFile"]); + expect(yield* directoryCalls).toEqual(["status", "ensure+wake"]); + }), + ).pipe(Effect.provide(TestLayer)), + ); + + it.live("never routes a path that is not a thread checkout", () => + Effect.scoped( + Effect.gen(function* () { + const { services, directoryCalls } = yield* setup({ state: "running" }); + const entries = Context.get(services, WorkspaceEntries); + const error = yield* entries + .search({ cwd: "/home/user/project", query: "readme", limit: 10 }) + .pipe(Effect.flip); + expect(error).toMatchObject({ + cause: { _tag: "ThreadMachineUnavailableError", reason: "not-a-thread-checkout" }, + }); + expect(yield* directoryCalls).toEqual([]); + }), + ).pipe(Effect.provide(TestLayer)), + ); + + it.live("terminal input needs a running machine; opening a terminal wakes it", () => + Effect.scoped( + Effect.gen(function* () { + const { services, calls, directoryCalls } = yield* setup( + { state: "paused" }, + { + "runner.terminal.open": (input) => + Effect.succeed({ + threadId: input.threadId, + terminalId: input.terminalId, + cwd: input.cwd, + worktreePath: null, + status: "running", + pid: 42, + history: "", + exitCode: null, + exitSignal: null, + label: "bash", + updatedAt: "t", + }), + }, + ); + const terminals = Context.get(services, TerminalManager.TerminalManager); + const writeError = yield* terminals + .write({ threadId, terminalId: "t1", data: "ls\r" }) + .pipe(Effect.flip); + expect(writeError._tag).toBe("TerminalNotRunningError"); + yield* terminals.close({ threadId, terminalId: "t1" }); + expect(yield* directoryCalls).toEqual(["status"]); + + const opened = yield* terminals.open({ threadId, terminalId: "t1", cwd }); + expect(opened.pid).toBe(42); + expect(calls).toEqual(["runner.terminal.open"]); + expect(yield* directoryCalls).toEqual(["status", "ensure+wake"]); + }), + ).pipe(Effect.provide(TestLayer)), + ); +}); diff --git a/apps/server/src/hub/RunnerConnectionPool.test.ts b/apps/server/src/hub/RunnerConnectionPool.test.ts new file mode 100644 index 000000000000..f845f41badd9 --- /dev/null +++ b/apps/server/src/hub/RunnerConnectionPool.test.ts @@ -0,0 +1,254 @@ +import { it } from "@effect/vitest"; +import { ThreadId } from "@t3tools/contracts"; +import { RunnerThreadMismatchError, type ThreadMachineStatus } from "@t3tools/contracts/runner"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; +import { describe, expect } from "vite-plus/test"; + +import { + type FakeMachine, + MachineDirectory, + makeFakeMachineDirectory, +} from "./MachineDirectory.ts"; +import { make as makePool, type RunnerConnectionPoolOptions } from "./RunnerConnectionPool.ts"; +import { fakeRunner, fakeRunnerHello, serveRunner } from "./testUtils/runnerServer.ts"; + +const threadId = ThreadId.make("thread-pool"); + +/** A fake runner for `threadId` that answers the handshake and counts connections. */ +const startRunner = (options?: { readonly bootId?: string; readonly serves?: ThreadId }) => + Effect.gen(function* () { + let hellos = 0; + const served = yield* serveRunner( + fakeRunner({ + "runner.hello": (input) => + Effect.suspend(() => { + hellos += 1; + const serves = options?.serves ?? input.threadId; + return serves === input.threadId + ? Effect.succeed( + fakeRunnerHello({ threadId: serves, bootId: options?.bootId ?? "boot-1" }), + ) + : Effect.fail( + new RunnerThreadMismatchError({ + requestedThreadId: input.threadId, + runnerThreadId: serves, + }), + ); + }), + }), + ); + return { url: served.url, hellos: () => hellos }; + }); + +const setup = ( + machine: FakeMachine | undefined, + options: RunnerConnectionPoolOptions & { + readonly onWake?: (current: FakeMachine | undefined) => FakeMachine; + } = {}, +) => + Effect.gen(function* () { + const fake = yield* makeFakeMachineDirectory({ + ...(machine ? { initial: [[threadId, machine]] } : {}), + ...(options.onWake ? { onWake: (_threadId, current) => options.onWake!(current) } : {}), + }); + const pool = yield* makePool({ + wakePollInterval: "5 millis", + idleCheckInterval: "1 hour", + ...options, + }).pipe(Effect.provideService(MachineDirectory, fake.directory)); + return { pool, fake }; + }); + +const callsOf = (fake: { + readonly calls: Effect.Effect< + ReadonlyArray<{ readonly method: string; readonly wake?: boolean }> + >; +}) => + fake.calls.pipe( + Effect.map((calls) => calls.map((call) => (call.wake ? `${call.method}+wake` : call.method))), + ); + +describe("RunnerConnectionPool wake semantics", () => { + it.live("never wakes a sleeping machine for a read and reports it asleep", () => + Effect.scoped( + Effect.gen(function* () { + const { pool, fake } = yield* setup({ state: "paused", bootId: "boot-1" }); + const error = yield* pool + .use(threadId, { wake: false, operation: "vcs.status" }, () => Effect.void) + .pipe(Effect.flip); + expect(error).toMatchObject({ + _tag: "ThreadMachineUnavailableError", + reason: "asleep", + state: "paused", + operation: "vcs.status", + }); + expect(yield* callsOf(fake)).toEqual(["status"]); + }), + ), + ); + + it.live("wakes a sleeping machine for an action, connects, and reuses the connection", () => + Effect.scoped( + Effect.gen(function* () { + const runner = yield* startRunner(); + const { pool, fake } = yield* setup( + { state: "saved" }, + { onWake: () => ({ state: "running", runnerUrl: runner.url, bootId: "boot-1" }) }, + ); + const first = yield* pool.use( + threadId, + { wake: true, operation: "provider.sendTurn" }, + (connection) => Effect.succeed(connection.hello.bootId), + ); + const second = yield* pool.use( + threadId, + { wake: false, operation: "vcs.status" }, + (connection) => Effect.succeed(connection.hello.threadId), + ); + expect(first).toBe("boot-1"); + expect(second).toBe(threadId); + expect(runner.hellos()).toBe(1); + expect(yield* callsOf(fake)).toEqual(["ensure+wake"]); + expect(Option.isSome(yield* pool.current(threadId))).toBe(true); + }), + ), + ); + + it.live("waits through starting states, reporting progress, until the runner is up", () => + Effect.scoped( + Effect.gen(function* () { + const runner = yield* startRunner(); + const { pool, fake } = yield* setup( + { state: "paused" }, + { onWake: () => ({ state: "starting" }) }, + ); + const progress: Array = []; + let polls = 0; + const fiber = yield* pool + .use( + threadId, + { + wake: true, + operation: "terminal.open", + onWakeProgress: (status) => + Effect.suspend(() => { + progress.push(status.state); + polls += 1; + return polls === 3 + ? fake.set(threadId, { state: "running", runnerUrl: runner.url }) + : Effect.void; + }), + }, + () => Effect.succeed("connected"), + ) + .pipe(Effect.forkChild); + expect(yield* Fiber.join(fiber)).toBe("connected"); + expect(progress).toEqual(["starting", "starting", "starting"]); + expect((yield* callsOf(fake)).filter((call) => call === "status")).toHaveLength(3); + }), + ), + ); + + it.live("fails with a wake timeout when the machine never starts", () => + Effect.scoped( + Effect.gen(function* () { + const { pool } = yield* setup( + { state: "paused" }, + { + onWake: () => ({ state: "starting", detail: "restoring snapshot" }), + wakeTimeout: "30 millis", + }, + ); + const error = yield* pool + .use(threadId, { wake: true, operation: "provider.sendTurn" }, () => Effect.void) + .pipe(Effect.flip); + expect(error).toMatchObject({ reason: "wake-timeout", state: "starting" }); + expect(error.detail).toContain("restoring snapshot"); + }), + ), + ); + + it.live("reports a failed machine as failed and announces it lost", () => + Effect.scoped( + Effect.gen(function* () { + const { pool } = yield* setup({ state: "failed", detail: "disk full" }); + const lost = yield* Deferred.make(); + yield* pool.lifecycle.pipe( + Stream.runForEach((event) => + event._tag === "lost" ? Deferred.succeed(lost, event.status.state) : Effect.void, + ), + Effect.forkScoped, + ); + yield* Effect.yieldNow; + const error = yield* pool + .use(threadId, { wake: false, operation: "vcs.status" }, () => Effect.void) + .pipe(Effect.flip); + expect(error).toMatchObject({ reason: "failed", detail: "disk full" }); + expect(yield* Deferred.await(lost)).toBe("failed"); + }), + ), + ); + + it.live("refuses a runner that serves another thread", () => + Effect.scoped( + Effect.gen(function* () { + const runner = yield* startRunner({ serves: ThreadId.make("other-thread") }); + const { pool } = yield* setup({ state: "running", runnerUrl: runner.url }); + const error = yield* pool + .use(threadId, { wake: false, operation: "vcs.status" }, () => Effect.void) + .pipe(Effect.flip); + expect(error).toMatchObject({ reason: "incompatible" }); + expect(Option.isNone(yield* pool.current(threadId))).toBe(true); + }), + ), + ); + + it.live("shares one connection attempt between concurrent callers", () => + Effect.scoped( + Effect.gen(function* () { + const runner = yield* startRunner(); + const { pool, fake } = yield* setup( + { state: "paused" }, + { onWake: () => ({ state: "running", runnerUrl: runner.url }) }, + ); + const results = yield* Effect.all( + [1, 2, 3].map(() => + pool.use(threadId, { wake: true, operation: "provider.sendTurn" }, (connection) => + Effect.succeed(connection.hello.runnerId), + ), + ), + { concurrency: "unbounded" }, + ); + expect(results).toEqual(["runner-test", "runner-test", "runner-test"]); + expect(runner.hellos()).toBe(1); + expect(yield* callsOf(fake)).toEqual(["ensure+wake"]); + }), + ), + ); + + it.live("closes idle connections and tells the directory; busy threads stay connected", () => + Effect.scoped( + Effect.gen(function* () { + const runner = yield* startRunner(); + const { pool, fake } = yield* setup( + { state: "running", runnerUrl: runner.url }, + { idleTimeout: "0 millis" }, + ); + yield* pool.use(threadId, { wake: false, operation: "vcs.status" }, () => Effect.void); + + yield* pool.setBusy(threadId, "turn", true); + yield* pool.sweepIdle; + expect(Option.isSome(yield* pool.current(threadId))).toBe(true); + + yield* pool.setBusy(threadId, "turn", false); + yield* pool.sweepIdle; + expect(Option.isNone(yield* pool.current(threadId))).toBe(true); + expect(yield* callsOf(fake)).toEqual(["status", "idle"]); + }), + ), + ); +}); diff --git a/apps/server/src/hub/RunnerEventDelivery.test.ts b/apps/server/src/hub/RunnerEventDelivery.test.ts new file mode 100644 index 000000000000..44df069344f0 --- /dev/null +++ b/apps/server/src/hub/RunnerEventDelivery.test.ts @@ -0,0 +1,316 @@ +import { it } from "@effect/vitest"; +import { + EventId, + ProviderDriverKind, + ProviderInstanceId, + type ProviderRuntimeEvent, + type ProviderSession, + ThreadId, + TurnId, +} from "@t3tools/contracts"; +import type { RunnerEventEnvelope } from "@t3tools/contracts/runner"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; +import * as Stream from "effect/Stream"; +import { describe, expect } from "vite-plus/test"; + +import { HubThreadMachineStateSqliteLive } from "../persistence/Layers/HubThreadMachineState.ts"; +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; +import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; +import { RunnerCursorStore } from "../persistence/Services/HubThreadMachineState.ts"; +import { + type FakeMachine, + MachineDirectory, + makeFakeMachineDirectory, +} from "./MachineDirectory.ts"; +import * as RemoteSessionRegistry from "./RemoteSessionRegistry.ts"; +import { make as makePool, RunnerConnectionPool } from "./RunnerConnectionPool.ts"; +import * as RunnerEventDelivery from "./RunnerEventDelivery.ts"; +import { fakeRunner, fakeRunnerHello, serveRunner } from "./testUtils/runnerServer.ts"; + +const threadId = ThreadId.make("thread-delivery"); +const provider = ProviderDriverKind.make("claudeAgent"); +const instanceId = ProviderInstanceId.make("claudeAgent"); + +const event = ( + id: string, + type: "session.started" | "turn.started" | "content.delta" | "turn.completed", +): ProviderRuntimeEvent => { + const base = { + eventId: EventId.make(id), + provider, + providerInstanceId: instanceId, + threadId, + createdAt: "2026-09-25T00:00:00.000Z", + turnId: TurnId.make("turn-1"), + }; + switch (type) { + case "session.started": + return { ...base, type, payload: {} }; + case "turn.started": + return { ...base, type, payload: {} }; + case "content.delta": + return { ...base, type, payload: { streamKind: "assistant_text", delta: "hi" } }; + case "turn.completed": + return { ...base, type, payload: { state: "completed" } }; + } +}; + +const envelope = (sequence: number, value: ProviderRuntimeEvent, bootId = "boot-1") => + ({ sequence, bootId, event: value }) satisfies RunnerEventEnvelope; + +/** A runner that replays `backlog`, then publishes whatever the test pushes. */ +const startRunner = (options: { + readonly bootId: string; + readonly backlog: ReadonlyArray; + readonly sessions?: ReadonlyArray; +}) => + Effect.gen(function* () { + const live = yield* PubSub.unbounded(); + const subscribedFrom: Array = []; + const acked: Array = []; + const headSequence = options.backlog.at(-1)?.sequence ?? 0; + const served = yield* serveRunner( + fakeRunner({ + "runner.hello": (input) => + Effect.succeed( + fakeRunnerHello({ + threadId: input.threadId, + bootId: options.bootId, + headSequence, + instances: [instanceId], + }), + ), + "runner.events.subscribe": ({ afterSequence }) => { + subscribedFrom.push(afterSequence); + return Stream.concat( + Stream.fromIterable(options.backlog.filter((entry) => entry.sequence > afterSequence)), + Stream.fromPubSub(live), + ); + }, + "runner.events.ack": ({ throughSequence }) => + Effect.sync(() => { + acked.push(throughSequence); + return { ackedSequence: throughSequence, retained: 0 }; + }), + "runner.provider.listSessions": () => Effect.succeed(options.sessions ?? []), + }), + ); + return { + url: served.url, + push: (entry: RunnerEventEnvelope) => PubSub.publish(live, entry), + subscribedFrom, + acked, + }; + }); + +const StoresLive = Layer.mergeAll( + HubThreadMachineStateSqliteLive, + ProviderSessionRuntime.layer, +).pipe(Layer.provideMerge(SqlitePersistenceMemory)); + +/** One hub lifetime: pool, registry and delivery over the shared stores. */ +const startHub = (machine: FakeMachine) => + Effect.gen(function* () { + const fake = yield* makeFakeMachineDirectory({ initial: [[threadId, machine]] }); + const pool = yield* makePool({ + wakePollInterval: "5 millis", + idleCheckInterval: "1 hour", + }).pipe(Effect.provideService(MachineDirectory, fake.directory)); + const registry = yield* RemoteSessionRegistry.make; + const delivery = yield* RunnerEventDelivery.make.pipe( + Effect.provideService(RunnerConnectionPool, pool), + Effect.provideService(RemoteSessionRegistry.RemoteSessionRegistry, registry), + ); + return { pool, registry, delivery, fake }; + }); + +const connect = (pool: RunnerConnectionPool["Service"]) => + pool.use(threadId, { wake: false, operation: "test.connect" }, () => Effect.void); + +const takeEvents = (delivery: RunnerEventDelivery.RunnerEventDelivery["Service"], count: number) => + delivery.events.pipe( + Stream.take(count), + Stream.runCollect, + Effect.map((events) => Array.from(events)), + Effect.forkScoped, + ); + +const session = (overrides: Partial = {}): ProviderSession => ({ + provider, + providerInstanceId: instanceId, + status: "running", + runtimeMode: "full-access", + threadId, + activeTurnId: TurnId.make("turn-1"), + createdAt: "2026-09-25T00:00:00.000Z", + updatedAt: "2026-09-25T00:00:00.000Z", + ...overrides, +}); + +describe("RunnerEventDelivery", () => { + it.live("delivers each event once and persists cursors only at turn boundaries", () => + Effect.scoped( + Effect.gen(function* () { + const runner = yield* startRunner({ + bootId: "boot-1", + backlog: [ + envelope(1, event("e1", "session.started")), + envelope(2, event("e2", "turn.started")), + envelope(3, event("e3", "content.delta")), + ], + }); + const hub = yield* startHub({ state: "running", runnerUrl: runner.url }); + const received = yield* takeEvents(hub.delivery, 4); + yield* Effect.yieldNow; + yield* hub.delivery.start; + yield* connect(hub.pool); + + const cursors = yield* RunnerCursorStore; + yield* Effect.repeat(hub.delivery.state, { + until: (state) => state.get(threadId)?.delivered === 3, + }); + yield* hub.delivery.flush({ minAgeMs: 0, drain: Effect.void }); + // The turn is open: only the event before it is durable. + expect(Option.getOrThrow(yield* cursors.get(threadId))).toMatchObject({ + outboxId: "runner-test", + bootId: "boot-1", + ackedSequence: 1, + }); + + yield* runner.push(envelope(4, event("e4", "turn.completed"))); + const events = yield* Fiber.join(received); + expect(events.map((value) => value.eventId)).toEqual(["e1", "e2", "e3", "e4"]); + yield* hub.delivery.flush({ minAgeMs: 0, drain: Effect.void }); + expect(Option.getOrThrow(yield* cursors.get(threadId)).ackedSequence).toBe(4); + yield* Effect.repeat( + Effect.sync(() => runner.acked), + { + until: (acked) => acked.includes(4), + }, + ); + expect(runner.acked).toEqual([1, 4]); + }), + ).pipe(Effect.provide(StoresLive)), + ); + + it.live("replays an open turn after a hub restart without re-delivering earlier events", () => + Effect.scoped( + Effect.gen(function* () { + const backlog = [ + envelope(1, event("e1", "session.started")), + envelope(2, event("e2", "turn.started")), + envelope(3, event("e3", "content.delta")), + ]; + const runner = yield* startRunner({ bootId: "boot-1", backlog }); + + yield* Effect.scoped( + Effect.gen(function* () { + const first = yield* startHub({ state: "running", runnerUrl: runner.url }); + yield* first.delivery.start; + yield* connect(first.pool); + yield* Effect.repeat(first.delivery.state, { + until: (state) => state.get(threadId)?.delivered === 3, + }); + yield* first.delivery.flush({ minAgeMs: 0, drain: Effect.void }); + }), + ); + + const second = yield* startHub({ state: "running", runnerUrl: runner.url }); + const received = yield* takeEvents(second.delivery, 2); + yield* Effect.yieldNow; + yield* second.delivery.start; + yield* connect(second.pool); + const replayed = yield* Fiber.join(received); + expect(runner.subscribedFrom).toEqual([0, 1]); + expect(replayed.map((value) => value.eventId)).toEqual(["e2", "e3"]); + }), + ).pipe(Effect.provide(StoresLive)), + ); + + it.live("settles a turn whose session a restarted runner no longer hosts", () => + Effect.scoped( + Effect.gen(function* () { + const runner = yield* startRunner({ bootId: "boot-2", backlog: [] }); + const hub = yield* startHub({ state: "running", runnerUrl: runner.url }); + yield* hub.registry.upsert({ + threadId, + instanceId, + provider, + session: session(), + bootId: "boot-1", + }); + const received = yield* takeEvents(hub.delivery, 2); + yield* Effect.yieldNow; + yield* hub.delivery.start; + yield* connect(hub.pool); + const settled = yield* Fiber.join(received); + expect(settled.map((value) => [value.type, value.eventId])).toEqual([ + ["turn.completed", `hub-reconcile:${threadId}:boot-2:turn-completed`], + ["session.exited", `hub-reconcile:${threadId}:boot-2:session-exited`], + ]); + expect(settled[0]).toMatchObject({ + turnId: "turn-1", + payload: { state: "interrupted" }, + }); + expect(Option.isNone(yield* hub.registry.get(threadId))).toBe(true); + }), + ).pipe(Effect.provide(StoresLive)), + ); + + it.live("adopts sessions a restarted runner still hosts", () => + Effect.scoped( + Effect.gen(function* () { + const runner = yield* startRunner({ + bootId: "boot-2", + backlog: [], + sessions: [session()], + }); + const hub = yield* startHub({ state: "running", runnerUrl: runner.url }); + yield* hub.registry.upsert({ + threadId, + instanceId, + provider, + session: session(), + bootId: "boot-1", + }); + yield* hub.delivery.start; + const connection = yield* hub.pool.use( + threadId, + { wake: false, operation: "test.connect" }, + (value) => Effect.succeed(value), + ); + yield* hub.delivery.awaitReconciled(connection); + expect(Option.getOrThrow(yield* hub.registry.get(threadId)).bootId).toBe("boot-2"); + }), + ).pipe(Effect.provide(StoresLive)), + ); + + it.live("settles sessions when the directory reports their machine failed", () => + Effect.scoped( + Effect.gen(function* () { + const hub = yield* startHub({ state: "failed", detail: "host lost" }); + yield* hub.registry.upsert({ + threadId, + instanceId, + provider, + session: session(), + bootId: "boot-1", + }); + const received = yield* takeEvents(hub.delivery, 2); + yield* Effect.yieldNow; + yield* hub.delivery.start; + yield* connect(hub.pool).pipe(Effect.ignore); + const settled = yield* Fiber.join(received); + expect(settled.map((value) => value.type)).toEqual(["turn.completed", "session.exited"]); + expect(settled[1]).toMatchObject({ + eventId: `hub-reconcile:${threadId}:lost:boot-1:session-exited`, + payload: { recoverable: true }, + }); + }), + ).pipe(Effect.provide(StoresLive)), + ); +}); diff --git a/apps/server/src/hub/testUtils/runnerServer.ts b/apps/server/src/hub/testUtils/runnerServer.ts new file mode 100644 index 000000000000..695c01fa7d58 --- /dev/null +++ b/apps/server/src/hub/testUtils/runnerServer.ts @@ -0,0 +1,102 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Loopback test server. +/** + * Test helpers: serve `RunnerRpcGroup` handlers on a loopback port, and a fake + * runner whose handlers default to failing loudly so tests implement exactly + * the calls they expect. + * + * @module hub/testUtils/runnerServer + */ +import * as NodeHttp from "node:http"; + +import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; +import { + RUNNER_PROTOCOL_VERSION, + RUNNER_WS_PATH, + type RunnerHello, + RunnerRpcGroup, +} from "@t3tools/contracts/runner"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import { HttpRouter, HttpServer } from "effect/unstable/http"; +import { RpcSerialization, RpcServer } from "effect/unstable/rpc"; +import type * as Rpc from "effect/unstable/rpc/Rpc"; +import type * as RpcGroup from "effect/unstable/rpc/RpcGroup"; + +type RunnerHandlers = Parameters[0]; + +const STREAM_TAGS = new Set([ + "runner.events.subscribe", + "runner.vcs.streamStatus", + "runner.git.runStackedAction", + "runner.terminal.attach", + "runner.terminal.events", + "runner.terminal.metadata", +]); + +/** Handlers for every RPC: streams stay open and emit nothing, calls die. */ +export const unimplementedRunnerHandlers = (): RunnerHandlers => + Object.fromEntries( + [...RunnerRpcGroup.requests.keys()].map((tag) => [ + tag, + STREAM_TAGS.has(tag) + ? () => Stream.never + : () => Effect.die(new Error(`fake runner: ${tag} is not implemented`)), + ]), + ) as unknown as RunnerHandlers; + +export const fakeRunnerHello = (overrides: Partial & Pick) => + ({ + protocolVersion: RUNNER_PROTOCOL_VERSION, + runnerId: "runner-test", + bootId: "boot-1", + checkout: `/workspace/t/${overrides.threadId}`, + headSequence: 0, + ackedSequence: 0, + firstRetainedSequence: 1, + instances: [], + ...overrides, + }) satisfies RunnerHello; + +/** + * Serves `handlers` at `RUNNER_WS_PATH` on 127.0.0.1 with an OS-assigned port + * for the lifetime of the scope. Returns the runner's WebSocket URL. + */ +export const serveRunner = (handlers: Layer.Layer) => + Effect.gen(function* () { + const routes = Layer.unwrap( + Effect.gen(function* () { + const built = yield* Layer.build( + handlers.pipe(Layer.provideMerge(RpcSerialization.layerJson)), + ); + const httpEffect = yield* RpcServer.toHttpEffectWebsocket(RunnerRpcGroup, { + disableTracing: true, + }).pipe(Effect.provide(built)); + return HttpRouter.add("GET", RUNNER_WS_PATH, httpEffect); + }), + ); + const context = yield* Layer.build( + HttpRouter.serve(routes, { disableListenLog: true, disableLogger: true }).pipe( + Layer.provideMerge( + NodeHttpServer.layer(NodeHttp.createServer, { + host: "127.0.0.1", + port: 0, + // Close open runner sockets at once, like a runner process exiting. + disablePreemptiveShutdown: true, + }), + ), + ), + ); + const address = Context.get(context, HttpServer.HttpServer).address as HttpServer.TcpAddress; + return { url: `ws://127.0.0.1:${address.port}${RUNNER_WS_PATH}` }; + }); + +/** A runner whose unimplemented calls die; `overrides` supplies the ones a test uses. */ +export const fakeRunner = ( + overrides: Partial, +): Layer.Layer>> => + // @effect-diagnostics-next-line anyUnknownInErrorContext:off - handler overrides are loosely typed test doubles. + RunnerRpcGroup.toLayer( + Effect.succeed(RunnerRpcGroup.of({ ...unimplementedRunnerHandlers(), ...overrides })), + ) as Layer.Layer>>; diff --git a/apps/server/src/persistence/Layers/HubThreadMachineState.test.ts b/apps/server/src/persistence/Layers/HubThreadMachineState.test.ts new file mode 100644 index 000000000000..a92aebd61d9d --- /dev/null +++ b/apps/server/src/persistence/Layers/HubThreadMachineState.test.ts @@ -0,0 +1,117 @@ +import { it } from "@effect/vitest"; +import { ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import { describe, expect } from "vite-plus/test"; + +import { + CheckpointTurnDiffStore, + RunnerCursorStore, + ThreadVcsStatusStore, +} from "../Services/HubThreadMachineState.ts"; +import { HubThreadMachineStateSqliteLive } from "./HubThreadMachineState.ts"; +import { SqlitePersistenceMemory } from "./Sqlite.ts"; + +const TestLayer = HubThreadMachineStateSqliteLive.pipe(Layer.provide(SqlitePersistenceMemory)); +const threadId = ThreadId.make("thread-1"); +const otherThread = ThreadId.make("thread-2"); + +describe("hub thread-machine state on SQLite", () => { + it.effect("upserts runner cursors transactionally and reads them back", () => + Effect.gen(function* () { + const cursors = yield* RunnerCursorStore; + expect(Option.isNone(yield* cursors.get(threadId))).toBe(true); + yield* cursors.saveAll([ + { threadId, outboxId: "runner-a", bootId: "boot-1", ackedSequence: 4, updatedAt: "t1" }, + { + threadId: otherThread, + outboxId: "runner-b", + bootId: "boot-9", + ackedSequence: 1, + updatedAt: "t1", + }, + ]); + yield* cursors.saveAll([ + { threadId, outboxId: "runner-a", bootId: "boot-2", ackedSequence: 9, updatedAt: "t2" }, + ]); + expect(Option.getOrThrow(yield* cursors.get(threadId))).toEqual({ + threadId, + outboxId: "runner-a", + bootId: "boot-2", + ackedSequence: 9, + updatedAt: "t2", + }); + yield* cursors.remove(otherThread); + expect((yield* cursors.list()).map((cursor) => cursor.threadId)).toEqual([threadId]); + }).pipe(Effect.provide(TestLayer)), + ); + + it.effect("keys diffs by whitespace mode and invalidates everything reading a turn", () => + Effect.gen(function* () { + const diffs = yield* CheckpointTurnDiffStore; + const key = (from: number, to: number, ignoreWhitespace = true) => ({ + threadId, + fromTurnCount: from, + toTurnCount: to, + ignoreWhitespace, + }); + for (const [from, to] of [ + [0, 1], + [1, 2], + [0, 2], + [2, 3], + ] as const) { + yield* diffs.put({ ...key(from, to), diff: `diff ${from}-${to}`, createdAt: "t" }); + } + yield* diffs.put({ ...key(1, 2, false), diff: "diff 1-2 exact", createdAt: "t" }); + expect(Option.getOrNull(yield* diffs.get(key(1, 2)))).toBe("diff 1-2"); + expect(Option.getOrNull(yield* diffs.get(key(1, 2, false)))).toBe("diff 1-2 exact"); + + // Recapturing turn 2 invalidates diffs from or to turn 2 and later. + yield* diffs.invalidateFromTurn({ threadId, turnCount: 2 }); + expect(Option.getOrNull(yield* diffs.get(key(0, 1)))).toBe("diff 0-1"); + expect(Option.isNone(yield* diffs.get(key(1, 2)))).toBe(true); + expect(Option.isNone(yield* diffs.get(key(0, 2)))).toBe(true); + expect(Option.isNone(yield* diffs.get(key(2, 3)))).toBe(true); + + yield* diffs.removeThread(threadId); + expect(Option.isNone(yield* diffs.get(key(0, 1)))).toBe(true); + }).pipe(Effect.provide(TestLayer)), + ); + + it.effect("stores the last git status per thread", () => + Effect.gen(function* () { + const statuses = yield* ThreadVcsStatusStore; + const local = { + isRepo: true, + hasPrimaryRemote: true, + isDefaultRef: false, + refName: "t3/work", + hasWorkingTreeChanges: true, + workingTree: { + files: [{ path: "a.ts", insertions: 2, deletions: 1 }], + insertions: 2, + deletions: 1, + }, + }; + yield* statuses.put({ threadId, local, remote: null, updatedAt: "t1" }); + yield* statuses.put({ + threadId, + local, + remote: { hasUpstream: true, aheadCount: 1, behindCount: 0, pr: null }, + updatedAt: "t2", + }); + const rows = yield* statuses.list(); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + threadId, + local, + remote: { aheadCount: 1 }, + updatedAt: "t2", + }); + yield* statuses.remove(threadId); + expect(yield* statuses.list()).toEqual([]); + }).pipe(Effect.provide(TestLayer)), + ); +}); diff --git a/apps/server/src/persistence/Postgres/HubThreadMachineState.test.ts b/apps/server/src/persistence/Postgres/HubThreadMachineState.test.ts new file mode 100644 index 000000000000..ddf5bf11b63e --- /dev/null +++ b/apps/server/src/persistence/Postgres/HubThreadMachineState.test.ts @@ -0,0 +1,111 @@ +/** + * Hub thread-machine state on Postgres (migration 050). Runs only when + * T3_HUB_TEST_DATABASE_URL points at a disposable database, e.g. + * T3_HUB_TEST_DATABASE_URL=postgres://postgres@127.0.0.1:54339/hub + */ +import * as NodeCrypto from "node:crypto"; + +import { it } from "@effect/vitest"; +import { ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import { expect } from "vite-plus/test"; + +import { makeHubPgClientLayer } from "../Layers/Postgres.ts"; +import { + CheckpointTurnDiffStore, + RunnerCursorStore, + ThreadVcsStatusStore, +} from "../Services/HubThreadMachineState.ts"; +import { HubThreadMachineStatePostgresLive } from "./HubThreadMachineState.ts"; +import { hubTenantLayer } from "./HubTenant.ts"; +import { + HUB_MIGRATION_050, + HUB_THREAD_MACHINE_STATE_TABLES, +} from "./migrations/050_HubThreadMachineState.ts"; + +const databaseUrl = process.env.T3_HUB_TEST_DATABASE_URL; +const runId = NodeCrypto.randomUUID().slice(0, 8); +const userA = `test-machines-a-${runId}`; +const userB = `test-machines-b-${runId}`; +const threadId = ThreadId.make("thread-shared"); + +const storesFor = (userId: string) => + HubThreadMachineStatePostgresLive.pipe(Layer.provide(hubTenantLayer(userId))); + +if (databaseUrl === undefined) { + it.skip("hub thread-machine state on Postgres (set T3_HUB_TEST_DATABASE_URL)", () => {}); +} else { + it.layer( + Layer.effectDiscard(HUB_MIGRATION_050).pipe( + Layer.provideMerge(makeHubPgClientLayer({ url: databaseUrl, maxConnections: 2 })), + ), + { excludeTestServices: true }, + )("hub thread-machine state on Postgres", (it) => { + it.effect("isolates cursors, diffs and git status per user", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const writeAs = (userId: string, sequence: number) => + Effect.gen(function* () { + yield* (yield* RunnerCursorStore).saveAll([ + { + threadId, + outboxId: "runner", + bootId: userId, + ackedSequence: sequence, + updatedAt: "t", + }, + ]); + yield* (yield* CheckpointTurnDiffStore).put({ + threadId, + fromTurnCount: 0, + toTurnCount: 1, + ignoreWhitespace: true, + diff: `diff of ${userId}`, + createdAt: "t", + }); + }).pipe(Effect.provide(storesFor(userId))); + yield* writeAs(userA, 3); + yield* writeAs(userB, 7); + + const readAs = (userId: string) => + Effect.gen(function* () { + const cursor = yield* (yield* RunnerCursorStore).get(threadId); + const diff = yield* (yield* CheckpointTurnDiffStore).get({ + threadId, + fromTurnCount: 0, + toTurnCount: 1, + ignoreWhitespace: true, + }); + return { + sequence: Option.getOrThrow(cursor).ackedSequence, + diff: Option.getOrNull(diff), + statuses: yield* (yield* ThreadVcsStatusStore).list(), + }; + }).pipe(Effect.provide(storesFor(userId))); + expect(yield* readAs(userA)).toEqual({ + sequence: 3, + diff: `diff of ${userA}`, + statuses: [], + }); + expect(yield* readAs(userB)).toEqual({ + sequence: 7, + diff: `diff of ${userB}`, + statuses: [], + }); + + yield* Effect.gen(function* () { + yield* (yield* CheckpointTurnDiffStore).invalidateFromTurn({ threadId, turnCount: 1 }); + }).pipe(Effect.provide(storesFor(userA))); + expect((yield* readAs(userA)).diff).toBeNull(); + expect((yield* readAs(userB)).diff).toBe(`diff of ${userB}`); + + for (const table of HUB_THREAD_MACHINE_STATE_TABLES) { + yield* sql`DELETE FROM ${sql(table)} WHERE ${sql.in("user_id", [userA, userB])}`; + } + }), + ); + }); +} diff --git a/apps/server/src/runner/RunnerCheckout.test.ts b/apps/server/src/runner/RunnerCheckout.test.ts new file mode 100644 index 000000000000..cf817ce6c1e0 --- /dev/null +++ b/apps/server/src/runner/RunnerCheckout.test.ts @@ -0,0 +1,157 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { it } from "@effect/vitest"; +import { ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { afterEach, describe, expect } from "vite-plus/test"; + +import * as ServerConfig from "../config.ts"; +import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; +import * as VcsProcess from "../vcs/VcsProcess.ts"; +import { RunnerCheckout, layer as RunnerCheckoutLayer } from "./RunnerCheckout.ts"; + +const git = (cwd: string, args: ReadonlyArray) => + NodeChildProcess.execFileSync( + "git", + ["-c", "user.email=test@example.com", "-c", "user.name=Test", ...args], + { cwd, stdio: ["ignore", "pipe", "pipe"], encoding: "utf8" }, + ).trim(); + +const tempDirs: Array = []; +afterEach(() => { + for (const dir of tempDirs.splice(0)) NodeFS.rmSync(dir, { recursive: true, force: true }); +}); + +/** A bare "origin" with `main` and a pushed `feature/existing` branch. */ +const makeOrigin = () => { + const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "runner-checkout-test-")); + tempDirs.push(root); + const source = NodePath.join(root, "source"); + const bare = NodePath.join(root, "origin.git"); + NodeFS.mkdirSync(source); + git(source, ["init", "--initial-branch=main"]); + NodeFS.writeFileSync(NodePath.join(source, "README.md"), "hello\n"); + git(source, ["add", "."]); + git(source, ["commit", "-m", "initial"]); + git(source, ["checkout", "-b", "feature/existing"]); + NodeFS.writeFileSync(NodePath.join(source, "feature.txt"), "feature\n"); + git(source, ["add", "."]); + git(source, ["commit", "-m", "feature"]); + git(root, ["clone", "--bare", source, bare]); + return { + root, + url: `file://${bare}`, + mainHead: git(source, ["rev-parse", "main"]), + featureHead: git(source, ["rev-parse", "feature/existing"]), + }; +}; + +const TestLayer = RunnerCheckoutLayer.pipe( + Layer.provide(GitVcsDriver.layer), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-runner-checkout-" })), + Layer.provideMerge(VcsProcess.layer), + Layer.provideMerge(NodeServices.layer), +); + +const threadId = ThreadId.make("thread-checkout"); + +describe("RunnerCheckout", () => { + it.effect("clones, creates the thread branch from the base ref, and is idempotent", () => + Effect.gen(function* () { + const origin = makeOrigin(); + const checkout = NodePath.join(origin.root, "t", threadId); + const runner = yield* RunnerCheckout; + const input = { + threadId, + checkout, + repository: { url: origin.url, ref: "main" }, + branch: "t3/work", + baseRef: "main", + }; + + const first = yield* runner.prepare(input); + expect(first).toEqual({ + checkout, + isRepository: true, + branch: "t3/work", + headCommit: origin.mainHead, + created: true, + }); + + // Local work in the checkout survives later prepares. + NodeFS.writeFileSync(NodePath.join(checkout, "work.txt"), "in progress\n"); + const second = yield* runner.prepare(input); + expect(second).toEqual({ ...first, created: false }); + expect(NodeFS.readFileSync(NodePath.join(checkout, "work.txt"), "utf8")).toBe( + "in progress\n", + ); + }).pipe(Effect.provide(TestLayer)), + ); + + it.effect("continues a branch that was already pushed from an earlier machine", () => + Effect.gen(function* () { + const origin = makeOrigin(); + const checkout = NodePath.join(origin.root, "t", threadId); + const result = yield* (yield* RunnerCheckout).prepare({ + threadId, + checkout, + repository: { url: origin.url, ref: null }, + branch: "feature/existing", + baseRef: "main", + }); + expect(result.branch).toBe("feature/existing"); + expect(result.headCommit).toBe(origin.featureHead); + expect(git(checkout, ["rev-parse", "--abbrev-ref", "@{upstream}"])).toBe( + "origin/feature/existing", + ); + }).pipe(Effect.provide(TestLayer)), + ); + + it.effect("initializes a blank project without a repository", () => + Effect.gen(function* () { + const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "runner-checkout-blank-")); + tempDirs.push(root); + const checkout = NodePath.join(root, "t", threadId); + const result = yield* (yield* RunnerCheckout).prepare({ + threadId, + checkout, + repository: null, + branch: "t3/blank", + baseRef: null, + }); + expect(result).toEqual({ + checkout, + isRepository: true, + branch: "t3/blank", + headCommit: null, + created: true, + }); + }).pipe(Effect.provide(TestLayer)), + ); + + it.effect("refuses to clone over a non-empty directory that is not a repository", () => + Effect.gen(function* () { + const origin = makeOrigin(); + const checkout = NodePath.join(origin.root, "t", threadId); + NodeFS.mkdirSync(checkout, { recursive: true }); + NodeFS.writeFileSync(NodePath.join(checkout, "stray.txt"), "keep me\n"); + const error = yield* (yield* RunnerCheckout) + .prepare({ + threadId, + checkout, + repository: { url: origin.url, ref: null }, + branch: null, + baseRef: null, + }) + .pipe(Effect.flip); + expect(error._tag).toBe("RunnerCheckoutError"); + expect(NodeFS.readFileSync(NodePath.join(checkout, "stray.txt"), "utf8")).toBe("keep me\n"); + }).pipe(Effect.provide(TestLayer)), + ); +}); diff --git a/apps/server/src/runner/RunnerOutbox.test.ts b/apps/server/src/runner/RunnerOutbox.test.ts index 7d2f2496e953..62fe583e6096 100644 --- a/apps/server/src/runner/RunnerOutbox.test.ts +++ b/apps/server/src/runner/RunnerOutbox.test.ts @@ -3,7 +3,6 @@ import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; -import * as NodeServices from "@effect/platform-node/NodeServices"; import { it } from "@effect/vitest"; import { EventId, @@ -13,36 +12,39 @@ import { } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; -import * as Layer from "effect/Layer"; import * as Stream from "effect/Stream"; -import { describe, expect } from "vite-plus/test"; +import { afterEach, describe, expect } from "vite-plus/test"; -import * as ServerConfig from "../config.ts"; -import { RunnerOutbox, layer as RunnerOutboxLayer } from "./RunnerOutbox.ts"; +import { makeWithOptions, type RunnerOutboxShape } from "./RunnerOutbox.ts"; -const event = (n: number): ProviderRuntimeEvent => ({ +const event = (id: string): ProviderRuntimeEvent => ({ type: "session.state.changed", - eventId: EventId.make(`evt-${n}`), + eventId: EventId.make(id), provider: ProviderDriverKind.make("claudeAgent"), threadId: ThreadId.make("thread-outbox-test"), createdAt: "2026-09-25T00:00:00.000Z", payload: { state: "ready" }, }); -/** One runner process lifetime against a persistent state directory. */ +const tempDirs: Array = []; +const makeDatabasePath = () => { + const dir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "runner-outbox-test-")); + tempDirs.push(dir); + return NodePath.join(dir, "runner", "outbox.sqlite"); +}; +afterEach(() => { + for (const dir of tempDirs.splice(0)) NodeFS.rmSync(dir, { recursive: true, force: true }); +}); + +/** One runner process lifetime against a persistent database. */ const boot = ( - baseDir: string, - body: (outbox: RunnerOutbox["Service"]) => Effect.Effect, + databasePath: string, + body: (outbox: RunnerOutboxShape) => Effect.Effect, + maxRetained?: number, ) => - Effect.gen(function* () { - const outbox = yield* RunnerOutbox; - return yield* body(outbox); - }).pipe( - Effect.provide( - RunnerOutboxLayer.pipe( - Layer.provide(ServerConfig.ServerConfig.layerTest(process.cwd(), baseDir)), - Layer.provide(NodeServices.layer), - ), + Effect.scoped( + makeWithOptions({ databasePath, ...(maxRetained ? { maxRetained } : {}) }).pipe( + Effect.flatMap(body), ), ); @@ -56,13 +58,12 @@ const sequencesOf = (stream: Stream.Stream<{ readonly sequence: number }>, count describe("RunnerOutbox", () => { it.effect("replays only unacknowledged events across restarts, in order", () => Effect.gen(function* () { - const baseDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "runner-outbox-test-")); - - const firstBoot = yield* boot(baseDir, (outbox) => + const databasePath = makeDatabasePath(); + const firstBoot = yield* boot(databasePath, (outbox) => Effect.gen(function* () { - yield* outbox.append(event(1)); - yield* outbox.append(event(2)); - yield* outbox.append(event(3)); + yield* outbox.append(event("e1")); + yield* outbox.append(event("e2")); + yield* outbox.append(event("e3")); const replayed = yield* sequencesOf(outbox.subscribe(0), 3); const acked = yield* outbox.ack(2); return { replayed, acked, stats: yield* outbox.stats }; @@ -71,45 +72,103 @@ describe("RunnerOutbox", () => { expect(firstBoot.replayed).toEqual([1, 2, 3]); expect(firstBoot.acked).toEqual({ ackedSequence: 2, retained: 1 }); - // A crash can leave a torn final line; it must not block the next boot. - const outboxPath = NodePath.join(baseDir, "userdata", "runner", "outbox.ndjson"); - NodeFS.appendFileSync(outboxPath, '{"sequence":4,"bootId":"torn","ev'); - - const secondBoot = yield* boot(baseDir, (outbox) => + const secondBoot = yield* boot(databasePath, (outbox) => Effect.gen(function* () { const before = yield* outbox.stats; - yield* outbox.append(event(4)); - const fromStart = yield* sequencesOf(outbox.subscribe(0), 2); + yield* outbox.append(event("e4")); + const fromAck = yield* sequencesOf(outbox.subscribe(2), 2); const fromCursor = yield* sequencesOf(outbox.subscribe(3), 1); - return { before, fromStart, fromCursor, after: yield* outbox.stats }; + return { before, fromAck, fromCursor, after: yield* outbox.stats }; }), ); expect(secondBoot.before.runnerId).toBe(firstBoot.stats.runnerId); expect(secondBoot.before.bootId).not.toBe(firstBoot.stats.bootId); - expect(secondBoot.before).toMatchObject({ headSequence: 3, ackedSequence: 2, retained: 1 }); - expect(secondBoot.fromStart).toEqual([3, 4]); + expect(secondBoot.before).toMatchObject({ + headSequence: 3, + ackedSequence: 2, + firstRetainedSequence: 3, + retained: 1, + }); + expect(secondBoot.fromAck).toEqual([3, 4]); expect(secondBoot.fromCursor).toEqual([4]); expect(secondBoot.after.headSequence).toBe(4); + }), + ); - NodeFS.rmSync(baseDir, { recursive: true, force: true }); + it.effect("keeps sequences increasing after every event is acknowledged", () => + Effect.gen(function* () { + const databasePath = makeDatabasePath(); + yield* boot(databasePath, (outbox) => + Effect.gen(function* () { + yield* outbox.append(event("e1")); + yield* outbox.append(event("e2")); + yield* outbox.ack(2); + }), + ); + const next = yield* boot(databasePath, (outbox) => + outbox.append(event("e3")).pipe(Effect.andThen(sequencesOf(outbox.subscribe(0), 1))), + ); + expect(next).toEqual([3]); }), ); it.effect("delivers events appended after subscribing exactly once", () => Effect.gen(function* () { - const baseDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "runner-outbox-test-")); - const sequences = yield* boot(baseDir, (outbox) => + const databasePath = makeDatabasePath(); + const sequences = yield* boot(databasePath, (outbox) => Effect.gen(function* () { - yield* outbox.append(event(1)); + yield* outbox.append(event("e1")); const fiber = yield* sequencesOf(outbox.subscribe(0), 3).pipe(Effect.forkChild); yield* Effect.yieldNow; - yield* outbox.append(event(2)); - yield* outbox.append(event(3)); + yield* outbox.append(event("e2")); + yield* outbox.append(event("e3")); return yield* Fiber.join(fiber); }), ); expect(sequences).toEqual([1, 2, 3]); - NodeFS.rmSync(baseDir, { recursive: true, force: true }); + }), + ); + + it.effect("makes duplicate adapter event ids unique so hub command ids cannot collide", () => + Effect.gen(function* () { + const databasePath = makeDatabasePath(); + const eventIds = yield* boot(databasePath, (outbox) => + Effect.gen(function* () { + yield* outbox.append(event("same")); + yield* outbox.append(event("same")); + return yield* outbox.subscribe(0).pipe( + Stream.take(2), + Stream.runCollect, + Effect.map((entries) => Array.from(entries, (entry) => entry.event.eventId)), + ); + }), + ); + expect(eventIds).toEqual(["same", "same:2"]); + }), + ); + + it.effect("drops the oldest unacknowledged events past retention and reports the gap", () => + Effect.gen(function* () { + const databasePath = makeDatabasePath(); + const result = yield* boot( + databasePath, + (outbox) => + Effect.gen(function* () { + for (const id of ["e1", "e2", "e3", "e4", "e5"]) yield* outbox.append(event(id)); + return { + stats: yield* outbox.stats, + replay: yield* sequencesOf(outbox.subscribe(0), 3), + }; + }), + 3, + ); + expect(result.stats).toMatchObject({ + headSequence: 5, + firstRetainedSequence: 3, + retained: 3, + dropped: 2, + }); + expect(result.replay).toEqual([3, 4, 5]); }), ); }); From 11e56871b8decf511463b675ee8a12f6cea6539c Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 23:37:25 +0000 Subject: [PATCH 21/58] fix(server): make checkpoint capture durable before acknowledging runner events Runner delivery cursors now advance only after both ingestion and the checkpoint reactor drain, so a hub crash right after a turn cannot lose the turn's checkpoint capture. At startup the hub resumes delivery from every thread with a live remote session whose machine is running, not only threads with an active turn, so events a runner emitted while the hub was down are ingested without waiting for the next wake. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/hub/HubRunnerLifecycle.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/apps/server/src/hub/HubRunnerLifecycle.ts b/apps/server/src/hub/HubRunnerLifecycle.ts index fd866c54c8e8..28b9a8a3110c 100644 --- a/apps/server/src/hub/HubRunnerLifecycle.ts +++ b/apps/server/src/hub/HubRunnerLifecycle.ts @@ -2,11 +2,14 @@ * HubRunnerLifecycle - hub-wide background work for thread machines. * * - Opens runner event delivery once the server is activated (ingestion - * subscribes at the same gate), then resumes delivery from machines that - * were running a turn when the hub stopped. Resuming never wakes a machine. + * subscribes at the same gate), then resumes delivery from every thread + * with a live remote session whose machine is running, so events emitted + * while the hub was down are ingested. Resuming never wakes a machine. * - Flushes delivery cursors: every `ACK_INTERVAL`, safe points older than - * `ACK_SETTLE` are persisted after ingestion drains, then acknowledged to - * their runners. A final flush runs on shutdown. + * `ACK_SETTLE` are persisted once ingestion and the checkpoint reactor have + * drained (so a turn's checkpoint capture is durable before its events are + * acknowledged), then acknowledged to their runners. A final flush runs on + * shutdown. * - Gives the connection pool the project and repository of each thread for * machine directory requests. * - Releases a thread's machine when the thread is archived or deleted, and @@ -23,6 +26,7 @@ import * as Stream from "effect/Stream"; import { isBootstrapCleanupDeletion } from "../orchestration/Layers/ThreadDeletionReactor.ts"; import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts"; +import { CheckpointReactor } from "../orchestration/Services/CheckpointReactor.ts"; import { ProjectionSnapshotQuery } from "../orchestration/Services/ProjectionSnapshotQuery.ts"; import { ProviderRuntimeIngestionService } from "../orchestration/Services/ProviderRuntimeIngestion.ts"; import { @@ -54,6 +58,7 @@ export const layer = Layer.effectDiscard( const delivery = yield* RunnerEventDelivery; const registry = yield* RemoteSessionRegistry; const ingestion = yield* ProviderRuntimeIngestionService; + const checkpointReactor = yield* CheckpointReactor; const engine = yield* OrchestrationEngineService; const projections = yield* ProjectionSnapshotQuery; const repositoryIdentity = yield* RepositoryIdentityResolver; @@ -83,16 +88,17 @@ export const layer = Layer.effectDiscard( }), ); - const flush = (minAgeMs: number) => delivery.flush({ minAgeMs, drain: ingestion.drain }); + const flush = (minAgeMs: number) => + delivery.flush({ + minAgeMs, + drain: ingestion.drain.pipe(Effect.andThen(checkpointReactor.drain)), + }); yield* forkParked( Effect.gen(function* () { yield* Effect.sleep(DELIVERY_START_DELAY); yield* delivery.start; for (const record of yield* registry.list) { - if (record.session.status !== "running" && record.session.activeTurnId === undefined) { - continue; - } yield* pool .use(record.threadId, { wake: false, operation: "resume-delivery" }, () => Effect.void) .pipe( From a106dfc47328b3b9fd64308f38bbe017dd9e06be Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 23:37:27 +0000 Subject: [PATCH 22/58] test(server): add a hub and runner end-to-end script apps/server/scripts/thread-machines-e2e.mjs starts `t3 runner` (4422) and a hub (4421) on loopback in a temporary directory and drives the hub's public API: per-thread checkout paths and virtual project roots, a turn that edits the runner checkout with an approval round trip, the turn diff through the hub and again with the runner stopped, a runner restart followed by a turn that resumes the provider session, a hub restart, and title generation on the runner. It uses the machine's existing provider login. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/scripts/thread-machines-e2e.mjs | 434 ++++++++++++++++++++ 1 file changed, 434 insertions(+) create mode 100644 apps/server/scripts/thread-machines-e2e.mjs diff --git a/apps/server/scripts/thread-machines-e2e.mjs b/apps/server/scripts/thread-machines-e2e.mjs new file mode 100644 index 000000000000..67aeea0cd64b --- /dev/null +++ b/apps/server/scripts/thread-machines-e2e.mjs @@ -0,0 +1,434 @@ +#!/usr/bin/env node +// @ts-check +/** + * End-to-end check of hub mode against a real runner process, on loopback. + * + * Starts `t3 runner` for one thread (port 4422) and a hub + * (`T3CODE_SERVER_MODE=hub`, port 4421) that reaches it through the static + * development machine directory (`T3CODE_RUNNER_URL`), then drives the hub's + * public HTTP and WebSocket API like a client: + * + * a. project and thread creation: the hub assigns the thread checkout path + * b. a turn that edits notes.txt on the runner, with an approval round trip + * c. the turn diff through the hub + * d. the same diff with the runner stopped (served from the hub cache) + * e. a runner restart (new boot), then a turn that resumes the session + * f. a hub restart; history and diffs survive + * + * It uses the real provider through T3's adapter on the runner, with this + * machine's existing provider login, for two short turns (plus title + * generation). The model defaults to claude-haiku-4-5 on the claudeAgent + * instance. Nothing outside the temporary directory is written; processes + * started here are stopped by PID on exit. + * + * node apps/server/scripts/thread-machines-e2e.mjs [--keep] [--model ] + */ +import * as NodeChildProcess from "node:child_process"; +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; + +const HUB_PORT = 4421; +const RUNNER_PORT = 4422; +const HUB = `http://127.0.0.1:${HUB_PORT}`; +const BIN = NodePath.resolve( + NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), + "../src/bin.ts", +); + +const args = process.argv.slice(2); +const flagValue = (name, fallback) => { + const index = args.indexOf(name); + return index >= 0 ? args[index + 1] : fallback; +}; +const KEEP = args.includes("--keep"); +const MODEL = { instanceId: "claudeAgent", model: flagValue("--model", "claude-haiku-4-5") }; + +const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-thread-machines-e2e-")); +const threadId = `thread-${NodeCrypto.randomUUID()}`; +const projectId = `project-${NodeCrypto.randomUUID()}`; +const checkoutRoot = NodePath.join(root, "t"); +const checkout = NodePath.join(checkoutRoot, threadId); +const runnerToken = NodeCrypto.randomBytes(24).toString("base64url"); +const logs = NodePath.join(root, "logs"); + +const stamp = () => new Date().toISOString().slice(11, 23); +const log = (...values) => console.log(`[${stamp()}]`, ...values); +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const now = () => new Date().toISOString(); +const id = () => NodeCrypto.randomUUID(); +const results = []; +const check = (name, ok, detail = "") => { + results.push({ name, ok, detail }); + log(`${ok ? "PASS" : "FAIL"} ${name}${detail ? ` - ${detail}` : ""}`); +}; + +// ── Processes ───────────────────────────────────────────────────────────── + +const children = new Map(); +const baseEnv = { + ...process.env, + T3CODE_TELEMETRY_ENABLED: "false", + T3CODE_NO_BROWSER: "true", + T3CODE_AUTO_BOOTSTRAP_PROJECT_FROM_CWD: "false", +}; + +function start(name, cwd, argv, env) { + const out = NodeFS.openSync(NodePath.join(logs, `${name}.log`), "a"); + const child = NodeChildProcess.spawn(process.execPath, [BIN, ...argv], { + cwd, + env: { ...baseEnv, ...env }, + stdio: ["ignore", out, out], + }); + children.set(name, child); + return child; +} + +async function stop(name) { + const child = children.get(name); + if (!child || child.exitCode !== null) return; + const started = Date.now(); + child.kill("SIGTERM"); + await new Promise((resolve) => { + const timer = setTimeout(() => { + child.kill("SIGKILL"); + resolve(undefined); + }, 15_000); + child.once("exit", () => { + clearTimeout(timer); + resolve(undefined); + }); + }); + children.delete(name); + log(`${name} stopped in ${Date.now() - started} ms`); +} + +async function waitFor(description, probe, timeoutMs = 60_000) { + const deadline = Date.now() + timeoutMs; + let last; + while (Date.now() < deadline) { + try { + last = await probe(); + if (last) return last; + } catch (error) { + last = error; + } + await sleep(250); + } + throw new Error(`timed out waiting for ${description}: ${String(last?.message ?? last)}`); +} + +const listening = (port) => + NodeChildProcess.execFileSync("ss", ["-H", "-ltn", `sport = :${port}`], { + encoding: "utf8", + }).trim().length > 0; + +async function startRunner() { + const started = Date.now(); + start( + "runner", + checkout, + ["runner", "--base-dir", NodePath.join(root, "runner"), "--port", `${RUNNER_PORT}`], + { + T3CODE_RUNNER_THREAD_ID: threadId, + T3CODE_RUNNER_CHECKOUT: checkout, + T3CODE_RUNNER_TOKEN: runnerToken, + T3CODE_RUNNER_DRIVERS: "claudeAgent", + }, + ); + await waitFor("runner to listen", () => listening(RUNNER_PORT)); + log(`runner listening in ${Date.now() - started} ms`); +} + +let hubToken = ""; +async function startHub() { + const started = Date.now(); + const hubDir = NodePath.join(root, "hub"); + start("hub", hubDir, ["--base-dir", hubDir, "--port", `${HUB_PORT}`], { + T3CODE_SERVER_MODE: "hub", + T3CODE_RUNNER_URL: `ws://127.0.0.1:${RUNNER_PORT}/runner/ws`, + T3CODE_RUNNER_TOKEN: runnerToken, + T3CODE_HUB_CHECKOUT_ROOT: checkoutRoot, + T3CODE_HUB_PUBLIC_URL: HUB, + }); + await waitFor("hub to listen", () => listening(HUB_PORT)); + if (!hubToken) { + hubToken = childProcess + .execFileSync( + process.execPath, + [BIN, "auth", "session", "issue", "--base-dir", hubDir, "--token-only"], + { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }, + ) + .trim(); + } + await waitFor( + "hub API", + async () => (await http("GET", "/api/orchestration/shell", undefined, 2_000)) !== undefined, + ); + log(`hub ready in ${Date.now() - started} ms`); +} + +// ── Hub client ──────────────────────────────────────────────────────────── + +async function http(method, route, body, timeoutMs = 60_000) { + const response = await fetch(`${HUB}${route}`, { + method, + // A request that reaches a server between listen and route registration + // is never answered; bound every request so probes retry instead. + signal: AbortSignal.timeout(timeoutMs), + headers: { + authorization: `Bearer ${hubToken}`, + ...(body !== undefined ? { "content-type": "application/json" } : {}), + }, + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), + }); + const text = await response.text(); + if (!response.ok) + throw new Error(`${method} ${route} -> ${response.status}: ${text.slice(0, 400)}`); + return text.length > 0 ? JSON.parse(text) : null; +} + +const dispatch = (command) => http("POST", "/api/orchestration/dispatch", command); +const thread = async () => (await http("GET", `/api/orchestration/threads/${threadId}`)).thread; + +/** One RPC over the hub's /ws endpoint (Effect RPC JSON wire format). */ +async function rpc(tag, payload) { + const { ticket } = await http("POST", "/api/auth/websocket-ticket"); + const socket = new WebSocket( + `${HUB.replace(/^http/, "ws")}/ws?wsTicket=${encodeURIComponent(ticket)}`, + ); + return await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`rpc ${tag} timed out`)), 60_000); + socket.addEventListener("open", () => + socket.send(JSON.stringify({ _tag: "Request", id: "1", tag, payload, headers: [] })), + ); + socket.addEventListener("message", (message) => { + const decoded = JSON.parse(String(message.data)); + for (const entry of Array.isArray(decoded) ? decoded : [decoded]) { + if (entry._tag === "Ping") socket.send(JSON.stringify({ _tag: "Pong" })); + if (entry._tag === "Exit" && String(entry.requestId) === "1") { + clearTimeout(timer); + socket.close(); + if (entry.exit._tag === "Success") resolve(entry.exit.value); + else reject(new Error(`rpc ${tag} failed: ${JSON.stringify(entry.exit).slice(0, 600)}`)); + } + } + }); + socket.addEventListener("error", () => reject(new Error(`rpc ${tag} socket error`))); + }); +} + +/** Starts a turn, approves every request, and waits until it settles. */ +async function runTurn(text, timeoutMs = 240_000) { + const before = await thread(); + const seen = new Set(before.activities.map((activity) => activity.id)); + const approved = new Set(); + const started = Date.now(); + await dispatch({ + type: "thread.turn.start", + commandId: id(), + threadId, + message: { messageId: `msg-${id()}`, role: "user", text, attachments: [] }, + modelSelection: MODEL, + runtimeMode: "approval-required", + interactionMode: "default", + createdAt: now(), + }); + log(`turn started: ${JSON.stringify(text)}`); + return await waitFor( + "turn to settle", + async () => { + const current = await thread(); + for (const activity of current.activities) { + if (seen.has(activity.id)) continue; + seen.add(activity.id); + log(` activity ${activity.kind}: ${activity.summary}`); + const requestId = activity.payload?.requestId; + if (activity.kind === "approval.requested" && requestId && !approved.has(requestId)) { + approved.add(requestId); + const t0 = Date.now(); + await dispatch({ + type: "thread.approval.respond", + commandId: id(), + threadId, + requestId, + decision: "accept", + createdAt: now(), + }); + log(` approved ${requestId} in ${Date.now() - t0} ms`); + } + } + const latest = current.latestTurn; + const settled = + latest && + latest.turnId !== before.latestTurn?.turnId && + latest.state !== "running" && + (current.session?.activeTurnId ?? null) === null; + if (!settled) return undefined; + log(`turn ${latest.turnId} ${latest.state} after ${Date.now() - started} ms`); + return current; + }, + timeoutMs, + ); +} + +// ── Scenario ────────────────────────────────────────────────────────────── + +async function main() { + NodeFS.mkdirSync(logs, { recursive: true }); + NodeFS.mkdirSync(checkout, { recursive: true }); + for (const dir of ["runner", "hub"]) { + NodeFS.mkdirSync(NodePath.join(root, dir, "userdata"), { recursive: true }); + NodeFS.writeFileSync( + NodePath.join(root, dir, "userdata", "settings.json"), + // Titles are generated on the thread's runner with the same cheap model. + JSON.stringify({ enableProviderUpdateChecks: false, textGenerationModelSelection: MODEL }), + ); + } + const git = (...gitArgs) => + NodeChildProcess.execFileSync( + "git", + ["-c", "user.email=e2e@example.com", "-c", "user.name=e2e", ...gitArgs], + { + cwd: checkout, + encoding: "utf8", + }, + ); + git("init", "--initial-branch=main"); + NodeFS.writeFileSync(NodePath.join(checkout, "notes.txt"), "hello from the runner\n"); + git("add", "."); + git("commit", "-m", "initial"); + log(`workspace ${root}`); + + await startRunner(); + await startHub(); + + // a. The hub assigns the per-thread checkout NodePath. + await dispatch({ + type: "project.create", + commandId: id(), + projectId, + title: "Thread machines e2e", + workspaceRoot: "/anywhere/the/client/says", + defaultModelSelection: MODEL, + createdAt: now(), + }); + await dispatch({ + type: "thread.create", + commandId: id(), + threadId, + projectId, + title: "New thread", + modelSelection: MODEL, + runtimeMode: "approval-required", + interactionMode: "default", + branch: null, + worktreePath: null, + createdAt: now(), + }); + const created = await thread(); + check("thread checkout path", created.worktreePath === checkout, created.worktreePath); + const shell = await http("GET", "/api/orchestration/shell"); + const project = (shell.projects ?? shell.snapshot?.projects ?? []).find( + (entry) => entry.id === projectId, + ); + check( + "virtual project root", + project?.workspaceRoot === `/workspace/p/${projectId}`, + project?.workspaceRoot, + ); + + // b. A turn edits the runner's checkout, with an approval round trip. + const first = await runTurn( + "Append exactly one line, 'edited by the runner', to notes.txt using your file editing tool. Then reply with just: done", + ); + const notes = NodeFS.readFileSync(NodePath.join(checkout, "notes.txt"), "utf8"); + check("turn 1 edited the runner checkout", notes.includes("edited by the runner")); + const checkpoint = first.checkpoints.find((entry) => entry.checkpointTurnCount === 1); + check( + "turn 1 checkpoint captured", + checkpoint?.status === "ready" && checkpoint.files.some((file) => file.path === "notes.txt"), + JSON.stringify(checkpoint?.files ?? []), + ); + + // c. The diff through the hub. + const t0 = Date.now(); + const diff = await rpc("orchestration.getTurnDiff", { + threadId, + fromTurnCount: 0, + toTurnCount: 1, + }); + check( + "turn diff through the hub", + diff.diff.includes("+edited by the runner"), + `${Date.now() - t0} ms`, + ); + + // d. With the runner stopped, the hub still serves the diff and git status. + await stop("runner"); + const t1 = Date.now(); + const cached = await rpc("orchestration.getTurnDiff", { + threadId, + fromTurnCount: 0, + toTurnCount: 1, + }); + check("turn diff with the runner stopped", cached.diff === diff.diff, `${Date.now() - t1} ms`); + + // e. A new runner boot; the next turn resumes the provider session. + await startRunner(); + const second = await runTurn( + "Which line did you add to notes.txt earlier? Reply with only that line.", + ); + const answer = second.messages.findLast((message) => message.role === "assistant")?.text ?? ""; + check( + "turn 2 resumed with memory of turn 1", + /edited by the runner/i.test(answer), + JSON.stringify(answer.slice(0, 120)), + ); + const runnerLog = NodeFS.readFileSync(NodePath.join(logs, "runner.log"), "utf8"); + check( + "runner resumed from the hub's resume cursor", + /runner rpc startSession[\s\S]{0,200}resume: true/.test(runnerLog), + ); + + // f. A hub restart keeps history and diffs. + const beforeRestart = await thread(); + await stop("hub"); + await startHub(); + const afterRestart = await thread(); + check( + "hub restart keeps history", + afterRestart.messages.length === beforeRestart.messages.length && + afterRestart.checkpoints.length === beforeRestart.checkpoints.length, + `${afterRestart.messages.length} messages, ${afterRestart.checkpoints.length} checkpoints`, + ); + const afterDiff = await rpc("orchestration.getTurnDiff", { + threadId, + fromTurnCount: 0, + toTurnCount: 1, + }); + check("turn diff after hub restart", afterDiff.diff === diff.diff); + check( + "title generated on the runner", + afterRestart.title !== "New thread", + JSON.stringify(afterRestart.title), + ); +} + +let failed = false; +try { + await main(); +} catch (error) { + failed = true; + log("ERROR", error?.stack ?? error); +} finally { + await stop("hub"); + await stop("runner"); + const failures = results.filter((result) => !result.ok); + log(`${results.length - failures.length}/${results.length} checks passed; logs in ${logs}`); + if (!KEEP && !failed && failures.length === 0) + NodeFS.rmSync(root, { recursive: true, force: true }); + process.exitCode = failed || failures.length > 0 ? 1 : 0; +} From 04b1dae6f04f84ab997a5e48933bba597c6f38c7 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 23:39:08 +0000 Subject: [PATCH 23/58] docs: document the thread-machine runner protocol, wake semantics and delivery Record the runner RPC list and version negotiation, the machine directory behavior (polling, timeouts, idle and reconnect), which calls wake a machine, durable delivery with turn-safe cursors and deterministic ingestion ids, boot reconciliation, the hub caches, thread bootstrap activities, composition, hub-mode gaps, the new hub variables (T3CODE_HUB_PUBLIC_URL, T3CODE_HUB_CHECKOUT_ROOT, optional database URL) and runner variables, migration 050, and how to test. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/internals/thread-machines.md | 230 ++++++++++++++++++++++++++---- 1 file changed, 202 insertions(+), 28 deletions(-) diff --git a/docs/internals/thread-machines.md b/docs/internals/thread-machines.md index 04ada5089536..5f8160fce8b6 100644 --- a/docs/internals/thread-machines.md +++ b/docs/internals/thread-machines.md @@ -24,15 +24,19 @@ checkout path. A runner never persists orchestration state. ### Hub configuration -| Variable | Meaning | -| ------------------------------- | ------------------------------------------------------------------------------ | -| `T3CODE_SERVER_MODE=hub` | Enables hub mode | -| `T3CODE_HUB_DATABASE_URL` | Postgres URL for the hub schema (a role without `BYPASSRLS`) | -| `T3CODE_HUB_DATABASE_ADMIN_URL` | Optional; used only to apply hub migrations. Defaults to the runtime URL | -| `T3CODE_HUB_TENANT_ID` | The Aldo user ID whose rows this process owns. One hub process serves one user | -| `T3CODE_HUB_SECRET_KEY` | Base64 32-byte key encrypting per-user secrets at rest in Postgres | -| `T3CODE_HUB_MACHINES_URL` | Base URL of the machine directory (served by the platform host process) | -| `T3CODE_HUB_MACHINES_TOKEN` | Bearer for the machine directory | +| Variable | Meaning | +| ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `T3CODE_SERVER_MODE=hub` | Enables hub mode | +| `T3CODE_HUB_DATABASE_URL` | Postgres URL for the hub schema (a role without `BYPASSRLS`). Without it the hub persists to its SQLite state database, which only tests and development do | +| `T3CODE_HUB_DATABASE_ADMIN_URL` | Optional; used only to apply hub migrations. Defaults to the runtime URL | +| `T3CODE_HUB_TENANT_ID` | The Aldo user ID whose rows this process owns. One hub process serves one user. Required with a database URL | +| `T3CODE_HUB_SECRET_KEY` | Base64 32-byte key encrypting per-user secrets at rest in Postgres. Required with a database URL | +| `T3CODE_HUB_MACHINES_URL` | Base URL of the machine directory (served by the platform host process) | +| `T3CODE_HUB_MACHINES_TOKEN` | Bearer for the machine directory | +| `T3CODE_HUB_PUBLIC_URL` | Base URL at which thread machines reach the hub. Runners are given `/mcp` as the `t3-code` MCP endpoint; without it only a runner on the hub's host can reach MCP | +| `T3CODE_HUB_CHECKOUT_ROOT` | Root of thread checkouts (`/workspace/t`). Hub and machines must agree; only tests and development change it | +| `T3CODE_RUNNER_URL` | Development only: a single `t3 runner` WebSocket URL used instead of the machine directory (static directory, always `running`) | +| `T3CODE_RUNNER_TOKEN` | Development only: the bearer presented to that runner | The existing managed variables (`T3CODE_MANAGED_DEVPC`, `WORKSPACE_GATEWAY_TOKEN`, port, base dir) keep their meaning. The base dir holds only disposable caches and logs in hub @@ -42,14 +46,18 @@ mode; losing it loses nothing. The runner is `t3 runner` with the usual base dir and port flags plus: -| Variable | Meaning | -| ------------------------- | -------------------------------------- | -| `T3CODE_RUNNER_TOKEN` | Bearer the hub must present | -| `T3CODE_RUNNER_THREAD_ID` | The single thread this machine serves | -| `T3CODE_RUNNER_CHECKOUT` | Absolute checkout path for that thread | +| Variable | Meaning | +| ------------------------- | -------------------------------------------------------------------------------------- | +| `T3CODE_RUNNER_TOKEN` | Bearer the hub must present (`Authorization: Bearer …` on the WebSocket upgrade) | +| `T3CODE_RUNNER_THREAD_ID` | The single thread this machine serves; required | +| `T3CODE_RUNNER_CHECKOUT` | Absolute checkout path for that thread; required. It is also the runner's server `cwd` | +| `T3CODE_RUNNER_DRIVERS` | Optional comma-separated driver kinds to host (development); every built-in by default | The runner listens on loopback. The platform exposes it to the hub through its own -authenticated tunnel; the runner never accepts other threads' calls. +authenticated tunnel. Every call is checked against the runner's binding: thread-keyed +calls must name its thread and `cwd`-keyed calls must stay inside its checkout, or they +fail with the RPC's own error type. The runner keeps its provider settings, the outbox +(`/runner/outbox.sqlite`) and provider session files on the machine's disk. ## Checkout paths and routing @@ -59,12 +67,14 @@ In hub mode every thread gets its own checkout path on its own machine: /workspace/t/ ``` -Thread creation in hub mode sets `worktreePath` to that path, and projects get a virtual -`workspaceRoot` of `/workspace/p/` that nothing reads. Because a checkout path -names exactly one thread, every existing `cwd`-keyed call (checkpoints, VCS, workspace -files, review) routes to that thread's runner without contract changes. Calls whose `cwd` -does not parse as a thread checkout fail with a typed error; the hub never falls back to its -local filesystem. +Client `thread.create` commands and bootstrap `createThread` get this `worktreePath`, and +`project.create` gets a virtual `workspaceRoot` of `/workspace/p/` that nothing +reads (a `project.meta.update` cannot move it). Thread ids are URL-encoded into one path +segment. Because a checkout path names exactly one thread, every existing `cwd`-keyed call +(checkpoints, VCS, workspace files, review) routes to that thread's runner without contract +changes. Calls whose `cwd` is not inside a thread checkout fail with +`ThreadMachineUnavailableError { reason: "not-a-thread-checkout" }`; the hub never falls +back to its local filesystem. ## Machine directory @@ -97,14 +107,163 @@ the hub reconciles sessions and turns that were active on the previous boot. `id the platform no session or turn needs the machine; the platform decides when to pause or save it. `release` retires the machine when the thread is archived or deleted. +Hub behavior (`hub/MachineDirectory.ts`, `hub/RunnerConnectionPool.ts`): + +- A waking call posts `wake: true` with the thread's project, the project's repository + identity (`RepositoryIdentityResolver`) and branch, then polls `GET` every 2 s while the + state is `preparing`, `starting`, `paused`, `saved` or `none`, for at most 5 minutes + (`wake-timeout`). Requests time out after 30 s (`directory`). +- A non-waking call uses `GET`; anything but `running` fails with + `ThreadMachineUnavailableError { reason: "asleep", state }`. +- `failed` fails with reason `failed`; `failed` and `none` also settle the thread's live + sessions (see crash reconciliation). +- One connection per thread is shared by concurrent callers. The runner token is sent as + a bearer header; a token is only needed for the handshake, so reconnects re-resolve it. +- A connection with no calls, open streams or running turn for 10 minutes is closed and + the directory is told `idle`. A connection lost while a turn runs reconnects with + backoff (250 ms to 15 s) without waking the machine. +- The runner WebSocket URL may omit its path; `/runner/ws` is assumed. + ## Runner protocol -`packages/contracts/src/runner.ts` defines `RunnerRpcGroup`, versioned independently of -the client protocol. It carries provider session control, the checkpoint group, routed -workspace, VCS, git, and terminal services, text generation, and the event stream. -Runners persist provider events in a SQLite outbox before acknowledging them; the hub -saves its per-machine cursor in the same transaction as the projection writes and -deduplicates by `(threadId, runner sequence)`. +`packages/contracts/src/runner.ts` (`@t3tools/contracts/runner`) defines `RunnerRpcGroup`, +versioned independently of the client protocol: `RUNNER_PROTOCOL_VERSION = 1`, +`RUNNER_MIN_PROTOCOL_VERSION = 1`. `runner.hello` sends the hub's range and the thread; a +runner whose range does not overlap refuses with `RunnerProtocolMismatchError`, and one +bound to another thread with `RunnerThreadMismatchError`. The reply carries the negotiated +version, `runnerId` (the outbox identity, stable across restarts), `bootId` (per process), +the outbox head, ack and first retained sequence, and the hosted provider instances. +Compatible changes add RPCs and bump the version; a hub checks `hello.protocolVersion` +before using newer calls. + +Payloads and results are client contract schemas. Contract errors (VCS, git, terminal, +review, text generation) cross the wire as themselves; server-internal error unions +(provider adapter, workspace entries and files) travel inside `RunnerRemoteError`, encoded +with the server's schema and decoded back on the hub. + +| Group | RPCs | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Handshake | `runner.hello` | +| Provider | `runner.provider.{startSession, sendTurn, interruptTurn, respondToRequest, respondToUserInput, stopSession, listSessions, readThread, rollbackThread, getCapabilities}` | +| Text | `runner.text.{generateThreadTitle, generateBranchName}` | +| Events | `runner.events.subscribe` (stream), `runner.events.ack` | +| Checkout | `runner.checkout.prepare` | +| Checkpoints | `runner.checkpoint.{isGitRepository, capture, hasRef, restore, diff, deleteRefs}` | +| Workspace | `runner.workspace.{listEntries, searchEntries, searchContents, readFile, writeFile, refreshIndex}` | +| VCS and git | `runner.vcs.{status, localStatus, remoteStatus, refreshStatus, refreshLocalStatus, streamStatus (stream), init}`, `runner.git.{pull, runStackedAction (stream), resolvePullRequest, preparePullRequestThread, listRefs, createWorktree, removeWorktree, pruneWorktrees, createRef, switchRef, renameBranch, fetchRemote, remoteExists, resolveRemoteTrackingCommit}` | +| Review | `runner.review.{getDiffPreview, getDiffFileContents}` | +| Terminals | `runner.terminal.{open, attach (stream), write, resize, clear, restart, close, events (stream), metadata (stream)}` | + +`startSession` carries the hub-minted MCP credential (`mcp`: environment, thread, +provider session, instance, endpoint, authorization header); the endpoint is rewritten to +`T3CODE_HUB_PUBLIC_URL/mcp` when set. `sendTurn` and text generation carry attachment +bytes (`RunnerAttachmentFile`: the attachment, its hub path, base64 bytes); the runner +writes them where its adapters read attachments and replaces each hub path in the prompt +with its own. Commit and PR text are generated on the runner inside git actions. + +`runner.checkout.prepare` is idempotent: a missing checkout is cloned (`git clone --origin +origin`, with the machine's own git configuration and credential helper) or, without a +repository, initialized; an existing one is never reset. When `branch` is not checked out +it is switched to if it exists locally, else created from `origin/` (pushed from an +earlier machine), else from `baseRef` (remote-tracking first), else `HEAD`. Network access +happens only when cloning or creating a branch. + +## Wake semantics + +Whether a call may resume or recreate a sleeping machine is decided per call: + +| Wakes the machine | Never wakes it | +| ----------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| Provider session start, `sendTurn` (turn start), approval and user-input responses, interrupting an active turn, rollback (revert) and `readThread` | `hasSession` / `listSessions` (answered by the hub; a session on a sleeping machine is resumable), `stopSession`, provider capability refresh | +| Thread bootstrap and checkout preparation before each turn, setup scripts | Git status: `getStatus`, `streamStatus`, refreshes (hub cache; a connected runner refreshes it), `listRefs`, invalidations | +| Terminal `open`, `attach`, `restart` | Terminal `write`, `resize`, `clear` (fail with `TerminalNotRunningError`), `close` (no-op), terminal events and metadata | +| File writes; `vcs.init`; git actions (pull, commit/push/PR stacked actions, refs, worktrees, branch rename, PR resolve and thread preparation, fetch) | File list, search, content search and read; review diff preview and file contents; workspace index refresh | +| Title and branch-name generation (only issued at turn start) | Turn and full-thread diffs that were captured (hub cache); a diff never captured wakes once and is then cached | + +A read that needs a running machine fails with its service's own error type whose cause +is `ThreadMachineUnavailableError { reason: "asleep", state }`, which clients can render as +a resumable state. Clients should offer an explicit action (a turn, a terminal) to wake. + +## Durable delivery and crash reconciliation + +Runner (`runner/RunnerOutbox.ts`): every provider runtime event is stamped with its +instance id and committed to `runner_outbox` (SQLite, WAL, `synchronous = FULL`, so each +commit is fsynced) with a strictly increasing `sequence` before any subscriber sees it. +Sequences never repeat for an outbox (`runnerId`) across restarts. An adapter event id that +repeats among retained rows is suffixed with its sequence. `subscribe(after)` replays +retained events then tails live ones without gaps or overlap; `ack(n)` deletes through `n`. +More than 200,000 unacknowledged events drop the oldest and advance +`firstRetainedSequence`; a hub behind it logs the gap (resynchronizing from `readThread` is +future work). On graceful shutdown the runner stops its sessions while its event pumps +still run, so exit events reach the outbox. + +Hub (`hub/RunnerEventDelivery.ts`): + +- Delivery starts after server activation, so ingestion is subscribed first. Each + connection subscribes from the thread's cursor, drops any `(thread, sequence)` already + delivered, and resubscribes if the stream fails on a live connection. +- The cursor (`RunnerCursorStore`: outbox id, last reconciled boot, sequence) advances only + to a turn-safe point: a sequence at which the thread had no open turn. Ingestion buffers + assistant text and plans in memory during a turn, so after a hub crash the runner replays + the open turn from its start. +- Replays are idempotent: in hub mode ingestion derives command ids from the runtime event + id (`provider:::`), so already-committed commands are answered from their + receipts instead of applied twice. +- A cursor is written once the safe point is 250 ms old and ingestion and the checkpoint + reactor have drained, then acknowledged to the runner. Acknowledgement only lets the + runner compact. +- Boot reconciliation: when a runner reports a boot other than the one that hosted a + thread's session, the hub asks it (after the backlog is delivered) whether it still hosts + the session. If not, the hub publishes an interrupted `turn.completed` (when a turn was + active) and a `session.exited`, with deterministic event ids, and forgets the session. The + same happens when the directory reports the machine `failed` or `none`. The next turn + restarts the session from the persisted resume cursor. A `sendTurn` that finds its session + lost while the machine slept restarts it the same way before sending. +- At hub startup delivery resumes, without waking, from every thread with a live remote + session whose machine is running. + +Session directory (`hub/RemoteSessionRegistry.ts`): the remote provider adapter answers +`hasSession` and `listSessions` from a registry of sessions it started, their runner boot +and active turn, updated from delivered events and seeded after a restart from the persisted +bindings that were still active. Hub shutdown leaves remote sessions running and their +bindings live (`ProviderAdapterCapabilities.sessionsOutliveServer`). + +## Hub caches that keep machines asleep + +- Per-turn diffs (`CheckpointTurnDiffStore`): after capturing checkpoint `n` the hub stores + the `n-1 → n` and `0 → n` patches in both whitespace modes while the machine is still + connected; any diff computed through the runner is stored too. Recapturing, restoring + past, or deleting a checkpoint invalidates every diff that read it. +- Git status (`ThreadVcsStatusStore` + `HubVcsStatusCache`): a connected runner streams its + checkout's status into the cache; subscribers get the cached snapshot first. Automatic + settlement reads a branch's pull-request state from the cache. + +## Thread bootstrap + +Bootstrap in hub mode (`hub/HubThreadCheckouts.ts`, used by `CommandDispatcher`) creates the +thread with its checkout path, ensures and wakes its machine with the project's repository +identity and the branch, and calls `runner.checkout.prepare`, then runs the setup script in a +routed terminal. Progress and failures are thread activities: `thread-machine.starting` +("Starting machine", while waking), `thread-machine.checkout.preparing` ("Preparing +checkout") and `thread-machine.failed` (tone `error`). A failed bootstrap deletes the new +thread as in standalone mode. Before every turn the checkout is prepared again (skipped when +the same runner boot already did). + +## Composition + +`server.ts` builds one runtime for every mode from the `ServerModeLayers` record; standalone +uses the local layers and a hub uses `hub/HubLayers.ts` for the checkout-bound groups +(checkpointing, git, VCS, terminals, workspace, repository identity, provider instances) +plus hub-only infrastructure and background work. Shared code sees hub mode only through +the `serverModeHooks.ts` references (deterministic ingestion ids, `HubThreadCheckouts`) and +`CheckoutGitProbe`, whose defaults are standalone behavior. `t3 runner` is composed in +`runner/RunnerServer.ts` without orchestration, projections or the client API. + +Not available on a hub (typed errors, never the hub's disk): filesystem browsing for the +project picker, repository clone and publish, and workspace-file, media and project-favicon +asset URLs. Still running where the hub runs, to move to runners or the credential service: +previews, provider sign-in (auth connector), provider maintenance, usage scanning, workflow +scripts, and pull-request listing (which needs provider credentials). ## What the hub persists @@ -112,4 +271,19 @@ Everything a standalone server keeps in its state directory moves to Postgres, s `user_id` with row-level security as a backstop: orchestration events, projections, command receipts, provider session runtime, auth sessions and pairing links, settings, keybindings, secrets (encrypted with `T3CODE_HUB_SECRET_KEY`), the environment ID, attachments, per-turn -diffs, repository identity, and the last reported VCS status per thread. +diffs (`hub_checkpoint_turn_diffs`), runner cursors (`hub_runner_cursors`), repository +identity, and the last reported VCS status per thread (`hub_thread_vcs_status`). The three +thread-machine tables are hub migration 050 +(`persistence/Postgres/migrations/050_HubThreadMachineState.ts`). + +## Testing + +- Unit: `apps/server/src/hub/*.test.ts` (wake semantics with a fake directory and fake + runner, delivery and reconciliation, routed services), + `apps/server/src/runner/*.test.ts` (outbox, checkout), + `packages/contracts/src/runner.test.ts`. +- Loopback integration: `apps/server/integration/hubRunnerLoopback.integration.test.ts` runs + `ProviderService` and the reactors on a hub against the real runner handlers. +- End to end with a real provider: + `node apps/server/scripts/thread-machines-e2e.mjs [--keep] [--model ]` starts a + runner (4422) and a hub (4421) on loopback and drives the hub's public API. From 0a3c72fa2831c2c62a908701cd69aa4165cb0344 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 23:46:14 +0000 Subject: [PATCH 24/58] fix(server): keep restarted sessions and record the prepared branch in hub bootstrap The remote session registry ignores a session exit that predates the current session, so a provider restart's late exit event cannot make the hub forget the new session. Hub bootstrap always records the checkout path and the branch the runner actually checked out. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/hub/RemoteSessionRegistry.ts | 8 ++++++- .../src/orchestration/CommandDispatcher.ts | 22 +++++++++---------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/apps/server/src/hub/RemoteSessionRegistry.ts b/apps/server/src/hub/RemoteSessionRegistry.ts index 6969e37350f9..6f33f4fbbfc7 100644 --- a/apps/server/src/hub/RemoteSessionRegistry.ts +++ b/apps/server/src/hub/RemoteSessionRegistry.ts @@ -159,7 +159,13 @@ export const make = Effect.gen(function* () { case "session.exited": return Effect.sync(() => { const current = records.get(event.threadId); - if (current && current.instanceId === event.providerInstanceId) { + // A restart emits the old session's exit before or while the new + // one starts; only an exit at or after the current session ends it. + if ( + current && + current.instanceId === event.providerInstanceId && + event.createdAt >= current.session.createdAt + ) { records.delete(event.threadId); } }); diff --git a/apps/server/src/orchestration/CommandDispatcher.ts b/apps/server/src/orchestration/CommandDispatcher.ts index a554e8e25942..883693af28bc 100644 --- a/apps/server/src/orchestration/CommandDispatcher.ts +++ b/apps/server/src/orchestration/CommandDispatcher.ts @@ -339,19 +339,17 @@ export const makeOrchestrationCommandDispatcher = Effect.gen(function* () { branch, baseRef: bootstrap?.prepareWorktree?.baseBranch ?? null, }); - if (prepared.worktreePath !== targetWorktreePath || prepared.branch !== branch) { - yield* orchestrationEngine.dispatch( - { - type: "thread.meta.update", - commandId: yield* serverCommandId("bootstrap-thread-meta-update"), - threadId: command.threadId, - branch: prepared.branch, - worktreePath: prepared.worktreePath, - }, - options, - ); - } targetWorktreePath = prepared.worktreePath; + yield* orchestrationEngine.dispatch( + { + type: "thread.meta.update", + commandId: yield* serverCommandId("bootstrap-thread-meta-update"), + threadId: command.threadId, + branch: prepared.branch, + worktreePath: prepared.worktreePath, + }, + options, + ); yield* refreshGitStatus(targetWorktreePath); } else if (bootstrap?.prepareWorktree) { let worktreeBaseRef = bootstrap.prepareWorktree.baseBranch; From 103ae6229fe93226b4027501621fe84b4ceaeea2 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 23:46:16 +0000 Subject: [PATCH 25/58] test(server): cover the machine directory contract, session registry, and hub bootstrap - The HTTP machine directory against a local server: paths, bearer, ensure body, and typed errors for error statuses and malformed replies. - RemoteSessionRegistry seeding from persisted bindings and turn tracking. - Loopback bootstrap through the normalizer and command dispatcher: the hub assigns the checkout path, the runner checks out the thread branch, and progress is recorded as thread activities. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../HubRunnerLoopbackHarness.integration.ts | 73 +++++++++- .../hubRunnerLoopback.integration.test.ts | 70 +++++++++ apps/server/src/hub/MachineDirectory.test.ts | 137 ++++++++++++++++++ .../src/hub/RemoteSessionRegistry.test.ts | 127 ++++++++++++++++ 4 files changed, 400 insertions(+), 7 deletions(-) create mode 100644 apps/server/src/hub/MachineDirectory.test.ts create mode 100644 apps/server/src/hub/RemoteSessionRegistry.test.ts diff --git a/apps/server/integration/HubRunnerLoopbackHarness.integration.ts b/apps/server/integration/HubRunnerLoopbackHarness.integration.ts index 00855c949ab8..9d9d734ecbe6 100644 --- a/apps/server/integration/HubRunnerLoopbackHarness.integration.ts +++ b/apps/server/integration/HubRunnerLoopbackHarness.integration.ts @@ -32,6 +32,10 @@ import { type ProviderApprovalDecision, type ServerProvider, } from "@t3tools/contracts"; +import { + type ClientOrchestrationCommand, + OrchestrationDispatchCommandError, +} from "@t3tools/contracts"; import { threadCheckoutPath } from "@t3tools/contracts/runner"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; @@ -40,6 +44,7 @@ import * as ManagedRuntime from "effect/ManagedRuntime"; import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; @@ -54,6 +59,8 @@ import * as RemoteSessionRegistry from "../src/hub/RemoteSessionRegistry.ts"; import { make as makePool, RunnerConnectionPool } from "../src/hub/RunnerConnectionPool.ts"; import * as RunnerEventDelivery from "../src/hub/RunnerEventDelivery.ts"; import { serveRunner } from "../src/hub/testUtils/runnerServer.ts"; +import { makeOrchestrationCommandDispatcher } from "../src/orchestration/CommandDispatcher.ts"; +import { normalizeDispatchCommand } from "../src/orchestration/Normalizer.ts"; import { CheckpointReactorLive } from "../src/orchestration/Layers/CheckpointReactor.ts"; import { OrchestrationEngineLive } from "../src/orchestration/Layers/OrchestrationEngine.ts"; import { OrchestrationReactorLive } from "../src/orchestration/Layers/OrchestrationReactor.ts"; @@ -94,6 +101,7 @@ import { ProviderInstanceRegistry } from "../src/provider/Services/ProviderInsta import { makeAdapterRegistryMock } from "../src/provider/testUtils/providerAdapterRegistryMock.ts"; import { makeProviderRegistryLayer } from "../src/provider/testUtils/providerRegistryMock.ts"; import * as AgentAwarenessRelay from "../src/relay/AgentAwarenessRelay.ts"; +import * as ProjectSetupScriptRunner from "../src/project/ProjectSetupScriptRunner.ts"; import * as ReviewService from "../src/review/ReviewService.ts"; import * as RunnerCheckout from "../src/runner/RunnerCheckout.ts"; import { RunnerRpcHandlersLive } from "../src/runner/RunnerHandlers.ts"; @@ -103,6 +111,7 @@ import { type RunnerOutboxShape, } from "../src/runner/RunnerOutbox.ts"; import { RunnerEventPumpLive } from "../src/runner/RunnerServer.ts"; +import { ServerRuntimeStartup } from "../src/serverRuntimeStartup.ts"; import { ServerSettingsService } from "../src/serverSettings.ts"; import { AnalyticsService } from "../src/telemetry/AnalyticsService.ts"; import * as TerminalManager from "../src/terminal/Manager.ts"; @@ -121,6 +130,7 @@ import { } from "./TestProviderAdapter.integration.ts"; export const LOOPBACK_PROVIDER = ProviderDriverKind.make("codex"); +const isDispatchError = Schema.is(OrchestrationDispatchCommandError); export const LOOPBACK_INSTANCE_ID = defaultInstanceIdForDriver(LOOPBACK_PROVIDER); const git = (cwd: string, args: ReadonlyArray) => @@ -180,6 +190,15 @@ const instanceRegistryLayer = (adapter: TestProviderAdapterHarness["adapter"]) = }); }; +const localStatus = { + isRepo: true, + hasPrimaryRemote: false, + isDefaultRef: true, + refName: "main", + hasWorkingTreeChanges: false, + workingTree: { files: [], insertions: 0, deletions: 0 }, +}; + export interface LoopbackRunner { readonly url: string; readonly adapterHarness: TestProviderAdapterHarness; @@ -201,6 +220,10 @@ export interface HubRunnerLoopbackHarness { readonly delivery: () => RunnerEventDelivery.RunnerEventDelivery["Service"]; /** Disposes the hub runtime and starts a new one on the same database. */ readonly restartHub: Effect.Effect; + /** A client command through normalization and the command dispatcher (bootstrap included). */ + readonly dispatchClientCommand: ( + command: ClientOrchestrationCommand, + ) => Effect.Effect; readonly waitForThread: ( predicate: (thread: OrchestrationThread) => boolean, description: string, @@ -267,14 +290,14 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") Layer.mock(GitWorkflowService.GitWorkflowService)({}), Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({ streamStatus: () => Stream.never, - refreshLocalStatus: () => + refreshLocalStatus: () => Effect.succeed(localStatus), + refreshStatus: () => Effect.succeed({ - isRepo: true, - hasPrimaryRemote: false, - isDefaultRef: true, - refName: "main", - hasWorkingTreeChanges: false, - workingTree: { files: [], insertions: 0, deletions: 0 }, + ...localStatus, + hasUpstream: false, + aheadCount: 0, + behindCount: 0, + pr: null, }), }), Layer.mock(VcsProvisioningService.VcsProvisioningService)({}), @@ -374,6 +397,7 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") ), ProjectionCheckpointRepositoryLive, ProjectionPendingApprovalRepositoryLive, + OrchestrationCommandReceiptRepositoryLive, HubLayers.hubCheckpointStoreLayer, providerLayer, RuntimeReceiptBusTest, @@ -382,6 +406,7 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") Layer.provideMerge(ThreadPlanProgress.layer), ); const checkoutServices = Layer.mergeAll( + HubLayers.hubWorkspacePathsLayer, HubLayers.hubVcsStatusBroadcasterLayer, HubLayers.hubWorkspaceEntriesLayer, HubLayers.hubGitWorkflowServiceLayer, @@ -437,6 +462,7 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") readonly diffStore: CheckpointTurnDiffStore["Service"]; readonly delivery: RunnerEventDelivery.RunnerEventDelivery["Service"]; readonly approvals: ProjectionPendingApprovalRepository["Service"]; + readonly dispatchClientCommand: HubRunnerLoopbackHarness["dispatchClientCommand"]; } const startHub = Effect.gen(function* () { @@ -455,10 +481,42 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") ), ); const { reactor, ...rest } = services; + // The transport's dispatch path: startup is already ready, setup scripts + // are covered elsewhere, and thread deletion needs no fence here. + const dispatchStubs = Layer.mergeAll( + Layer.succeed(ServerRuntimeStartup, { + awaitCommandReady: Effect.void, + markHttpListening: Effect.void, + enqueueCommand: (effect) => effect, + }), + Layer.succeed(ProjectSetupScriptRunner.ProjectSetupScriptRunner, { + runForThread: () => Effect.succeed({ status: "no-script" as const }), + }), + Layer.succeed(ThreadDeletionReactor, { + start: () => Effect.void, + drainThrough: () => Effect.void, + }), + ); + const dispatchClientCommand: HubRunnerLoopbackHarness["dispatchClientCommand"] = (command) => + Effect.tryPromise({ + try: () => + runtime.runPromise( + Effect.gen(function* () { + const dispatch = yield* makeOrchestrationCommandDispatcher; + const normalized = yield* normalizeDispatchCommand(command); + yield* dispatch(normalized); + }).pipe(Effect.provide(dispatchStubs)), + ), + catch: (cause) => + isDispatchError(cause) + ? cause + : new OrchestrationDispatchCommandError({ message: String(cause), cause }), + }); const instance = { runtime, scope: yield* Scope.make("sequential"), ...rest, + dispatchClientCommand, } satisfies HubInstance; yield* Effect.promise(() => runtime.runPromise(reactor.start().pipe(Scope.provide(instance.scope))), @@ -495,6 +553,7 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") checkpointStore: () => hub.checkpointStore, diffStore: () => hub.diffStore, delivery: () => hub.delivery, + dispatchClientCommand: (command) => Effect.suspend(() => hub.dispatchClientCommand(command)), restartHub: Effect.gen(function* () { yield* stopHub(hub); hub = yield* startHub; diff --git a/apps/server/integration/hubRunnerLoopback.integration.test.ts b/apps/server/integration/hubRunnerLoopback.integration.test.ts index 602e1239f4d1..40c24cf03fb6 100644 --- a/apps/server/integration/hubRunnerLoopback.integration.test.ts +++ b/apps/server/integration/hubRunnerLoopback.integration.test.ts @@ -329,3 +329,73 @@ it.live("replays an open turn after a hub restart without duplicating anything", }), ), ); + +it.live("bootstraps a new thread on its machine: checkout path, branch, and progress", () => + withHarness((harness) => + Effect.gen(function* () { + yield* harness.engine().dispatch({ + type: "project.create", + commandId: CommandId.make("cmd-project-create"), + projectId: PROJECT_ID, + title: "Loopback", + workspaceRoot: projectVirtualRoot(PROJECT_ID), + defaultModelSelection: MODEL, + createdAt: at(0), + }); + yield* harness.runner().adapterHarness.queueTurnResponseForNextSession({ + events: [ + { type: "turn.started", ...fixture(harness, "evt-b1") }, + { type: "message.delta", ...fixture(harness, "evt-b2"), delta: "Bootstrapped.\n" }, + ], + }); + yield* harness.dispatchClientCommand({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-bootstrap-turn"), + threadId: harness.threadId, + message: { + messageId: MessageId.make("msg-bootstrap"), + role: "user", + text: "Hi", + attachments: [], + }, + runtimeMode: "approval-required", + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + bootstrap: { + createThread: { + projectId: PROJECT_ID, + title: "New thread", + modelSelection: MODEL, + runtimeMode: "approval-required", + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + branch: null, + worktreePath: "/somewhere/the/client/chose", + createdAt: at(1), + }, + prepareWorktree: { + projectCwd: projectVirtualRoot(PROJECT_ID), + baseBranch: "main", + branch: "t3/feature-x", + }, + }, + createdAt: at(2), + }); + const thread = yield* harness.waitForThread( + (value) => + value.latestTurn?.state === "completed" && + value.messages.some((message) => message.role === "assistant"), + "bootstrapped turn", + ); + assert.equal(thread.worktreePath, harness.checkout); + assert.equal(thread.branch, "t3/feature-x"); + assert.include( + thread.activities.map((activity) => activity.kind), + "thread-machine.checkout.preparing", + ); + const checkedOut = NodeFS.readFileSync( + NodePath.join(harness.checkout, ".git", "HEAD"), + "utf8", + ); + assert.equal(checkedOut.trim(), "ref: refs/heads/t3/feature-x"); + }), + ), +); diff --git a/apps/server/src/hub/MachineDirectory.test.ts b/apps/server/src/hub/MachineDirectory.test.ts new file mode 100644 index 000000000000..354b5eab93b4 --- /dev/null +++ b/apps/server/src/hub/MachineDirectory.test.ts @@ -0,0 +1,137 @@ +// @effect-diagnostics nodeBuiltinImport:off -- A plain HTTP server stands in for the platform. +import * as NodeHttp from "node:http"; + +import { it } from "@effect/vitest"; +import { ProjectId, ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import { FetchHttpClient } from "effect/unstable/http"; +import { describe, expect } from "vite-plus/test"; + +import { makeHttpMachineDirectory } from "./MachineDirectory.ts"; + +interface Recorded { + readonly method: string; + readonly url: string; + readonly authorization: string | undefined; + readonly body: unknown; +} + +/** A directory that answers from `respond` and records every request. */ +const serveDirectory = ( + respond: (request: Recorded) => { readonly status: number; readonly body?: unknown }, +) => + Effect.acquireRelease( + Effect.callback<{ readonly url: string; readonly requests: Array }>((resume) => { + const requests: Array = []; + const server = NodeHttp.createServer((request, response) => { + let raw = ""; + request.on("data", (chunk) => (raw += chunk)); + request.on("end", () => { + const recorded: Recorded = { + method: request.method ?? "", + url: request.url ?? "", + authorization: request.headers.authorization, + body: raw.length > 0 ? JSON.parse(raw) : undefined, + }; + requests.push(recorded); + const reply = respond(recorded); + response.writeHead(reply.status, { "content-type": "application/json" }); + response.end(reply.body === undefined ? "" : JSON.stringify(reply.body)); + }); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address() as { port: number }; + resume(Effect.succeed({ url: `http://127.0.0.1:${address.port}/api`, requests, server })); + }); + }).pipe(Effect.map((value) => value as typeof value & { server: NodeHttp.Server })), + (value) => Effect.sync(() => value.server.close()), + ); + +const threadId = ThreadId.make("thread/with space"); + +describe("HTTP machine directory", () => { + it.live("speaks the documented contract", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* serveDirectory((request) => + request.method === "GET" + ? { status: 200, body: { state: "paused", bootId: "boot-1", detail: null } } + : request.url.endsWith("/machine") + ? { + status: 200, + body: { + state: "running", + runner: { url: "wss://tunnel/runner", token: "short-lived", expiresAt: null }, + bootId: "boot-2", + }, + } + : { status: 204 }, + ); + const directory = yield* makeHttpMachineDirectory({ + baseUrl: `${server.url}/`, + token: "directory-token", + }); + + const status = yield* directory.status(threadId); + expect(status).toEqual({ state: "paused", bootId: "boot-1", detail: null }); + + const ensured = yield* directory.ensure(threadId, { + projectId: ProjectId.make("project-1"), + repository: { url: "https://github.com/acme/app.git", ref: "main" }, + branch: "t3/work", + checkout: "/workspace/t/thread%2Fwith%20space", + wake: true, + }); + expect(ensured.runner?.url).toBe("wss://tunnel/runner"); + yield* directory.idle(threadId); + yield* directory.release(threadId); + + expect( + server.requests.map((request) => [request.method, request.url, request.authorization]), + ).toEqual([ + ["GET", "/api/threads/thread%2Fwith%20space/machine", "Bearer directory-token"], + ["POST", "/api/threads/thread%2Fwith%20space/machine", "Bearer directory-token"], + ["POST", "/api/threads/thread%2Fwith%20space/machine/idle", "Bearer directory-token"], + ["POST", "/api/threads/thread%2Fwith%20space/machine/release", "Bearer directory-token"], + ]); + expect(server.requests[1]?.body).toEqual({ + projectId: "project-1", + repository: { url: "https://github.com/acme/app.git", ref: "main" }, + branch: "t3/work", + checkout: "/workspace/t/thread%2Fwith%20space", + wake: true, + }); + }).pipe(Effect.provide(FetchHttpClient.layer)), + ), + ); + + it.live("fails with typed errors for error statuses and malformed replies", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* serveDirectory((request) => + request.method === "GET" + ? { status: 503, body: { error: "capacity" } } + : { status: 200, body: { state: "sleeping" } }, + ); + const directory = yield* makeHttpMachineDirectory({ baseUrl: server.url, token: "t" }); + const unavailable = yield* directory.status(threadId).pipe(Effect.flip); + expect(unavailable).toMatchObject({ + _tag: "MachineDirectoryError", + operation: "status", + status: 503, + }); + expect(unavailable.detail).toContain("capacity"); + const malformed = yield* directory + .ensure(threadId, { + projectId: null, + repository: null, + branch: null, + checkout: "/workspace/t/x", + wake: true, + }) + .pipe(Effect.flip); + expect(malformed).toMatchObject({ _tag: "MachineDirectoryError", operation: "ensure" }); + }).pipe(Effect.provide(FetchHttpClient.layer)), + ), + ); +}); diff --git a/apps/server/src/hub/RemoteSessionRegistry.test.ts b/apps/server/src/hub/RemoteSessionRegistry.test.ts new file mode 100644 index 000000000000..850a36736189 --- /dev/null +++ b/apps/server/src/hub/RemoteSessionRegistry.test.ts @@ -0,0 +1,127 @@ +import { it } from "@effect/vitest"; +import { + EventId, + ProviderDriverKind, + ProviderInstanceId, + type ProviderRuntimeEvent, + ThreadId, + TurnId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import { describe, expect } from "vite-plus/test"; + +import { HubThreadMachineStateSqliteLive } from "../persistence/Layers/HubThreadMachineState.ts"; +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; +import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; +import { RunnerCursorStore } from "../persistence/Services/HubThreadMachineState.ts"; +import { make as makeRegistry } from "./RemoteSessionRegistry.ts"; + +const threadId = ThreadId.make("thread-registry"); +const provider = ProviderDriverKind.make("claudeAgent"); +const instanceId = ProviderInstanceId.make("claudeAgent"); + +const TestLayer = Layer.mergeAll( + HubThreadMachineStateSqliteLive, + ProviderSessionRuntime.layer, +).pipe(Layer.provideMerge(SqlitePersistenceMemory)); + +const lifecycleEvent = ( + type: "turn.started" | "turn.completed" | "session.exited", + createdAt: string, +): ProviderRuntimeEvent => + ({ + type, + eventId: EventId.make(`${type}-${createdAt}`), + provider, + providerInstanceId: instanceId, + threadId, + turnId: TurnId.make("turn-1"), + createdAt, + payload: type === "turn.completed" ? { state: "completed" } : {}, + }) as ProviderRuntimeEvent; + +describe("RemoteSessionRegistry", () => { + it.effect("seeds live sessions from persisted bindings with the last reconciled boot", () => + Effect.gen(function* () { + const rows = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + yield* rows.upsert({ + threadId, + providerName: provider, + providerInstanceId: instanceId, + adapterKey: provider, + runtimeMode: "full-access", + status: "running", + lastSeenAt: "2026-09-25T00:00:00.000Z", + resumeCursor: { resume: "session-1" }, + runtimePayload: { cwd: "/workspace/t/thread-registry", activeTurnId: "turn-1" }, + }); + yield* rows.upsert({ + threadId: ThreadId.make("thread-stopped"), + providerName: provider, + providerInstanceId: instanceId, + adapterKey: provider, + runtimeMode: "full-access", + status: "stopped", + lastSeenAt: "2026-09-25T00:00:00.000Z", + resumeCursor: null, + runtimePayload: null, + }); + yield* (yield* RunnerCursorStore).saveAll([ + { threadId, outboxId: "runner-1", bootId: "boot-7", ackedSequence: 3, updatedAt: "t" }, + ]); + + const registry = yield* makeRegistry; + const records = yield* registry.list; + expect(records).toHaveLength(1); + expect(records[0]).toMatchObject({ + threadId, + instanceId, + bootId: "boot-7", + session: { + status: "running", + cwd: "/workspace/t/thread-registry", + activeTurnId: "turn-1", + resumeCursor: { resume: "session-1" }, + }, + }); + }).pipe(Effect.provide(TestLayer)), + ); + + it.effect("tracks turns and ignores the exit of a session that was already replaced", () => + Effect.gen(function* () { + const registry = yield* makeRegistry; + yield* registry.upsert({ + threadId, + instanceId, + provider, + bootId: "boot-1", + session: { + provider, + providerInstanceId: instanceId, + status: "ready", + runtimeMode: "full-access", + threadId, + createdAt: "2026-09-25T00:00:10.000Z", + updatedAt: "2026-09-25T00:00:10.000Z", + }, + }); + yield* registry.applyEvent(lifecycleEvent("turn.started", "2026-09-25T00:00:11.000Z")); + expect(Option.getOrThrow(yield* registry.get(threadId)).session).toMatchObject({ + status: "running", + activeTurnId: "turn-1", + }); + yield* registry.applyEvent(lifecycleEvent("turn.completed", "2026-09-25T00:00:12.000Z")); + const settled = Option.getOrThrow(yield* registry.get(threadId)).session; + expect(settled.status).toBe("ready"); + expect(settled.activeTurnId).toBeUndefined(); + + // The previous session's exit, emitted before this session started. + yield* registry.applyEvent(lifecycleEvent("session.exited", "2026-09-25T00:00:09.000Z")); + expect(Option.isSome(yield* registry.get(threadId))).toBe(true); + yield* registry.applyEvent(lifecycleEvent("session.exited", "2026-09-25T00:00:13.000Z")); + expect(Option.isNone(yield* registry.get(threadId))).toBe(true); + }).pipe(Effect.provide(TestLayer)), + ); +}); From 31548b1a8d17f9c83f7ad6e2f33a2c5509e65ed6 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Fri, 25 Sep 2026 23:51:17 +0000 Subject: [PATCH 26/58] test(server): cover runner handler guards, attachments, MCP, and remote sessions Runner handlers materialize shipped attachments and rewrite prompt paths, install the hub's MCP session before the provider starts, and refuse calls for another thread or outside the checkout. The remote provider driver answers hasSession/listSessions from the hub registry without waking a machine and targets the hub's public MCP URL. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/hub/RemoteProviderDriver.test.ts | 158 +++++++++++ apps/server/src/runner/RunnerHandlers.test.ts | 256 ++++++++++++++++++ 2 files changed, 414 insertions(+) create mode 100644 apps/server/src/hub/RemoteProviderDriver.test.ts create mode 100644 apps/server/src/runner/RunnerHandlers.test.ts diff --git a/apps/server/src/hub/RemoteProviderDriver.test.ts b/apps/server/src/hub/RemoteProviderDriver.test.ts new file mode 100644 index 000000000000..a7b8842cf4b0 --- /dev/null +++ b/apps/server/src/hub/RemoteProviderDriver.test.ts @@ -0,0 +1,158 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { it } from "@effect/vitest"; +import { + EnvironmentId, + ProviderDriverKind, + ProviderInstanceId, + ThreadId, + TurnId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { afterEach, describe, expect } from "vite-plus/test"; + +import { resolveAttachmentPath } from "../attachmentStore.ts"; +import { ServerConfig } from "../config.ts"; +import * as McpProviderSession from "../mcp/McpProviderSession.ts"; +import { HubThreadMachineStateSqliteLive } from "../persistence/Layers/HubThreadMachineState.ts"; +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; +import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; +import { CodexDriver } from "../provider/Drivers/CodexDriver.ts"; +import { MachineDirectory, makeFakeMachineDirectory } from "./MachineDirectory.ts"; +import { + attachmentFilesForRunner, + makeRemoteProviderDriver, + mcpSessionForRunner, +} from "./RemoteProviderDriver.ts"; +import * as RemoteSessionRegistry from "./RemoteSessionRegistry.ts"; +import { make as makePool, RunnerConnectionPool } from "./RunnerConnectionPool.ts"; +import * as RunnerEventDelivery from "./RunnerEventDelivery.ts"; + +const threadId = ThreadId.make("thread-remote-driver"); +const provider = ProviderDriverKind.make("codex"); +const instanceId = ProviderInstanceId.make("codex"); + +const tempDirs: Array = []; +afterEach(() => { + for (const dir of tempDirs.splice(0)) NodeFS.rmSync(dir, { recursive: true, force: true }); +}); + +describe("remote provider driver helpers", () => { + it("points runners at the hub's public MCP endpoint when one is configured", () => { + McpProviderSession.setMcpProviderSession({ + environmentId: EnvironmentId.make("env-1"), + threadId, + providerSessionId: "session-1", + providerInstanceId: instanceId, + endpoint: "http://127.0.0.1:4421/mcp", + authorizationHeader: "Bearer token", + }); + expect(mcpSessionForRunner(threadId, "https://hub.example/u/1/")?.endpoint).toBe( + "https://hub.example/u/1/mcp", + ); + expect(mcpSessionForRunner(threadId, undefined)?.endpoint).toBe("http://127.0.0.1:4421/mcp"); + McpProviderSession.clearMcpProviderSession(threadId); + expect(mcpSessionForRunner(threadId, "https://hub.example")).toBeNull(); + }); + + it.effect("ships the bytes of stored attachments and skips missing ones", () => + Effect.gen(function* () { + const dir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "hub-attachments-test-")); + tempDirs.push(dir); + const stored = { + type: "file" as const, + id: `${threadId}-2e2e2e2e-0000-4000-8000-000000000000`, + name: "notes.md", + mimeType: "text/markdown", + sizeBytes: 3, + }; + const missing = { ...stored, id: `${threadId}-3f3f3f3f-0000-4000-8000-000000000000` }; + const storedPath = resolveAttachmentPath({ attachmentsDir: dir, attachment: stored })!; + NodeFS.mkdirSync(NodePath.dirname(storedPath), { recursive: true }); + NodeFS.writeFileSync(storedPath, "abc"); + const files = yield* attachmentFilesForRunner(dir, [stored, missing]); + expect(files).toEqual([ + { + attachment: stored, + hubPath: storedPath, + bytesBase64: Buffer.from("abc").toString("base64"), + }, + ]); + }), + ); +}); + +const StoresLive = Layer.mergeAll( + HubThreadMachineStateSqliteLive, + ProviderSessionRuntime.layer, +).pipe(Layer.provideMerge(SqlitePersistenceMemory), Layer.provideMerge(NodeServices.layer)); + +describe("remote provider adapter session answers", () => { + it.live("answers hasSession and listSessions from the hub and never wakes a machine", () => + Effect.scoped( + Effect.gen(function* () { + const fake = yield* makeFakeMachineDirectory({ + initial: [[threadId, { state: "paused" }]], + }); + const pool = yield* makePool({ idleCheckInterval: "1 hour" }).pipe( + Effect.provideService(MachineDirectory, fake.directory), + ); + const registry = yield* RemoteSessionRegistry.make; + const delivery = yield* RunnerEventDelivery.make.pipe( + Effect.provideService(RunnerConnectionPool, pool), + Effect.provideService(RemoteSessionRegistry.RemoteSessionRegistry, registry), + ); + const instance = yield* makeRemoteProviderDriver(CodexDriver) + .create({ + instanceId, + displayName: undefined, + environment: [], + enabled: true, + config: CodexDriver.defaultConfig(), + }) + .pipe( + Effect.provideService(RunnerConnectionPool, pool), + Effect.provideService(RunnerEventDelivery.RunnerEventDelivery, delivery), + Effect.provideService(RemoteSessionRegistry.RemoteSessionRegistry, registry), + Effect.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-remote-driver-" })), + ); + const adapter = instance.adapter; + expect(adapter.capabilities.sessionsOutliveServer).toBe(true); + expect(yield* adapter.hasSession(threadId)).toBe(false); + + yield* registry.upsert({ + threadId, + instanceId, + provider, + bootId: "boot-1", + session: { + provider, + providerInstanceId: instanceId, + status: "ready", + runtimeMode: "full-access", + threadId, + createdAt: "t", + updatedAt: "t", + }, + }); + // The machine is asleep, yet the session is resumable, so it exists. + expect(yield* adapter.hasSession(threadId)).toBe(true); + expect((yield* adapter.listSessions()).map((session) => session.threadId)).toEqual([ + threadId, + ]); + + // No active turn: interrupting does not wake the machine; stopping forgets the session. + yield* adapter.interruptTurn(threadId, TurnId.make("turn-1")); + yield* adapter.stopSession(threadId); + expect(yield* adapter.hasSession(threadId)).toBe(false); + yield* adapter.stopAll(); + expect(yield* fake.calls).toEqual([]); + }), + ).pipe(Effect.provide(StoresLive)), + ); +}); diff --git a/apps/server/src/runner/RunnerHandlers.test.ts b/apps/server/src/runner/RunnerHandlers.test.ts new file mode 100644 index 000000000000..d2d09c73c984 --- /dev/null +++ b/apps/server/src/runner/RunnerHandlers.test.ts @@ -0,0 +1,256 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { it } from "@effect/vitest"; +import { + EnvironmentId, + ProviderDriverKind, + ProviderInstanceId, + type ProviderSendTurnInput, + type ServerProvider, + ThreadId, + TurnId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import * as Stream from "effect/Stream"; +import { afterEach, describe, expect } from "vite-plus/test"; + +import { resolveAttachmentPath } from "../attachmentStore.ts"; +import * as CheckpointStore from "../checkpointing/CheckpointStore.ts"; +import { ServerConfig } from "../config.ts"; +import * as GitWorkflowService from "../git/GitWorkflowService.ts"; +import { MachineDirectory, makeFakeMachineDirectory } from "../hub/MachineDirectory.ts"; +import { make as makePool } from "../hub/RunnerConnectionPool.ts"; +import { serveRunner } from "../hub/testUtils/runnerServer.ts"; +import * as McpProviderSession from "../mcp/McpProviderSession.ts"; +import type { ProviderInstance } from "../provider/ProviderDriver.ts"; +import type { ProviderAdapterShape } from "../provider/Services/ProviderAdapter.ts"; +import { ProviderInstanceRegistry } from "../provider/Services/ProviderInstanceRegistry.ts"; +import * as ReviewService from "../review/ReviewService.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; +import * as TerminalManager from "../terminal/Manager.ts"; +import type { TextGenerationShape } from "../textGeneration/TextGeneration.ts"; +import * as VcsProvisioningService from "../vcs/VcsProvisioningService.ts"; +import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; +import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; +import * as WorkspaceFileSystem from "../workspace/WorkspaceFileSystem.ts"; +import * as RunnerCheckout from "./RunnerCheckout.ts"; +import { RunnerRpcHandlersLive } from "./RunnerHandlers.ts"; +import { makeWithOptions as makeOutbox, RunnerOutbox } from "./RunnerOutbox.ts"; + +const threadId = ThreadId.make("thread-handlers"); +const provider = ProviderDriverKind.make("claudeAgent"); +const instanceId = ProviderInstanceId.make("claudeAgent"); + +const tempDirs: Array = []; +afterEach(() => { + for (const dir of tempDirs.splice(0)) NodeFS.rmSync(dir, { recursive: true, force: true }); +}); + +/** A runner with a recording provider adapter, served on loopback, and a client pool. */ +const setup = Effect.gen(function* () { + const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "runner-handlers-test-")); + tempDirs.push(root); + const checkout = NodePath.join(root, "t", threadId); + NodeFS.mkdirSync(checkout, { recursive: true }); + const turns: Array = []; + const mcpAtStart: Array = []; + const events = yield* PubSub.unbounded(); + const adapter = { + provider, + capabilities: { sessionModelSwitch: "in-session" }, + startSession: (input) => + Effect.sync(() => { + mcpAtStart.push(McpProviderSession.readMcpProviderSession(input.threadId)); + return { + provider, + status: "ready", + runtimeMode: input.runtimeMode, + threadId: input.threadId, + createdAt: "t", + updatedAt: "t", + } as const; + }), + sendTurn: (input) => + Effect.sync(() => { + turns.push(input); + return { threadId: input.threadId, turnId: TurnId.make("turn-1") }; + }), + interruptTurn: () => Effect.void, + respondToRequest: () => Effect.void, + respondToUserInput: () => Effect.void, + stopSession: () => Effect.void, + listSessions: () => Effect.succeed([]), + hasSession: () => Effect.succeed(false), + readThread: () => Effect.succeed({ threadId, turns: [] }), + rollbackThread: () => Effect.succeed({ threadId, turns: [] }), + stopAll: () => Effect.void, + streamEvents: Stream.fromPubSub(events), + } satisfies ProviderAdapterShape; + const instance: ProviderInstance = { + instanceId, + driverKind: provider, + continuationIdentity: { driverKind: provider, continuationKey: "claudeAgent" }, + displayName: undefined, + enabled: true, + snapshot: { + maintenanceCapabilities: { provider, packageName: null, update: null }, + getSnapshot: Effect.succeed({} as ServerProvider), + refresh: Effect.succeed({} as ServerProvider), + streamChanges: Stream.empty, + }, + adapter, + textGeneration: {} as TextGenerationShape, + }; + const config = Layer.effect( + ServerConfig, + Effect.gen(function* () { + const base = yield* ServerConfig; + return { ...base, runnerThreadId: threadId, runnerCheckout: checkout }; + }), + ).pipe(Layer.provide(ServerConfig.layerTest(checkout, NodePath.join(root, "runner")))); + const runnerLayer = RunnerRpcHandlersLive.pipe( + Layer.provideMerge( + Layer.effect( + RunnerOutbox, + makeOutbox({ databasePath: NodePath.join(root, "runner", "outbox.sqlite") }), + ), + ), + Layer.provideMerge( + Layer.succeed(ProviderInstanceRegistry, { + getInstance: (id) => Effect.succeed(id === instanceId ? instance : undefined), + listInstances: Effect.succeed([instance]), + listUnavailable: Effect.succeed([]), + streamChanges: Stream.empty, + subscribeChanges: Effect.flatMap(PubSub.unbounded(), (pubsub) => + PubSub.subscribe(pubsub), + ), + }), + ), + Layer.provideMerge( + Layer.mergeAll( + Layer.mock(RunnerCheckout.RunnerCheckout)({}), + Layer.mock(CheckpointStore.CheckpointStore)({}), + Layer.mock(WorkspaceEntries.WorkspaceEntries)({}), + Layer.mock(WorkspaceFileSystem.WorkspaceFileSystem)({}), + Layer.mock(GitWorkflowService.GitWorkflowService)({}), + Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({}), + Layer.mock(VcsProvisioningService.VcsProvisioningService)({}), + Layer.mock(ReviewService.ReviewService)({}), + Layer.mock(TerminalManager.TerminalManager)({}), + ServerSettingsService.layerTest(), + ), + ), + Layer.provideMerge(config), + Layer.provideMerge(NodeServices.layer), + ); + const runner = yield* serveRunner(runnerLayer); + const fake = yield* makeFakeMachineDirectory({ + initial: [[threadId, { state: "running", runnerUrl: runner.url }]], + }); + const pool = yield* makePool({ checkoutRoot: NodePath.join(root, "t") }).pipe( + Effect.provideService(MachineDirectory, fake.directory), + ); + const runnerConfig = yield* ServerConfig.pipe( + Effect.provide(config.pipe(Layer.provide(NodeServices.layer))), + ); + return { root, checkout, turns, mcpAtStart, pool, attachmentsDir: runnerConfig.attachmentsDir }; +}); + +describe("runner handlers", () => { + it.live("materializes shipped attachments and rewrites the prompt to the runner's paths", () => + Effect.scoped( + Effect.gen(function* () { + const { pool, turns, attachmentsDir } = yield* setup; + const attachment = { + type: "image" as const, + id: `${threadId}-1f1f1f1f-0000-4000-8000-000000000000`, + name: "screen.png", + mimeType: "image/png", + sizeBytes: 4, + }; + const hubPath = `/hub/state/attachments/${attachment.id}.png`; + yield* pool.use(threadId, { wake: false, operation: "test" }, (connection) => + connection.client["runner.provider.sendTurn"]({ + instanceId, + input: { + threadId, + input: `Look at this\n\n[Attached image "screen.png" is saved at: ${hubPath}]`, + attachments: [attachment], + }, + attachments: [ + { attachment, hubPath, bytesBase64: Buffer.from([1, 2, 3, 4]).toString("base64") }, + ], + }), + ); + const runnerPath = resolveAttachmentPath({ attachmentsDir, attachment })!; + expect(Array.from(NodeFS.readFileSync(runnerPath))).toEqual([1, 2, 3, 4]); + expect(turns[0]?.input).toBe( + `Look at this\n\n[Attached image "screen.png" is saved at: ${runnerPath}]`, + ); + expect(turns[0]?.attachments).toEqual([attachment]); + }), + ), + ); + + it.live("installs the hub's MCP session before starting the provider session", () => + Effect.scoped( + Effect.gen(function* () { + const { pool, mcpAtStart } = yield* setup; + const mcp = { + environmentId: EnvironmentId.make("env-1"), + threadId, + providerSessionId: "provider-session-1", + providerInstanceId: instanceId, + endpoint: "https://hub.example/mcp", + authorizationHeader: "Bearer mcp-token", + }; + yield* pool.use(threadId, { wake: false, operation: "test" }, (connection) => + connection.client["runner.provider.startSession"]({ + instanceId, + input: { threadId, provider, runtimeMode: "full-access" }, + mcp, + }), + ); + expect(mcpAtStart).toEqual([mcp]); + McpProviderSession.clearMcpProviderSession(threadId); + }), + ), + ); + + it.live("refuses calls for another thread or outside the checkout with typed errors", () => + Effect.scoped( + Effect.gen(function* () { + const { pool, turns } = yield* setup; + const otherThread = yield* pool + .use(threadId, { wake: false, operation: "test" }, (connection) => + connection.client["runner.provider.sendTurn"]({ + instanceId, + input: { threadId: ThreadId.make("someone-else"), input: "hi" }, + attachments: [], + }), + ) + .pipe(Effect.flip); + expect(otherThread).toMatchObject({ + _tag: "RunnerRemoteError", + errorTag: "ProviderAdapterValidationError", + }); + const outside = yield* pool + .use(threadId, { wake: false, operation: "test" }, (connection) => + connection.client["runner.checkpoint.hasRef"]({ + cwd: "/etc", + checkpointRef: "refs/t3/checkpoints/x/turn/1" as never, + }), + ) + .pipe(Effect.flip); + expect(outside).toMatchObject({ _tag: "VcsRepositoryDetectionError", cwd: "/etc" }); + expect(turns).toEqual([]); + }), + ), + ); +}); From c933f0bcc2e242e97fcb99127221b17b19018517 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 00:16:21 +0000 Subject: [PATCH 27/58] test(server): run the hub-runner loopback on Postgres when a test database is set With T3_HUB_TEST_DATABASE_URL the loopback hub persists through the production hub database (migrations, tenant client, row-level security) in a disposable schema that outlives hub restarts, and selects its thread-machine stores and repository identity exactly as a hub does; without it the hub stays on SQLite in its base directory. The runner's status mock now reports the branch actually checked out. A fixed `main` raced the bootstrap test: the checkpoint reactor adopts the reported branch after each turn, which on the slower Postgres path overwrote the branch bootstrap recorded. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../HubRunnerLoopbackHarness.integration.ts | 65 ++++++++++++++----- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/apps/server/integration/HubRunnerLoopbackHarness.integration.ts b/apps/server/integration/HubRunnerLoopbackHarness.integration.ts index dfd4caec448f..3c8219fe5566 100644 --- a/apps/server/integration/HubRunnerLoopbackHarness.integration.ts +++ b/apps/server/integration/HubRunnerLoopbackHarness.integration.ts @@ -7,8 +7,10 @@ * SQLite outbox, real git checkpoints and checkout preparation in * `/t/`, hosting the scripted `TestProviderAdapter`. * - * Hub: the orchestration engine, `ProviderService` and reactors on SQLite, - * with the hub layer set for everything checkout-bound: the remote provider + * Hub: the orchestration engine, `ProviderService` and reactors on SQLite, or + * on Postgres through the production hub database when + * T3_HUB_TEST_DATABASE_URL is set, with the hub layer set for everything + * checkout-bound: the remote provider * driver, routed checkpoint store with the diff cache, git status cache, * workspace routing, reactor hooks, event delivery and its ack loop, and a * fake machine directory pointing at the runner. @@ -79,13 +81,17 @@ import { ThreadDeletionReactor } from "../src/orchestration/Services/ThreadDelet import * as ThreadBackgroundLiveness from "../src/orchestration/ThreadBackgroundLiveness.ts"; import * as ThreadPlanProgress from "../src/orchestration/ThreadPlanProgress.ts"; import * as ThreadSettlementReactor from "../src/orchestration/ThreadSettlementReactor.ts"; -import { HubThreadMachineStateSqliteLive } from "../src/persistence/Layers/HubThreadMachineState.ts"; import { OrchestrationCommandReceiptRepositoryLive } from "../src/persistence/Layers/OrchestrationCommandReceipts.ts"; import { OrchestrationEventStoreLive } from "../src/persistence/Layers/OrchestrationEventStore.ts"; import { ProjectionCheckpointRepositoryLive } from "../src/persistence/Layers/ProjectionCheckpoints.ts"; import { ProjectionPendingApprovalRepositoryLive } from "../src/persistence/Layers/ProjectionPendingApprovals.ts"; -import { makeSqlitePersistenceLive } from "../src/persistence/Layers/Sqlite.ts"; +import { layerConfig as ServerPersistenceLive } from "../src/persistence/Layers/Sqlite.ts"; import * as HubRepositoryIdentityResolver from "../src/persistence/Postgres/HubRepositoryIdentityResolver.ts"; +import { + hubTestDatabaseLayer, + hubTestDatabaseUrl, + makeHubTestSchema, +} from "../src/persistence/Postgres/hubTestDatabase.ts"; import * as ProviderSessionRuntime from "../src/persistence/ProviderSessionRuntime.ts"; import { CheckpointTurnDiffStore } from "../src/persistence/Services/HubThreadMachineState.ts"; import { ProjectionPendingApprovalRepository } from "../src/persistence/Services/ProjectionPendingApprovals.ts"; @@ -133,6 +139,7 @@ import { export const LOOPBACK_PROVIDER = ProviderDriverKind.make("codex"); const isDispatchError = Schema.is(OrchestrationDispatchCommandError); export const LOOPBACK_INSTANCE_ID = defaultInstanceIdForDriver(LOOPBACK_PROVIDER); +const LOOPBACK_TENANT_ID = "user_loopback"; const git = (cwd: string, args: ReadonlyArray) => NodeChildProcess.execFileSync( @@ -191,13 +198,21 @@ const instanceRegistryLayer = (adapter: TestProviderAdapterHarness["adapter"]) = }); }; -const localStatus = { - isRepo: true, - hasPrimaryRemote: false, - isDefaultRef: true, - refName: "main", - hasWorkingTreeChanges: false, - workingTree: { files: [], insertions: 0, deletions: 0 }, +/** + * A clean status on the branch actually checked out: the checkpoint reactor + * adopts the reported branch after each turn, so a fixed one would race the + * branch that bootstrap recorded. + */ +const localStatusOf = (checkout: string) => { + const refName = git(checkout, ["rev-parse", "--abbrev-ref", "HEAD"]).trim(); + return { + isRepo: true, + hasPrimaryRemote: false, + isDefaultRef: refName === "main", + refName, + hasWorkingTreeChanges: false, + workingTree: { files: [], insertions: 0, deletions: 0 }, + }; }; export interface LoopbackRunner { @@ -291,15 +306,15 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") Layer.mock(GitWorkflowService.GitWorkflowService)({}), Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({ streamStatus: () => Stream.never, - refreshLocalStatus: () => Effect.succeed(localStatus), + refreshLocalStatus: () => Effect.sync(() => localStatusOf(checkout)), refreshStatus: () => - Effect.succeed({ - ...localStatus, + Effect.sync(() => ({ + ...localStatusOf(checkout), hasUpstream: false, aheadCount: 0, behindCount: 0, pr: null, - }), + })), }), Layer.mock(VcsProvisioningService.VcsProvisioningService)({}), Layer.mock(ReviewService.ReviewService)({}), @@ -343,6 +358,20 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") // ── Hub ────────────────────────────────────────────────────────────── const hubBaseDir = NodePath.join(rootDir, "hub"); + // With T3_HUB_TEST_DATABASE_URL the hub persists to a disposable Postgres + // schema through the production hub database (tenant client, migrations, + // row-level security); otherwise to SQLite in its base directory. Either + // way the database outlives hub restarts. + const harnessScope = yield* Scope.make(); + const hubSchema = + hubTestDatabaseUrl === undefined + ? undefined + : yield* makeHubTestSchema(hubTestDatabaseUrl).pipe( + Scope.provide(harnessScope), + Effect.orDie, + ); + const hubDatabaseLayer = + hubSchema === undefined ? Layer.empty : hubTestDatabaseLayer(hubSchema, LOOPBACK_TENANT_ID); const hubConfigLayer = Layer.effect( ServerConfig, Effect.gen(function* () { @@ -352,7 +381,7 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") ).pipe(Layer.provide(ServerConfig.layerTest(rootDir, hubBaseDir))); const makeHubLayer = () => { - const persistence = makeSqlitePersistenceLive(NodePath.join(hubBaseDir, "hub.sqlite")); + const persistence = ServerPersistenceLive; const infrastructure = Layer.mergeAll(RunnerEventDelivery.layer, hubVcsStatusCacheLayer).pipe( Layer.provideMerge( RemoteSessionRegistry.layer.pipe(Layer.provide(ProviderSessionRuntime.layer)), @@ -364,7 +393,7 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") ), ), Layer.provideMerge(Layer.succeed(MachineDirectory, directory.directory)), - Layer.provideMerge(HubThreadMachineStateSqliteLive), + Layer.provideMerge(HubLayers.makeHubStateStoresLayer(persistence)), ); const remoteAdapterRegistry = Layer.effect( ProviderAdapterRegistry, @@ -450,6 +479,7 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") Layer.provideMerge(persistence), Layer.provideMerge(ServerSettingsService.layerTest()), Layer.provideMerge(hubConfigLayer), + Layer.provideMerge(hubDatabaseLayer), Layer.provideMerge(NodeServices.layer), ); }; @@ -571,6 +601,7 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") dispose: Effect.gen(function* () { yield* stopHub(hub); if (runnerScope) yield* Scope.close(runnerScope, Exit.void); + yield* Scope.close(harnessScope, Exit.void); NodeFS.rmSync(rootDir, { recursive: true, force: true }); }), } satisfies HubRunnerLoopbackHarness; From 659341812dd9547c1737f907a18bc4ee9f7f41ed Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 00:16:27 +0000 Subject: [PATCH 28/58] fix(server): restore hub attachments from Postgres before shipping them to runners A hub with a database keeps attachment bytes in `hub_attachments` and its local attachments directory is only a cache that a new process or base directory starts without. The remote provider driver read attachment files straight from that directory, so a turn, title or branch-name request processed after a hub moved or restarted shipped no attachments. It now restores each attachment into the cache first, like the other id lookups do; without a hub database this is a no-op. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/hub/RemoteProviderDriver.test.ts | 41 ++++++++++++- apps/server/src/hub/RemoteProviderDriver.ts | 57 ++++++++++++------- 2 files changed, 77 insertions(+), 21 deletions(-) diff --git a/apps/server/src/hub/RemoteProviderDriver.test.ts b/apps/server/src/hub/RemoteProviderDriver.test.ts index a7b8842cf4b0..dadcdeefc0f1 100644 --- a/apps/server/src/hub/RemoteProviderDriver.test.ts +++ b/apps/server/src/hub/RemoteProviderDriver.test.ts @@ -14,6 +14,7 @@ import { } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Reactivity from "effect/unstable/reactivity/Reactivity"; import { afterEach, describe, expect } from "vite-plus/test"; import { resolveAttachmentPath } from "../attachmentStore.ts"; @@ -21,6 +22,12 @@ import { ServerConfig } from "../config.ts"; import * as McpProviderSession from "../mcp/McpProviderSession.ts"; import { HubThreadMachineStateSqliteLive } from "../persistence/Layers/HubThreadMachineState.ts"; import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; +import { persistHubAttachment } from "../persistence/Postgres/HubAttachments.ts"; +import { + hubTestDatabaseLayer, + hubTestDatabaseUrl, + makeHubTestSchema, +} from "../persistence/Postgres/hubTestDatabase.ts"; import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; import { CodexDriver } from "../provider/Drivers/CodexDriver.ts"; import { MachineDirectory, makeFakeMachineDirectory } from "./MachineDirectory.ts"; @@ -83,7 +90,39 @@ describe("remote provider driver helpers", () => { bytesBase64: Buffer.from("abc").toString("base64"), }, ]); - }), + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect.skipIf(hubTestDatabaseUrl === undefined)( + "restores an attachment the hub's local cache lost before shipping it", + () => + Effect.gen(function* () { + const schema = yield* makeHubTestSchema(hubTestDatabaseUrl!); + const dir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "hub-attachments-test-")); + tempDirs.push(dir); + const attachment = { + type: "file" as const, + id: `${threadId}-4a4a4a4a-0000-4000-8000-000000000000`, + name: "notes.md", + mimeType: "text/markdown", + sizeBytes: 3, + }; + const hubPath = resolveAttachmentPath({ attachmentsDir: dir, attachment })!; + NodeFS.mkdirSync(NodePath.dirname(hubPath), { recursive: true }); + NodeFS.writeFileSync(hubPath, "abc"); + const files = yield* Effect.gen(function* () { + yield* persistHubAttachment({ + attachmentsDir: dir, + relativePath: NodePath.relative(dir, hubPath), + }); + // A new hub process on a fresh base directory. + NodeFS.rmSync(hubPath); + return yield* attachmentFilesForRunner(dir, [attachment]); + }).pipe(Effect.provide(hubTestDatabaseLayer(schema, "user-attachments"))); + expect(files).toEqual([ + { attachment, hubPath, bytesBase64: Buffer.from("abc").toString("base64") }, + ]); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, Reactivity.layer))), ); }); diff --git a/apps/server/src/hub/RemoteProviderDriver.ts b/apps/server/src/hub/RemoteProviderDriver.ts index 05104df6bc09..5228f62bc7ae 100644 --- a/apps/server/src/hub/RemoteProviderDriver.ts +++ b/apps/server/src/hub/RemoteProviderDriver.ts @@ -46,7 +46,9 @@ import { } from "@t3tools/contracts/runner"; import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; +import type * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; +import type * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; @@ -55,6 +57,7 @@ import * as Stream from "effect/Stream"; import { resolveAttachmentPath } from "../attachmentStore.ts"; import { ServerConfig } from "../config.ts"; import * as McpProviderSession from "../mcp/McpProviderSession.ts"; +import { hydrateHubAttachment } from "../persistence/Postgres/HubAttachments.ts"; import { type ProviderAdapterError, ProviderAdapterRequestError, @@ -82,7 +85,9 @@ export type RemoteProviderDriverEnv = | RunnerConnectionPool | RunnerEventDelivery | RemoteSessionRegistry - | ServerConfig; + | ServerConfig + | FileSystem.FileSystem + | Path.Path; const isThreadMachineUnavailable = Schema.is(ThreadMachineUnavailableError); const isProviderAdapterError = Schema.is(ProviderAdapterErrorSchema); @@ -115,22 +120,29 @@ export class RunnerAttachmentReadError extends Schema.TaggedErrorClass | undefined, ) => Effect.forEach(attachments ?? [], (attachment) => - Effect.try({ - try: (): ReadonlyArray => { - const hubPath = resolveAttachmentPath({ attachmentsDir, attachment }); - if (hubPath === null || !NodeFS.existsSync(hubPath)) return []; - return [ - { attachment, hubPath, bytesBase64: NodeFS.readFileSync(hubPath).toString("base64") }, - ]; - }, - catch: (cause) => new RunnerAttachmentReadError({ attachmentId: attachment.id, cause }), - }), + Effect.andThen( + hydrateHubAttachment({ attachmentsDir, attachmentId: attachment.id }), + Effect.try({ + try: (): ReadonlyArray => { + const hubPath = resolveAttachmentPath({ attachmentsDir, attachment }); + if (hubPath === null || !NodeFS.existsSync(hubPath)) return []; + return [ + { attachment, hubPath, bytesBase64: NodeFS.readFileSync(hubPath).toString("base64") }, + ]; + }, + catch: (cause) => new RunnerAttachmentReadError({ attachmentId: attachment.id, cause }), + }), + ), ).pipe(Effect.map((files) => files.flat())); interface InstanceContext { @@ -139,7 +151,9 @@ interface InstanceContext { readonly pool: RunnerConnectionPool["Service"]; readonly delivery: RunnerEventDelivery["Service"]; readonly registry: RemoteSessionRegistry["Service"]; - readonly attachmentsDir: string; + readonly readAttachments: ( + attachments: ReadonlyArray | undefined, + ) => Effect.Effect, RunnerAttachmentReadError>; readonly publicUrl: string | undefined; } @@ -155,13 +169,17 @@ export function makeRemoteProviderDriver( create: ({ instanceId, displayName, accentColor, enabled }) => Effect.gen(function* () { const config = yield* ServerConfig; + const platform = yield* Effect.context(); const context: InstanceContext = { driverKind, instanceId, pool: yield* RunnerConnectionPool, delivery: yield* RunnerEventDelivery, registry: yield* RemoteSessionRegistry, - attachmentsDir: config.attachmentsDir, + readAttachments: (attachments) => + attachmentFilesForRunner(config.attachmentsDir, attachments).pipe( + Effect.provideContext(platform), + ), publicUrl: config.hub?.publicUrl, }; const capabilities: { @@ -281,10 +299,9 @@ function makeRemoteAdapter( // Captured before waking: if the machine was recreated while asleep, // reconciliation settles the old session and this restarts it. const before = yield* hostedSession(input.threadId); - const attachments = yield* attachmentFilesForRunner( - context.attachmentsDir, - input.attachments, - ).pipe(Effect.mapError(toAdapterError("sendTurn"))); + const attachments = yield* context + .readAttachments(input.attachments) + .pipe(Effect.mapError(toAdapterError("sendTurn"))); return yield* onRunner(input.threadId, "sendTurn", true, (connection) => Effect.gen(function* () { const current = yield* hostedSession(input.threadId); @@ -497,7 +514,7 @@ function makeRemoteTextGeneration(context: InstanceContext): TextGenerationShape ); return { generateThreadTitle: (request) => - attachmentFilesForRunner(context.attachmentsDir, request.attachments).pipe( + context.readAttachments(request.attachments).pipe( Effect.mapError(toTextGenerationError("generateThreadTitle")), Effect.flatMap((attachments) => onThreadRunner("generateThreadTitle", request.cwd, (connection) => @@ -515,7 +532,7 @@ function makeRemoteTextGeneration(context: InstanceContext): TextGenerationShape ), ), generateBranchName: (request) => - attachmentFilesForRunner(context.attachmentsDir, request.attachments).pipe( + context.readAttachments(request.attachments).pipe( Effect.mapError(toTextGenerationError("generateBranchName")), Effect.flatMap((attachments) => onThreadRunner("generateBranchName", request.cwd, (connection) => From b97d90894e0de8059d8a999cb69c92cd1b5c2944 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 00:17:58 +0000 Subject: [PATCH 29/58] test(server): run the thread-machines e2e against a Postgres hub `--hub-database-url` (and optionally `--hub-database-admin-url` for a separate migration role) runs the hub as production hubs run: on Postgres, for a fresh tenant with a random secret key, seeded with the scenario's settings through `t3 hub import`, issuing its bearer session into the tenant, and restarting on an empty base directory. Without the flag the hub keeps persisting to SQLite in its base directory. Also fixes the hub token issue, which called an undefined `childProcess`. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/scripts/thread-machines-e2e.mjs | 97 ++++++++++++++++----- 1 file changed, 73 insertions(+), 24 deletions(-) diff --git a/apps/server/scripts/thread-machines-e2e.mjs b/apps/server/scripts/thread-machines-e2e.mjs index 67aeea0cd64b..a2eb417008d3 100644 --- a/apps/server/scripts/thread-machines-e2e.mjs +++ b/apps/server/scripts/thread-machines-e2e.mjs @@ -15,13 +15,21 @@ * e. a runner restart (new boot), then a turn that resumes the session * f. a hub restart; history and diffs survive * + * By default the hub persists to SQLite in its base directory. With + * `--hub-database-url` it runs as production hubs do: on Postgres, for a fresh + * tenant with a random secret key, seeded through `t3 hub import` from a + * standalone state directory, with a separate migration role when + * `--hub-database-admin-url` is given, and restarting on an empty base + * directory. Use a disposable database; the tenant's rows are left in it. + * * It uses the real provider through T3's adapter on the runner, with this * machine's existing provider login, for two short turns (plus title * generation). The model defaults to claude-haiku-4-5 on the claudeAgent - * instance. Nothing outside the temporary directory is written; processes - * started here are stopped by PID on exit. + * instance. Nothing outside the temporary directory (and the hub database) + * is written; processes started here are stopped by PID on exit. * * node apps/server/scripts/thread-machines-e2e.mjs [--keep] [--model ] + * [--hub-database-url [--hub-database-admin-url ]] */ import * as NodeChildProcess from "node:child_process"; import * as NodeCrypto from "node:crypto"; @@ -45,6 +53,8 @@ const flagValue = (name, fallback) => { }; const KEEP = args.includes("--keep"); const MODEL = { instanceId: "claudeAgent", model: flagValue("--model", "claude-haiku-4-5") }; +const HUB_DATABASE_URL = flagValue("--hub-database-url", undefined); +const HUB_DATABASE_ADMIN_URL = flagValue("--hub-database-admin-url", undefined); const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-thread-machines-e2e-")); const threadId = `thread-${NodeCrypto.randomUUID()}`; @@ -53,6 +63,12 @@ const checkoutRoot = NodePath.join(root, "t"); const checkout = NodePath.join(checkoutRoot, threadId); const runnerToken = NodeCrypto.randomBytes(24).toString("base64url"); const logs = NodePath.join(root, "logs"); +// The settings every process starts with; titles are generated on the +// thread's runner with the same cheap model. +const SETTINGS = JSON.stringify({ + enableProviderUpdateChecks: false, + textGenerationModelSelection: MODEL, +}); const stamp = () => new Date().toISOString().slice(11, 23); const log = (...values) => console.log(`[${stamp()}]`, ...values); @@ -142,26 +158,62 @@ async function startRunner() { log(`runner listening in ${Date.now() - started} ms`); } +/** Postgres hub settings: a fresh tenant and secret key per run (never printed). */ +const hubDatabaseEnv = HUB_DATABASE_URL + ? { + T3CODE_HUB_DATABASE_URL: HUB_DATABASE_URL, + ...(HUB_DATABASE_ADMIN_URL ? { T3CODE_HUB_DATABASE_ADMIN_URL: HUB_DATABASE_ADMIN_URL } : {}), + T3CODE_HUB_TENANT_ID: `e2e-${NodeCrypto.randomUUID()}`, + T3CODE_HUB_SECRET_KEY: NodeCrypto.randomBytes(32).toString("base64"), + } + : {}; +const hubEnv = { + T3CODE_SERVER_MODE: "hub", + T3CODE_RUNNER_URL: `ws://127.0.0.1:${RUNNER_PORT}/runner/ws`, + T3CODE_RUNNER_TOKEN: runnerToken, + T3CODE_HUB_CHECKOUT_ROOT: checkoutRoot, + T3CODE_HUB_PUBLIC_URL: HUB, + ...hubDatabaseEnv, +}; + +const t3 = (argv, env) => + NodeChildProcess.execFileSync(process.execPath, [BIN, ...argv], { + encoding: "utf8", + env: { ...baseEnv, ...env }, + stdio: ["ignore", "pipe", "pipe"], + }); + +/** + * Postgres hubs keep settings in the database: import a standalone state + * directory holding them (its state.sqlite is created by a standalone command). + */ +function seedHubDatabase() { + const seedDir = NodePath.join(root, "seed"); + t3(["auth", "session", "list", "--base-dir", seedDir], {}); + const report = t3(["hub", "import", NodePath.join(seedDir, "userdata")], hubDatabaseEnv); + const documents = /documents: (\d+)/.exec(report)?.[1]; + check( + "hub tenant seeded through t3 hub import", + Number(documents) >= 1, + `${documents} documents`, + ); +} + let hubToken = ""; +let hubBoots = 0; async function startHub() { const started = Date.now(); - const hubDir = NodePath.join(root, "hub"); - start("hub", hubDir, ["--base-dir", hubDir, "--port", `${HUB_PORT}`], { - T3CODE_SERVER_MODE: "hub", - T3CODE_RUNNER_URL: `ws://127.0.0.1:${RUNNER_PORT}/runner/ws`, - T3CODE_RUNNER_TOKEN: runnerToken, - T3CODE_HUB_CHECKOUT_ROOT: checkoutRoot, - T3CODE_HUB_PUBLIC_URL: HUB, - }); + // A Postgres hub's base directory is disposable: every boot gets an empty one. + hubBoots += 1; + const hubDir = NodePath.join(root, HUB_DATABASE_URL ? `hub-${hubBoots}` : "hub"); + NodeFS.mkdirSync(hubDir, { recursive: true }); + start("hub", hubDir, ["--base-dir", hubDir, "--port", `${HUB_PORT}`], hubEnv); await waitFor("hub to listen", () => listening(HUB_PORT)); if (!hubToken) { - hubToken = childProcess - .execFileSync( - process.execPath, - [BIN, "auth", "session", "issue", "--base-dir", hubDir, "--token-only"], - { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }, - ) - .trim(); + hubToken = t3( + ["auth", "session", "issue", "--base-dir", hubDir, "--token-only"], + hubEnv, + ).trim(); } await waitFor( "hub API", @@ -279,13 +331,9 @@ async function runTurn(text, timeoutMs = 240_000) { async function main() { NodeFS.mkdirSync(logs, { recursive: true }); NodeFS.mkdirSync(checkout, { recursive: true }); - for (const dir of ["runner", "hub"]) { + for (const dir of ["runner", HUB_DATABASE_URL ? "seed" : "hub"]) { NodeFS.mkdirSync(NodePath.join(root, dir, "userdata"), { recursive: true }); - NodeFS.writeFileSync( - NodePath.join(root, dir, "userdata", "settings.json"), - // Titles are generated on the thread's runner with the same cheap model. - JSON.stringify({ enableProviderUpdateChecks: false, textGenerationModelSelection: MODEL }), - ); + NodeFS.writeFileSync(NodePath.join(root, dir, "userdata", "settings.json"), SETTINGS); } const git = (...gitArgs) => NodeChildProcess.execFileSync( @@ -300,8 +348,9 @@ async function main() { NodeFS.writeFileSync(NodePath.join(checkout, "notes.txt"), "hello from the runner\n"); git("add", "."); git("commit", "-m", "initial"); - log(`workspace ${root}`); + log(`workspace ${root}; hub persistence: ${HUB_DATABASE_URL ? "Postgres" : "SQLite"}`); + if (HUB_DATABASE_URL) seedHubDatabase(); await startRunner(); await startHub(); From f932a9243251459a259827a9838c7eaaaa083aad Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 00:21:18 +0000 Subject: [PATCH 30/58] chore: remove the thread-machine prototype leftovers Delete prototype-2/ (scenario scripts and evidence), PROTOTYPE-2.md, and the hub prototype scripts (latency proxy, temporary SQLite migration). The backend parity check is kept as apps/server/scripts/hub-import-compare.ts, reworked for operators verifying a `t3 hub import`: it reads the hub database and tenant from the same T3CODE_HUB_* environment as the import, compares every thread by default, treats the import's NUL-to-U+FFFD replacement as equal, reports absent threads instead of crashing, and exits non-zero on differences. Verified against a scenario import on a local Postgres: ALL EQUAL. Co-Authored-By: Claude Opus 5.5 (1M context) --- PROTOTYPE-2.md | 435 ------------------ apps/server/scripts/hub-import-compare.ts | 185 ++++++++ .../scripts/hub-proto/compare-backends.ts | 167 ------- .../server/scripts/hub-proto/latency-proxy.ts | 31 -- .../scripts/hub-proto/migrate-temp-sqlite.ts | 18 - prototype-2/ctl.sh | 31 -- prototype-2/evidence/control-interrupt.out | 42 -- prototype-2/evidence/control-turn1.out | 49 -- prototype-2/evidence/diff-0-1.out | 7 - prototype-2/evidence/hub-delivery-log.txt | 350 -------------- prototype-2/evidence/hub-restart-test.out | 11 - .../evidence/hub-strace-all-lifetimes.txt | 32 -- prototype-2/evidence/outbox-summary.txt | 14 - .../evidence/runner-restart-midturn-test.out | 17 - prototype-2/evidence/runner-restart-test.out | 16 - prototype-2/evidence/runner-rpc-log.txt | 258 ----------- prototype-2/evidence/turn1.out | 45 -- prototype-2/evidence/turn10-continue.out | 110 ----- prototype-2/evidence/turn11-title.out | 33 -- prototype-2/evidence/turn6-interrupt.out | 72 --- prototype-2/evidence/turn7-hub-restart.out | 124 ----- .../evidence/turn8-after-runner-restart.out | 85 ---- .../evidence/turn9-runner-restart-midturn.out | 92 ---- prototype-2/hub-restart-test.sh | 18 - prototype-2/outbox-tail.mjs | 23 - prototype-2/runner-restart-midturn-test.sh | 18 - prototype-2/scenario.mjs | 335 -------------- prototype-2/start-control.sh | 7 - prototype-2/start-hub.sh | 14 - prototype-2/start-runner.sh | 8 - 30 files changed, 185 insertions(+), 2462 deletions(-) delete mode 100644 PROTOTYPE-2.md create mode 100644 apps/server/scripts/hub-import-compare.ts delete mode 100644 apps/server/scripts/hub-proto/compare-backends.ts delete mode 100644 apps/server/scripts/hub-proto/latency-proxy.ts delete mode 100644 apps/server/scripts/hub-proto/migrate-temp-sqlite.ts delete mode 100755 prototype-2/ctl.sh delete mode 100644 prototype-2/evidence/control-interrupt.out delete mode 100644 prototype-2/evidence/control-turn1.out delete mode 100644 prototype-2/evidence/diff-0-1.out delete mode 100644 prototype-2/evidence/hub-delivery-log.txt delete mode 100644 prototype-2/evidence/hub-restart-test.out delete mode 100644 prototype-2/evidence/hub-strace-all-lifetimes.txt delete mode 100644 prototype-2/evidence/outbox-summary.txt delete mode 100644 prototype-2/evidence/runner-restart-midturn-test.out delete mode 100644 prototype-2/evidence/runner-restart-test.out delete mode 100644 prototype-2/evidence/runner-rpc-log.txt delete mode 100644 prototype-2/evidence/turn1.out delete mode 100644 prototype-2/evidence/turn10-continue.out delete mode 100644 prototype-2/evidence/turn11-title.out delete mode 100644 prototype-2/evidence/turn6-interrupt.out delete mode 100644 prototype-2/evidence/turn7-hub-restart.out delete mode 100644 prototype-2/evidence/turn8-after-runner-restart.out delete mode 100644 prototype-2/evidence/turn9-runner-restart-midturn.out delete mode 100755 prototype-2/hub-restart-test.sh delete mode 100644 prototype-2/outbox-tail.mjs delete mode 100755 prototype-2/runner-restart-midturn-test.sh delete mode 100644 prototype-2/scenario.mjs delete mode 100755 prototype-2/start-control.sh delete mode 100755 prototype-2/start-hub.sh delete mode 100755 prototype-2/start-runner.sh diff --git a/PROTOTYPE-2.md b/PROTOTYPE-2.md deleted file mode 100644 index 2feb9ff87eb1..000000000000 --- a/PROTOTYPE-2.md +++ /dev/null @@ -1,435 +0,0 @@ -# Prototype 2: remote provider driver (hub + runner) - -Question from `docs/cloud-agents-architecture.md` (remote-dev): does a remote provider -driver preserve T3 behavior? - -**Answer: yes for the turn lifecycle.** A checkout-less hub and a runner in separate -processes, with separate state directories, completed real Claude turns. The runs covered -streamed output, approvals, interrupts, checkpoints and diffs, a hub restart during a turn, -and runner restarts between and during turns. No events were lost or duplicated. For the -same prompts, a local-mode control run of the same build produced the same projections -and the same diff bytes. - -The cut works at the driver level, as proposed. `ProviderService` (the session directory, -resume cursors and `recoverSessionForThread`) and every orchestration reactor ran -unchanged in the hub, except that two `.git` checks now go through a service. The work -still needed is outside the turn loop: per-thread routing, caches that keep machines -asleep, crash reconciliation, and about 60 other call sites that act on the checkout -(terminals, files, VCS, attachments, MCP and so on). These are listed below. - -Branch `prototype/remote-runner` of the T3 fork. It is local only and not pushed. The -runs used Node 22.23 with sources run directly (`node src/bin.ts`), and the model -`claude-haiku-4-5` through the unmodified Claude Code 2.1.282 CLI and T3's -`ClaudeAdapter`, using this VM's existing login. Model usage: 12 short turns and 1 title -generation. - -## What was built - -``` -client ──HTTP/WS──▶ HUB :4411 (T3 server, T3CODE_RUNNER_URL set) - orchestration · projections · settings · ProviderService/session directory - RemoteProviderDriver ×7 kinds · RemoteCheckpointStore · CheckoutGitProbe - RunnerClient ──── WebSocket, Effect RPC (RunnerRpcGroup) ────┐ - ▼ - RUNNER :4412 (`t3 runner`) - real ClaudeDriver (+CodexDriver) · CheckpointStore/git - WorkspacePaths · RunnerOutbox (NDJSON, fsync) - cwd = /tmp/proto-runner/checkout -``` - -| File | Lines | Role | -| ---------------------------------------------------------------------------------------------- | ---------: | ------------------------------------------------------------------------------------------------------------------------------------ | -| `apps/server/src/runner/RunnerProtocol.ts` | 317 | Hub↔runner RPC group. Payloads are `@t3tools/contracts` schemas; errors are the adapter/VCS error classes | -| `apps/server/src/runner/RunnerServer.ts` | 360 | Runner build: registry of real drivers, event pump into the outbox, RPC handlers, token-checked WS route, and a shutdown drain | -| `apps/server/src/runner/RunnerOutbox.ts` (+ `.test.ts`) | 240 (+115) | Durable, replayable event log | -| `apps/server/src/runner/hub/RunnerClient.ts` | 313 | Supervised connection with reconnect, bounded wait for calls, cursor-based subscription, dedup, fan-out, cursor file | -| `apps/server/src/runner/hub/RemoteProviderDriver.ts` | 314 | Wraps each built-in driver's kind, schema and defaults. The adapter, snapshot and title/branch text generation forward to the runner | -| `apps/server/src/runner/hub/HubLayers.ts` | 225 | Hub replacements for services that touch the checkout (delegated or stubbed; table below) | -| `apps/server/src/runner/hub/RunnerDelivery.ts` | 50 | Starts delivery after activation. Every 500 ms it drains ingestion, then acknowledges | -| `apps/server/src/git/CheckoutGitProbe.ts` | 18 | `Context.Reference` that replaces the synchronous `isGitRepository` in two reactors. The default keeps the local behavior | -| edits: `server.ts`, `ProviderInstanceRegistryHydration.ts`, `config.ts`, `cli/*`, two reactors | +163/−33 | `byRunnerMode(local, hub)` layer switches; driver-list hydration; `T3CODE_RUNNER_URL`/`T3CODE_RUNNER_TOKEN`; `t3 runner` subcommand | -| `prototype-2/` | | Scenario driver (client over the hub's public API), launch and stop scripts, evidence | - -Checks: `pnpm typecheck` (apps/server) is clean. `vp lint` and `vp fmt --check` pass on -the changed files. `vp test run` passes for `RunnerOutbox.test.ts` (2 tests) and for the -touched suites `CheckpointReactor.test.ts`, `ProviderRuntimeIngestion.test.ts` and -`ProviderInstanceRegistryHydration.test.ts` (68 tests). Repo-wide suites were not run, per -AGENTS.md. - -### Running it - -```bash -# layout: /tmp/proto-runner/{checkout,hub,runner,logs}, settings.json with -# enableProviderUpdateChecks=false (no manifest/npm fetches), telemetry off. -cp prototype-2/{start-hub.sh,start-runner.sh,ctl.sh} /tmp/proto-runner/ -/tmp/proto-runner/ctl.sh start-runner # node src/bin.ts runner --base-dir /tmp/proto-runner/runner --port 4412 -/tmp/proto-runner/ctl.sh start-hub # T3CODE_RUNNER_URL=ws://127.0.0.1:4412/runner/ws node src/bin.ts --base-dir /tmp/proto-runner/hub --port 4411 -node apps/server/src/bin.ts auth session issue --base-dir /tmp/proto-runner/hub --token-only > /tmp/proto-runner/hub-token -node prototype-2/scenario.mjs setup -node prototype-2/scenario.mjs turn "" --approve [--interrupt-after-ms 4000] -node prototype-2/scenario.mjs diff 0 1 -prototype-2/hub-restart-test.sh ; prototype-2/runner-restart-midturn-test.sh -``` - -**How we show the hub never touches the checkout.** `start-hub.sh` runs the hub inside -`bwrap --dev-bind / / --tmpfs /tmp/proto-runner/checkout --chmod 0000 …`. In the hub's -mount namespace the checkout path is an empty directory with mode 000, so any stat, read, -`chdir` or git call on it fails with EACCES. The hub also runs under -`strace -f --seccomp-bpf -e trace=%file,%process,chdir,fchdir -P /tmp/proto-runner/checkout`, -which records every syscall, in the hub or any child process, that names that path. -**Result: across 4 hub lifetimes, 0 such syscalls.** The only lines recorded are 4 -SIGTERMs from `ctl.sh` and 28 SIGCHLDs from short-lived child processes -(`prototype-2/evidence/hub-strace-all-lifetimes.txt`). Everything the hub knows about the -checkout came from the runner. - -## Evidence - -Timestamps are from one VM clock. The raw output is in `prototype-2/evidence/`. - -### a) Turn runs on the runner and streams into hub projections - -`project.create` for `workspaceRoot=/tmp/proto-runner/checkout` was accepted in 89 ms. The -hub cannot stat that path; the runner validated it through -`runner.workspace.normalizeRoot`. `thread.create` took 21 ms. Turn 1, "append a line to -notes.txt", used runtime mode `approval-required`: - -``` -19:17:17.733 thread.turn.start accepted in 40 ms -19:17:21.057 activity tool.started / 21.468 tool.completed (first tool call) -19:17:22.695 activity tool.started: File change started -19:17:23.513 activity approval.requested: File-change approval requested -19:17:23.532 approval dispatched in 19 ms -19:17:23.941 activity approval.resolved / tool.completed: File change -19:17:24.782 message assistant: "done" -19:17:25.194 activity checkpoint.captured → turn completed after 7460 ms -checkpoints: turn 1 ready, notes.txt +1/-0 -``` - -On the runner, `claude` ran with `--add-dir /tmp/proto-runner/checkout --session-id 202a75b8…` -and `notes.txt` gained the line. Streaming: the "count to 1500" turn moved 117 -`content.delta` events through the outbox in 3.1 s, and the "count to 300" turn moved 151 -in 4.0 s. Hub ingestion buffered them into the message, as T3 does by default -(`enableLegacyTokenStreaming=false`). - -### b) Approval round trip - -Approvals are produced on the runner as `request.opened`, projected by the hub as -`approval.requested`, answered by the client with `thread.approval.respond`, and routed by -`ProviderCommandReactor` → `ProviderService` → the remote adapter → the runner's -`respondToRequest`. From the client's dispatch (19:27:40.447) to the runner handling -`respondToRequest` (19:27:40.470) took **23 ms**. In the control run the dispatch alone took -13 ms, against 17–46 ms remote. Case e1 below answered an approval that was created while -the hub was down. - -### c) Interrupt a running turn - -"Count from 1 to 1500" was interrupted 4 s in (`turn6-interrupt.out`): - -``` -client 19:24:27.260 thread.turn.interrupt -runner 19:24:27.277 rpc interruptTurn ok (2 ms) ← 17 ms end to end -outbox #365 item.completed · #367 turn.completed {state: interrupted, "Session stopped."} · #368 session.exited -hub turn settled 19:24:27.685, partial assistant text kept, session "stopped" -``` - -The local control run had the identical outcome: latestTurn `completed`, session -`stopped`, 2× `checkpoint.captured`. The Claude adapter implements interrupt as stop -session, so this is T3 behavior, not something the split introduced. - -### d) Checkpoint captured on the runner; diff served through the hub - -Refs `refs/t3/checkpoints//turn/{0,1,…}` exist only in the runner's checkout. -Runner-side `checkpoint.capture` took 42–74 ms. `orchestration.getTurnDiff` over the hub's -`/ws` returned the real patch in 121 ms, including the ws-ticket request and the socket -connect (control: 91 ms): - -``` -diff --git a/notes.txt b/notes.txt -index f31bc06..77957df 100644 -@@ -1 +1,2 @@ - hello from the runner -+edited by the remote runner -``` - -This is byte-identical to the control run, blob hashes included. After later turns the -per-turn summaries were still computed through the runner, for example turn 8 -`notes.txt +1/-0` after the hub restart. - -### e1) Hub restarted mid-turn; approval produced while the hub was down - -`prototype-2/hub-restart-test.sh`, prompt "sleep 20 in the foreground, then append 'after -hub restart'": - -``` -19:24:56.968 command running on runner → stop hub (SIGTERM, 107 ms). Outbox head 379, acked 379. - runner keeps working; events 380..395 appended with no hub attached -19:25:20.148 #395 request.opened file_change_approval ← approval requested while hub down -19:25:20.590 hub start (bwrap+strace, sources via type stripping) -19:25:27.725 hub: "runner event subscription opened { afterSequence: 379 }" -19:25:28.306 client sees approval.requested → approves (46 ms) → runner respondToRequest 19:25:28.372 -19:25:30.482 turn completed; notes.txt has 'after hub restart'; checkpoint turn 8 notes.txt +1/-0 -hub: throughSequence 405, deliveredCount 26, duplicateCount 0 -``` - -On restart the hub did not mark the session orphaned. `reconcileProviderSessions` asked -the remote adapters' `listSessions`, the runner reported the live session, and the session -was adopted. In the local architecture a hub restart kills the turn; here the turn -survived. The remote `stopAll` is a no-op, so hub shutdown does not stop sessions that -another machine hosts. - -### e2) Runner restarted between turns; resumed through the persisted `resumeCursor` - -``` -19:25:51.248 stop runner (107 ms) → hub: "runner connection ended" (retries every 1 s) -19:25:56.639 runner listening: outbox loaded head 405 / acked 405, same runnerId, new bootId -19:25:57.311 hub reconnected (0.67 s) → subscription afterSequence 405 -turn "which lines did you add to notes.txt earlier?": - runner rpc startSession {resumeCursor: {resume: '202a75b8…', resumeSessionAt: '3c54100f…'}} ok 110 ms - runner rpc sendTurn ok 1045 ms - assistant: "edited by the remote runner\nafter hub restart" ← memory from before both restarts -``` - -The runner no longer listed the session, so `hasSession` returned false and -`ProviderService.recoverSessionForThread` restarted it from the hub's -`provider_session_runtime.resume_cursor_json`. No recovery code was added. The first -restart of both processes, at 19:20, also resumed the same way. - -### e3) Runner restarted mid-turn - -`prototype-2/runner-restart-midturn-test.sh`. On graceful stop the runner stops its -sessions while the event pumps are still alive, so the exit events are persisted before -the process exits: - -``` -19:27:29.383 command running → stop runner (412 ms incl. drain) - outbox #432 item.completed(command, failed) #434 turn.completed {interrupted} #435 session.exited -19:27:31.860 runner restarted → hub reconnected 19:27:34.44 → turn settled in hub at 19:27:34.66 -follow-up turn "finish the cut-off task": startSession(resumeCursor) → approval → notes.txt 'after runner restart' ✓ -``` - -A runner crash (SIGKILL) would lose these exit events; see the gaps below. - -### Title generation through the runner - -For a second thread titled "New thread", `ProviderCommandReactor` called the remote -instance's `generateThreadTitle`. The runner ran it (`runner rpc text.generateThreadTitle` -took 14.2 s, spawning a fresh Claude CLI) and the hub renamed the thread to "Capital of -France". This also showed two threads sharing one runner and the session directory. - -### Totals - -| Measure | Value | -| ------------------------------------- | -------------------------------------------------------------------------------------------------- | -| Runner events delivered to the hub | 478 (sequences 1–478), **0 duplicates**, 0 schema rejects, 0 runner warnings | -| Process lifetimes | 6 runner boots, 4 hub boots, 0 lost events (every head equaled the acked sequence after reconnect) | -| Outbox cost | ~1.05 KB per event (NDJSON), one `fsync` per append; compacted to empty once acked | -| Turn 1, dispatch → settled | 7.46 s remote vs 7.89 s local control (model variance dominates) | -| Added hub hops | approval 23 ms and interrupt 17 ms client→runner; `getTurnDiff` +30 ms vs local | -| Runner reconnect after restart | 0.67 s (1 s retry loop) | -| Hub boot to replayed approval visible | 7.7 s (Node type stripping + strace; not tuned) | - -## Runner protocol (`RunnerRpcGroup`, protocol version 1) - -Transport: one WebSocket to `/runner/ws`, JSON serialization, runner token in the `token` -query parameter (prototype only). Each provider call names `instanceId`; each checkpoint -call names `cwd`. Provider errors carry the original tags -(`ProviderAdapter{Validation,SessionNotFound,SessionClosed,Request,Process}Error`), and -checkpoint errors are the contracts `VcsError` union. On the hub, transport failures -(`RunnerUnavailableError`, `RpcClientError`) become `ProviderAdapterRequestError` or -`VcsRepositoryDetectionError`, so upstream error handling is unchanged. - -| RPC | Payload → success | -| ---------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -| `runner.hello` | {} → `{protocolVersion, runnerId, bootId, headSequence, ackedSequence, instances}` | -| `runner.provider.startSession` | `{instanceId, input: ProviderSessionStartInput}` → `ProviderSession` | -| `runner.provider.sendTurn` | `{instanceId, input: ProviderSendTurnInput}` → `ProviderTurnStartResult` | -| `runner.provider.interruptTurn` / `stopSession` | `{instanceId, threadId, turnId?}` → void | -| `runner.provider.respondToRequest` / `respondToUserInput` | `{instanceId, threadId, requestId, decision \| answers}` → void | -| `runner.provider.listSessions` | `{instanceId}` → `ProviderSession[]` (answers `hasSession` too) | -| `runner.provider.readThread` / `rollbackThread` | `{instanceId, threadId, numTurns?}` → `{threadId, turns[]}` | -| `runner.provider.getCapabilities` | `{instanceId, refresh?}` → `{snapshot: ServerProvider, sessionModelSwitch}` | -| `runner.text.generateThreadTitle` / `generateBranchName` | title/branch input + `instanceId` → `{title}` / `{branch}` | -| `runner.events.subscribe` (stream) | `{afterSequence}` → stream of `{sequence, bootId, event: ProviderRuntimeEvent}` | -| `runner.events.ack` | `{throughSequence}` → `{ackedSequence, retained}` | -| `runner.checkpoint.isGitRepository` / `capture` / `hasRef` / `restore` / `diff` / `deleteRefs` | `CheckpointStore` inputs → bool / void / bool / bool / patch / void | -| `runner.workspace.normalizeRoot` | `{workspaceRoot, createIfMissing?}` → normalized path \| `RunnerWorkspaceError` | - -The hub keeps `ProviderService` unchanged. It still persists resume cursors and cwd, -mints MCP credentials, holds the per-thread lifecycle locks, and runs recovery. The runner -stamps `providerInstanceId` on every event before appending it, so the hub fans events -out by instance. - -## Event outbox - -**Prototype** (`RunnerOutbox.ts`): - -- Every adapter's `streamEvents` is piped into `append`. Each event is schema-encoded as - one NDJSON line with `sequence = head + 1` and the current `bootId`, written with - `writeSync`, `fsync`ed, and only then published to live subscribers. An event that fails - the wire schema is counted and dropped, so it cannot poison every subscription. None - were dropped in this run. -- `subscribe(after)` subscribes to the live PubSub first, then replays retained entries - with `sequence > after`, then tails live entries past the replay point. There is no gap - and no overlap. -- `ack(n)` persists `outbox-ack.json`, drops acknowledged entries, and rewrites the file - when it is empty or when 500 acknowledged lines have accumulated. It is atomic - (tmp + rename, fd reopened). -- On load, unparseable (torn) lines are discarded, and `head = max(file, acked)`, so - sequences keep increasing across restarts. `runner-id` is stable; `bootId` changes on - every start. -- Hub (`RunnerClient` + `RunnerDelivery`): it subscribes from its persisted cursor - (`runner-client-cursor.json`, reset if `runnerId` changes) and drops any - `sequence <= delivered`. Every 500 ms it drains `ProviderRuntimeIngestion`, then writes - the cursor and acks. Delivery waits for server activation plus 500 ms, so ingestion is - subscribed before replay starts. -- Guarantee: at-least-once, in order, per runner. After a hub crash, events ingested - inside the last ack window (≤ 500 ms) could be ingested again, because ingestion command - ids are random. - -**Production design:** - -1. Runner outbox in the machine's SQLite, in the checkout's state disk that survives - pause and recreation: `(sequence PK, boot_id, event_json, created_at)`, with an ack - watermark and size- and time-based retention. If an event past retention is still - unacknowledged, the hub must resynchronize from `readThread`. -2. The hub stores `runner_cursor(thread_machine_id, acked_sequence)` in the same Postgres - transaction as the orchestration events that ingestion produced. Ingestion also dedups - on `(machine, sequence)`, so replay is exactly-once in effect, and ack becomes a - notification rather than a correctness step. -3. The hub consumes through the control-plane tunnel, one subscription per awake thread - machine. Sequence and `bootId` gaps are explicit signals (see crash reconciliation - below). -4. Backpressure: the runner keeps writing while the hub is away, and the outbox is the - buffer. The hub bounds the replay batch and paces delivery against the ingestion - queue depth. - -## Hub-side call sites that touch the checkout, and how each was handled - -A read-only survey by a subagent found about 60 call sites. The hot path is marked 🔥. -This table covers the ones that matter for the split. - -| Call site | What it does | Handling in this prototype | -| ----------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | -| 🔥 `CheckpointReactor` `resolveCheckpointCwd` and the revert check (was sync `isGitRepository`) | `.git` existence | **Delegated** through `CheckoutGitProbe` → `runner.checkpoint.isGitRepository` (same `.git` rule) | -| 🔥 `ProviderRuntimeIngestion` `turn.diff.updated` placeholder (sync `isGitRepository`) | `.git` existence | **Delegated** (same probe) | -| 🔥 `CheckpointStore` capture/hasRef/restore/diff/deleteRefs (CheckpointReactor, CheckpointDiffQuery) | git plumbing in cwd | **Delegated** (`RemoteCheckpointStore`, cwd-keyed) | -| 🔥 `Normalizer` `project.create`/`meta.update` → `WorkspacePaths.normalizeWorkspaceRoot` | stat / mkdir -p | **Delegated** (`runner.workspace.normalizeRoot`); relative-path resolution stays pure on the hub | -| 🔥 `ProviderService` → adapters (`startSession` cwd, turns, approvals, interrupt, rollback, readThread) | spawn provider CLI in cwd | **Delegated** (remote driver) | -| 🔥 `ProviderCommandReactor` first-turn title / branch-name generation | provider CLI in cwd | **Delegated** (remote `textGeneration`); commit/PR/structured generation **fail** with "not delegated" | -| 🔥 `ProjectionSnapshotQuery` via `RepositoryIdentityResolver` (every shell snapshot, project lookups, reactors, watchdog seed) | `git rev-parse` + `git remote -v` | **Stubbed** (identity `null`). Production: persist identity on the project, reported by the runner | -| 🔥 `CheckpointReactor` → `WorkspaceEntries.refresh` after capture | stat + file-index rescan | **Stubbed** (no-op); search/list/browse return empty | -| 🔥 `CheckpointReactor.refreshLocalGitStatusFromTurnCompletion`, CommandDispatcher bootstrap, `vcs*` RPCs → `VcsStatusBroadcaster` | realpath + git status/fetch | **Stubbed** (fails with "not delegated"; logged once per turn). Branch-drift following is off. The sidebar has no git status | -| `ProviderCommandReactor.ensureThreadWorktree` (turn start when the thread has a worktree) | `fs.exists(worktreePath)`, `git worktree prune/add` | **Still broken** (not exercised; local-mode threads have no worktree). It would stat the hub's disk and then run git locally | -| `ProviderCommandReactor` worktree branch rename (`git branch -m`) + `refreshStatus` | git in worktree | **Still broken** (worktree mode only) | -| `CommandDispatcher` bootstrap: `git fetch`, `worktree add`, setup script → `ProjectSetupScriptRunner` → terminal PTY | git + PTY in the checkout | **Still broken** (the prototype uses `/api/orchestration/dispatch`, which bypasses bootstrap) | -| `ThreadSettlementReactor` sweep → `GitManager.branchPullRequest({cwd: workspaceRoot})` (every minute, threads with branches) | git remote/for-each-ref + gh | **Still broken** (not exercised; the test threads have `branch: null`) | -| `ws.ts`: file search/read/write, `filesystemBrowse`, terminals, VCS/git/PR actions, review diff preview, open in editor, source-control clone/publish | FS, PTY, git, gh in cwd | **Still local**, so they fail on a hub. Need routed runner RPCs | -| `http.ts` asset routes (project favicon, workspace/media files), `ProjectFaviconResolver`, `T3ProjectFileLoader` (`t3.json`) | stat/read files in the checkout | **Still local** (UI only). Proxy to the runner or persist on the hub | -| `workflowScriptQuery` (reads `~/.claude/projects/...`) | provider home on the machine | **Still local**; belongs on the runner | -| Server cwd fallbacks (`?? process.cwd()` for title cwd, OpenCode `directory: serverConfig.cwd`, probes) | provider work in the server cwd | Harmless on the hub because nothing spawns there. Runner probes use the runner cwd | - -## What breaks or is missing - -- **Terminals.** `TerminalManager` still runs on the hub. The RPCs are already keyed by - `threadId`, so they need a routed streaming runner service (PTY data, resize, restart, - port scanning for previews). -- **Workspace file RPCs.** Search, read and write, `@`-mention indexing (the hub stub - returns empty lists), and filesystem browse for the project picker. Browse must target - the machine that will own the project. -- **VCS status.** Stubbed, so the sidebar has no git status, branch-drift following is - off, and there is no PR and branch lookup. Production needs the runner to push status - into a hub cache (the design's "caches that keep machines asleep"), and routed RPCs for - pull, commit, push and create PR. -- **Attachments.** The hub writes uploads to its own `attachmentsDir`, but the runner's - adapter reads `/`. A turn with an image fails or loses the - image. Uploads need to be copied to the runner with each turn and the paths rewritten. - The `ChatAttachment` ids already carry through `sendTurn`. -- **MCP endpoint.** `ProviderService.prepareMcpSession` mints the credential in the hub - and stores it in the hub process's `McpProviderSession` global. The runner's - `ClaudeAdapter` reads its own global and finds nothing, so agents get no `t3-code` MCP - tools (browser, review). The credential and the hub's public MCP URL have to travel in - `startSession`, which means extending `ProviderSessionStartInput` or adding an MCP field - to the protocol. -- **Text generation.** Title and branch generation are delegated and work (14 s, same - path as local). Commit message, PR content and `generateStructured` fail; the schema is - not serializable, so the runner needs to own those callers (git actions) instead. -- **Auth connector and provider sign-in.** `authConnector` still runs CLI logins where the - hub is. It must run on the runner, or be replaced by the credential service. - Provider maintenance (update CLI) and usage scanning are runner or host concerns too. -- **Provider settings sync.** The runner reads its own `settings.json`. The hub's provider - instance config (binary path, env, custom instances, enabled flags) is not pushed; the - hub's remote instance only mirrors id, kind, display name and enabled. The runner hosts - only `claudeAgent` and `codex` (`T3CODE_RUNNER_DRIVERS`), and the other kinds show as - unavailable. The snapshot and model list are fetched once at hub boot and on `refresh`, - with no push from the runner. `continuationIdentity` uses the default key rather than - Claude's HOME-based group key. `sessionModelSwitch` is hard-coded to `in-session` on the - hub. `uploadFeedback` is not forwarded. -- **Runner crash (SIGKILL) mid-turn.** No exit events are written, so the hub turn stays - `running` until `TurnLivenessWatchdog` fires (10 min silence). Fix: when a reconnect - shows a new `bootId`, the hub diffs its active bindings against the runner's - `listSessions` and synthesizes `session.exited` and an interrupted `turn.completed` for - the missing sessions. The graceful path is covered by the shutdown drain (e3). -- **`hasSession`/`listSessions` go to the runner on every routed call**, and startup - reconcile calls every runner. With sleeping machines this wakes them. The hub should - answer from the session directory plus runner-pushed lifecycle events, and treat - "machine asleep" as "session resumable", as the design says. -- **Hub shutdown semantics.** `ProviderService`'s finalizer still marks every binding - `stopped` on hub exit, even though the runner keeps the session. It is harmless here - because routing asks the runner, but the directory is wrong until the next event. -- **Routing.** There is one runner URL per hub. Adapter calls are keyed by `threadId` and - route easily. `listSessions`, `stopAll` and the cwd-keyed `CheckpointStore` need to - become thread-keyed, or resolve thread by cwd, to pick a machine. -- **Security.** A static shared token sits in the query string. Production uses the - control-plane execution token over the machine's outbound tunnel. The hub never dials - machines directly. - -Provider behavior seen in these runs; the split did not cause it, and the control run -matches: - -- Claude interrupt stops the session, and T3 projects an interrupted turn as `completed`. -- The user's Claude settings (`--setting-sources=user,…`) auto-allow `Bash`, so only file - edits asked for approval. -- The model backgrounded a `sleep 45`. After resume, Claude reported the stopped - background task as a separate short turn. - -## Productionization plan (rough size) - -| Step | Work | Size (files / lines) | -| ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------- | -| 1 | Move `RunnerRpcGroup` into contracts with versioning. Hub and runner become separate compositions (`hub.ts`, `runner.ts`) instead of `byRunnerMode` switches, and host layers are dropped from the hub build | 6 / ~700 | -| 2 | Per-thread routing: a `ThreadMachineDirectory` (thread → machine, connection pool, wake through the control plane); thread-keyed `CheckpointStore`/`CheckpointDiffQuery` API | 8 / ~900 | -| 3 | Durable delivery: SQLite outbox on the runner, cursor stored in the same transaction as projection writes plus `(machine, sequence)` dedup, `bootId` crash reconciliation | 6 / ~800 + migration | -| 4 | Session state on the hub: `hasSession`/`listSessions` from the directory plus pushed lifecycle, "asleep = resumable", remote-aware `stopAll`/finalizer | 4 / ~350 | -| 5 | Hub caches: persisted repository identity, pushed VCS status, per-turn diffs in `checkpoint_diff_blobs`, so history and diffs never wake a machine | 8 / ~900 + migrations | -| 6 | Routed runner services for `ws.ts`: terminals (streaming), workspace files and search, VCS/git/PR/review, assets and favicon, worktree/bootstrap/setup scripts | 15 / ~2,500 | -| 7 | Attachments shipped with turns; MCP credential and URL in `startSession`; provider settings push and credential-service login; auth connector on the runner | 8 / ~1,000 | -| 8 | Contract tests that run the existing adapter/reactor suites through a loopback runner, plus a hub and runner e2e in CI | 6 / ~800 | - -That totals about 60 files and ~8,000 lines. Steps 1–4 are the milestone-1 core, about -2,700 lines. Steps 5–7 are needed before the product loses its always-on machine. - -**Risks:** - -- **Fork divergence.** `server.ts` composition, `ProviderService` session semantics, the - reactors and `ws.ts` routing are hot upstream files, so every upstream merge touches - them. Keep the seams narrow: driver, `CheckpointStore`, probes, routed services. -- **Resume fidelity.** Resume depends on the provider's own session files on the machine - (`~/.claude/projects//.jsonl`) and on the checkout living at an - identical absolute path. Recreate from snapshot must preserve both, and prototype 3 - should verify it. -- **Wake latency.** Every uncached hub read or `hasSession` call becomes a machine wake. - Missing a cache turns a fast UI into a spinner. -- **Exactly-once ingestion.** This needs the cursor and projection write in one - transaction. Without it, crash windows duplicate deltas and activities. -- **Security surface.** The runner executes whatever the hub sends. Its authentication - must be per-machine, scoped and short-lived. -- **Provider-specific behavior.** Background tasks and synthetic turns after resume show - up more often when machines restart freely. The UI must handle them gracefully. - -## Cleanup - -Every process started for this prototype has been stopped (hub, runner, local control). -`/tmp/proto-runner` holds the throwaway checkouts, state directories and logs. Claude -Code wrote its normal session files under -`~/.claude/projects/-tmp-proto-runner-{checkout,control-checkout}` (about 0.5 MB). No -credential file was read, copied or modified by this work. diff --git a/apps/server/scripts/hub-import-compare.ts b/apps/server/scripts/hub-import-compare.ts new file mode 100644 index 000000000000..901a995df8f9 --- /dev/null +++ b/apps/server/scripts/hub-import-compare.ts @@ -0,0 +1,185 @@ +// @effect-diagnostics nodeBuiltinImport:off +// Verifies a `t3 hub import`: reads the standalone state directory and the hub +// tenant it was imported into through the same snapshot queries (the +// standalone SQLite layer and the hub's Postgres layer, selected by +// HubDatabase as in a server) and compares the results. Prints only +// equal/different plus counts, never row contents. +// +// Reads the hub database and tenant from the same environment as +// `t3 hub import` (T3CODE_HUB_DATABASE_URL, T3CODE_HUB_DATABASE_ADMIN_URL, +// T3CODE_HUB_TENANT_ID). Every thread is compared unless thread ids are given. +// NUL characters, which the import stores as U+FFFD, compare equal. +// +// node apps/server/scripts/hub-import-compare.ts [thread-id,...] +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { ThreadId, type OrchestrationThreadDetailWindow } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as ManagedRuntime from "effect/ManagedRuntime"; +import * as Option from "effect/Option"; + +import { ServerConfig } from "../src/config.ts"; +import { OrchestrationProjectionSnapshotQueryLive } from "../src/orchestration/Layers/ProjectionSnapshotQuery.ts"; +import { ProjectionSnapshotQuery } from "../src/orchestration/Services/ProjectionSnapshotQuery.ts"; +import * as ThreadBackgroundLiveness from "../src/orchestration/ThreadBackgroundLiveness.ts"; +import * as ThreadPlanProgress from "../src/orchestration/ThreadPlanProgress.ts"; +import { + layerConfig as ServerPersistenceLive, + makeSqlitePersistenceLive, +} from "../src/persistence/Layers/Sqlite.ts"; +import { HubDatabase } from "../src/persistence/Postgres/HubDatabase.ts"; +import { makeHubDatabase } from "../src/persistence/Postgres/HubDatabaseLive.ts"; +import { RepositoryIdentityResolver } from "../src/project/RepositoryIdentityResolver.ts"; + +const [stateDir, threadArg] = process.argv.slice(2); +const databaseUrl = process.env.T3CODE_HUB_DATABASE_URL; +const tenantId = process.env.T3CODE_HUB_TENANT_ID; +if (!stateDir || !databaseUrl || !tenantId) { + throw new Error( + "usage: T3CODE_HUB_DATABASE_URL=… T3CODE_HUB_TENANT_ID=… hub-import-compare.ts [thread-id,...]", + ); +} +const out = (line: string) => process.stdout.write(`${line}\n`); + +// Repository identity is resolved from checkouts standalone and recorded in the +// hub; neither is under test here. +const queryLayer = OrchestrationProjectionSnapshotQueryLive.pipe( + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide(Layer.succeed(RepositoryIdentityResolver, { resolve: () => Effect.succeed(null) })), +); +const platform = ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-import-compare-" }).pipe( + Layer.provideMerge(NodeServices.layer), +); +const sqliteQuery = ManagedRuntime.make( + queryLayer.pipe( + Layer.provide(makeSqlitePersistenceLive(NodePath.join(stateDir, "state.sqlite"))), + Layer.provide(platform), + ), +); +const hubQuery = ManagedRuntime.make( + queryLayer.pipe( + Layer.provide(ServerPersistenceLive), + Layer.provide(platform), + Layer.provide( + Layer.effect( + HubDatabase, + makeHubDatabase({ + databaseUrl, + databaseAdminUrl: process.env.T3CODE_HUB_DATABASE_ADMIN_URL, + tenantId, + }), + ), + ), + ), +); + +const both = async ( + read: (query: ProjectionSnapshotQuery["Service"]) => Effect.Effect, +) => ({ + sqlite: await sqliteQuery.runPromise( + Effect.flatMap(Effect.service(ProjectionSnapshotQuery), read), + ), + hub: await hubQuery.runPromise(Effect.flatMap(Effect.service(ProjectionSnapshotQuery), read)), +}); + +// Sequences continue per tenant in the hub, so they are not compared, and the +// hub stores NUL characters as U+FFFD (Postgres text cannot hold them). +const normalize = (value: unknown): unknown => + JSON.parse(JSON.stringify(value), (key, inner) => + key === "snapshotSequence" || key === "threadSequence" + ? undefined + : typeof inner === "string" + ? inner.replaceAll("\u0000", "\uFFFD") + : inner, + ); + +let failures = 0; +const check = (label: string, sqlite: unknown, hub: unknown, counts: string) => { + const left = JSON.stringify(normalize(sqlite)); + const right = JSON.stringify(normalize(hub)); + const equal = left === right; + if (!equal) failures += 1; + out(`${equal ? "equal" : "DIFFERENT"}: ${label} (${counts}; ${left.length} bytes)`); +}; + +const readModel = await both((query) => query.getCommandReadModel()); +check( + "getCommandReadModel", + { ...readModel.sqlite, updatedAt: undefined }, + { ...readModel.hub, updatedAt: undefined }, + `threads=${readModel.hub.threads.length}`, +); +const shell = await both((query) => query.getShellSnapshot()); +check( + "getShellSnapshot", + { ...shell.sqlite, updatedAt: undefined }, + { ...shell.hub, updatedAt: undefined }, + `threads=${shell.hub.threads.length}`, +); +const archived = await both((query) => query.getArchivedShellSnapshot()); +check( + "getArchivedShellSnapshot", + { ...archived.sqlite, updatedAt: undefined }, + { ...archived.hub, updatedAt: undefined }, + `threads=${archived.hub.threads.length}`, +); +const counts = await both((query) => query.getCounts()); +check("getCounts", counts.sqlite, counts.hub, `threads=${counts.hub.threadCount}`); + +const threadIds = threadArg + ? threadArg.split(",").map((id) => ThreadId.make(id)) + : readModel.sqlite.threads.map((thread) => thread.id); +for (const [index, threadId] of threadIds.entries()) { + const label = `thread#${index + 1}`; + const full = await both((query) => query.getThreadDetailSnapshot(threadId)); + const fullHub = Option.getOrUndefined(full.hub)?.thread; + check( + `${label} full`, + full.sqlite, + full.hub, + fullHub + ? `messages=${fullHub.messages.length} activities=${fullHub.activities.length}` + : "absent", + ); + const byId = await both((query) => query.getThreadDetailById(threadId)); + check(`${label} getThreadDetailById`, byId.sqlite, byId.hub, "raw activities"); + if (fullHub === undefined) continue; + // Walk every page with the client's page sizes and compare each page. + for (const turnLimit of [10, 3]) { + let window: OrchestrationThreadDetailWindow = { turnLimit }; + for (let page = 1; page <= 200; page += 1) { + const current = window; + const result = await both((query) => query.getThreadDetailSnapshot(threadId, current)); + const hubPage = Option.getOrUndefined(result.hub); + check( + `${label} turnLimit=${turnLimit} page ${page}`, + result.sqlite, + result.hub, + hubPage + ? `messages=${hubPage.thread.messages.length} activities=${hubPage.thread.activities.length} hasMore=${hubPage.page?.hasMore}` + : "absent", + ); + const cursor = hubPage?.page?.beforeCursor; + if (!hubPage?.page?.hasMore || !cursor) break; + window = { turnLimit, beforeCursor: cursor }; + } + } +} + +const search = await both((query) => query.searchThreads({ query: "the", limit: 50 })); +check("searchThreads('the')", search.sqlite, search.hub, `matches=${search.hub.matches.length}`); +const searchUpper = await both((query) => query.searchThreads({ query: "THE", limit: 50 })); +check( + "searchThreads('THE')", + searchUpper.sqlite, + searchUpper.hub, + `matches=${searchUpper.hub.matches.length}`, +); + +out(failures === 0 ? "ALL EQUAL" : `${failures} DIFFERENCES`); +await hubQuery.dispose(); +await sqliteQuery.dispose(); +process.exitCode = failures === 0 ? 0 : 1; diff --git a/apps/server/scripts/hub-proto/compare-backends.ts b/apps/server/scripts/hub-proto/compare-backends.ts deleted file mode 100644 index 25f9441791f4..000000000000 --- a/apps/server/scripts/hub-proto/compare-backends.ts +++ /dev/null @@ -1,167 +0,0 @@ -// Compares the SQLite and Postgres read paths over the same imported history. -// Prints only equal/different plus counts, never row contents. -// -// node scripts/hub-proto/compare-backends.ts [skip-thread-ids,] -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { ThreadId, type OrchestrationThreadDetailWindow } from "@t3tools/contracts"; -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; -import * as ManagedRuntime from "effect/ManagedRuntime"; -import * as Option from "effect/Option"; - -import { ServerConfig } from "../../src/config.ts"; -import { OrchestrationProjectionSnapshotQueryLive } from "../../src/orchestration/Layers/ProjectionSnapshotQuery.ts"; -import { ProjectionSnapshotQuery } from "../../src/orchestration/Services/ProjectionSnapshotQuery.ts"; -import * as ThreadBackgroundLiveness from "../../src/orchestration/ThreadBackgroundLiveness.ts"; -import * as ThreadPlanProgress from "../../src/orchestration/ThreadPlanProgress.ts"; -import { makeSqlitePersistenceLive } from "../../src/persistence/Layers/Sqlite.ts"; -import { layerConfig as ServerPersistenceLive } from "../../src/persistence/Layers/Sqlite.ts"; -import { HubDatabase } from "../../src/persistence/Postgres/HubDatabase.ts"; -import { makeHubDatabase } from "../../src/persistence/Postgres/HubDatabaseLive.ts"; -import { RepositoryIdentityResolver } from "../../src/project/RepositoryIdentityResolver.ts"; - -const [sqliteFile, pgUrl, owner, threadArg, skipArg] = process.argv.slice(2); -if (!sqliteFile || !pgUrl || !owner || !threadArg) { - throw new Error("usage: compare-backends.ts [skip,]"); -} -const threadIds = threadArg.split(",").map((id) => ThreadId.make(id)); -// Threads the benchmark wrote to differ by design; leave them out of whole-model checks. -const skipThreads = new Set(skipArg ? skipArg.split(",") : []); -const out = (line: string) => process.stdout.write(`${line}\n`); - -const sqliteQuery = ManagedRuntime.make( - OrchestrationProjectionSnapshotQueryLive.pipe( - Layer.provide(ThreadBackgroundLiveness.layer), - Layer.provide(ThreadPlanProgress.layer), - Layer.provide( - Layer.succeed(RepositoryIdentityResolver, { resolve: () => Effect.succeed(null) }), - ), - Layer.provide(makeSqlitePersistenceLive(sqliteFile)), - Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-compare-" })), - Layer.provide(NodeServices.layer), - ), -); -// The hub reads through the same live layer as the server, selected by HubDatabase. -const pgQuery = ManagedRuntime.make( - OrchestrationProjectionSnapshotQueryLive.pipe( - Layer.provide(ThreadBackgroundLiveness.layer), - Layer.provide(ThreadPlanProgress.layer), - Layer.provide( - Layer.succeed(RepositoryIdentityResolver, { resolve: () => Effect.succeed(null) }), - ), - Layer.provide(ServerPersistenceLive), - Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-compare-" })), - Layer.provide(NodeServices.layer), - Layer.provide( - Layer.effect(HubDatabase, makeHubDatabase({ databaseUrl: pgUrl, tenantId: owner })), - ), - ), -); - -const both = async ( - read: (query: ProjectionSnapshotQuery["Service"]) => Effect.Effect, -) => { - const sqlite = await sqliteQuery.runPromise( - Effect.flatMap(Effect.service(ProjectionSnapshotQuery), read), - ); - const pg = await pgQuery.runPromise( - Effect.flatMap(Effect.service(ProjectionSnapshotQuery), read), - ); - return { sqlite, pg }; -}; - -// Drops fields that legitimately differ between the two stores after the -// benchmark appended events (sequences, max updatedAt) and the skipped threads. -const normalize = (value: unknown): unknown => - JSON.parse(JSON.stringify(value), (key, inner) => { - if (key === "snapshotSequence" || key === "threadSequence") return undefined; - if (Array.isArray(inner)) { - return inner.filter( - (item) => - !( - item && - typeof item === "object" && - skipThreads.has((item as { id?: string }).id ?? "") - ), - ); - } - return inner; - }); - -let failures = 0; -const check = (label: string, sqlite: unknown, pg: unknown, counts: string) => { - const left = JSON.stringify(normalize(sqlite)); - const right = JSON.stringify(normalize(pg)); - const equal = left === right; - if (!equal) failures += 1; - out(`${equal ? "equal" : "DIFFERENT"}: ${label} (${counts}; ${left.length} bytes)`); -}; - -const readModel = await both((query) => query.getCommandReadModel()); -check( - "getCommandReadModel", - { ...readModel.sqlite, updatedAt: undefined }, - { ...readModel.pg, updatedAt: undefined }, - `threads=${readModel.pg.threads.length}`, -); -const shell = await both((query) => query.getShellSnapshot()); -check( - "getShellSnapshot", - { ...shell.sqlite, updatedAt: undefined }, - { ...shell.pg, updatedAt: undefined }, - `threads=${shell.pg.threads.length}`, -); -const archived = await both((query) => query.getArchivedShellSnapshot()); -check( - "getArchivedShellSnapshot", - { ...archived.sqlite, updatedAt: undefined }, - { ...archived.pg, updatedAt: undefined }, - `threads=${archived.pg.threads.length}`, -); -const counts = await both((query) => query.getCounts()); -check("getCounts", counts.sqlite, counts.pg, `threads=${counts.pg.threadCount}`); - -for (const [index, threadId] of threadIds.entries()) { - const full = await both((query) => query.getThreadDetailSnapshot(threadId)); - const fullPg = Option.getOrThrow(full.pg).thread; - check( - `thread#${index + 1} full`, - full.sqlite, - full.pg, - `messages=${fullPg.messages.length} activities=${fullPg.activities.length}`, - ); - const byId = await both((query) => query.getThreadDetailById(threadId)); - check(`thread#${index + 1} getThreadDetailById`, byId.sqlite, byId.pg, "raw activities"); - // Walk every page with the client's page sizes and compare each page. - for (const turnLimit of [10, 3]) { - let window: OrchestrationThreadDetailWindow = { turnLimit }; - for (let page = 1; page <= 200; page += 1) { - const current = window; - const result = await both((query) => query.getThreadDetailSnapshot(threadId, current)); - const pgSnapshot = Option.getOrThrow(result.pg); - check( - `thread#${index + 1} turnLimit=${turnLimit} page ${page}`, - result.sqlite, - result.pg, - `messages=${pgSnapshot.thread.messages.length} activities=${pgSnapshot.thread.activities.length} hasMore=${pgSnapshot.page?.hasMore}`, - ); - const cursor = pgSnapshot.page?.beforeCursor; - if (!pgSnapshot.page?.hasMore || !cursor) break; - window = { turnLimit, beforeCursor: cursor }; - } - } -} - -const search = await both((query) => query.searchThreads({ query: "the", limit: 50 })); -check("searchThreads('the')", search.sqlite, search.pg, `matches=${search.pg.matches.length}`); -const searchUpper = await both((query) => query.searchThreads({ query: "THE", limit: 50 })); -check( - "searchThreads('THE')", - searchUpper.sqlite, - searchUpper.pg, - `matches=${searchUpper.pg.matches.length}`, -); - -out(failures === 0 ? "ALL EQUAL" : `${failures} DIFFERENCES`); -await pgQuery.dispose(); -await sqliteQuery.dispose(); diff --git a/apps/server/scripts/hub-proto/latency-proxy.ts b/apps/server/scripts/hub-proto/latency-proxy.ts deleted file mode 100644 index 52f0a850cf93..000000000000 --- a/apps/server/scripts/hub-proto/latency-proxy.ts +++ /dev/null @@ -1,31 +0,0 @@ -// TCP proxy that delays every chunk in both directions, to approximate the -// network round trip between a hub machine and PlanetScale. -// -// node scripts/hub-proto/latency-proxy.ts -// @effect-diagnostics globalTimers:off -import * as NodeNet from "node:net"; - -const [listenPort, targetPort, delayArg] = process.argv.slice(2).map(Number); -if (!listenPort || !targetPort || delayArg === undefined) { - throw new Error("usage: latency-proxy.ts "); -} -const delayMs = delayArg; - -const pipeDelayed = (from: NodeNet.Socket, to: NodeNet.Socket) => { - from.on("data", (chunk) => { - setTimeout(() => to.write(chunk), delayMs); - }); - from.on("end", () => setTimeout(() => to.end(), delayMs)); - from.on("error", () => to.destroy()); -}; - -NodeNet.createServer((client) => { - client.setNoDelay(true); - const upstream = NodeNet.connect({ host: "127.0.0.1", port: targetPort }, () => { - upstream.setNoDelay(true); - }); - pipeDelayed(client, upstream); - pipeDelayed(upstream, client); -}).listen(listenPort, "127.0.0.1", () => { - process.stdout.write(`latency proxy :${listenPort} -> :${targetPort} (+${delayMs}ms each way)\n`); -}); diff --git a/apps/server/scripts/hub-proto/migrate-temp-sqlite.ts b/apps/server/scripts/hub-proto/migrate-temp-sqlite.ts deleted file mode 100644 index 38f2958ebf50..000000000000 --- a/apps/server/scripts/hub-proto/migrate-temp-sqlite.ts +++ /dev/null @@ -1,18 +0,0 @@ -// Creates a freshly migrated SQLite database so the final schema can be -// inspected with `sqlite3 .schema`. Usage: node migrate-temp-sqlite.ts -import * as NodeServices from "@effect/platform-node/NodeServices"; -import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; - -import { makeSqlitePersistenceLive } from "../../src/persistence/Layers/Sqlite.ts"; - -const target = process.argv[2]; -if (!target) throw new Error("usage: migrate-temp-sqlite.ts "); - -await Effect.runPromise( - Layer.build(makeSqlitePersistenceLive(target)).pipe( - Effect.scoped, - Effect.provide(NodeServices.layer), - ), -); -process.stdout.write(`migrated ${target}\n`); diff --git a/prototype-2/ctl.sh b/prototype-2/ctl.sh deleted file mode 100755 index 9727858da18a..000000000000 --- a/prototype-2/ctl.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# ctl.sh start-hub|stop-hub|start-runner|stop-runner|status -# Stops only the process listening on our port after confirming its cwd. -set -uo pipefail -cd /tmp/proto-runner -ms() { echo $(( $(date +%s%N) / 1000000 )); } -port_pid() { ss -H -ltnp "sport = :$1" | grep -o 'pid=[0-9]*' | head -1 | cut -d= -f2; } -wait_http() { # port path - local t0=$(ms) - for i in $(seq 1 240); do - code=$(curl -s --max-time 2 -o /dev/null -w '%{http_code}' -H "authorization: Bearer $(cat hub-token 2>/dev/null)" "http://127.0.0.1:$1$2" 2>/dev/null) - [ "$code" = "200" ] && { echo "ready in $(( $(ms) - t0 )) ms"; return 0; } - sleep 0.25 - done - echo "not ready"; return 1 -} -stop_port() { # port expected-cwd - local pid=$(port_pid $1) - [ -z "$pid" ] && { echo "nothing on :$1"; return 0; } - [ "$(readlink /proc/$pid/cwd)" = "$2" ] || { echo "refusing: pid $pid cwd $(readlink /proc/$pid/cwd)"; return 1; } - local t0=$(ms); kill -TERM "$pid" - for i in $(seq 1 100); do kill -0 "$pid" 2>/dev/null || break; sleep 0.1; done - echo "stopped pid $pid on :$1 in $(( $(ms) - t0 )) ms" -} -case "$1" in - start-runner) setsid -f ./start-runner.sh /dev/null 2>&1; for i in $(seq 1 120); do [ -n "$(port_pid 4412)" ] && break; sleep 0.25; done; echo "runner pid $(port_pid 4412)";; - start-hub) t0=$(ms); setsid -f ./start-hub.sh /dev/null 2>&1; wait_http 4411 /api/orchestration/shell; echo "hub pid $(port_pid 4411) (start->ready $(( $(ms) - t0 )) ms)";; - stop-hub) stop_port 4411 /tmp/proto-runner/hub;; - stop-runner) stop_port 4412 /tmp/proto-runner/checkout;; - status) ss -H -ltnp | grep -E ":(4411|4412) " || echo "none";; -esac diff --git a/prototype-2/evidence/control-interrupt.out b/prototype-2/evidence/control-interrupt.out deleted file mode 100644 index 2eedc409bc95..000000000000 --- a/prototype-2/evidence/control-interrupt.out +++ /dev/null @@ -1,42 +0,0 @@ -[19:28:23.900] thread.turn.start accepted in 35 ms -[19:28:23.937] message user: "Count from 1 to 1500, writing each number on its own line as plain text. Do not use any tools." -[19:28:28.041] interrupting turn f0ad36d1-d1e6-464b-b924-f808ba4eeb96 -[19:28:28.471] activity context-window.updated: Context window updated -[19:28:28.471] activity checkpoint.captured: Checkpoint captured -[19:28:28.471] activity checkpoint.captured: Checkpoint captured -[19:28:28.471] message assistant: "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n21\n22\n23\n24\n25\n26\n27\n28\n29\n30\n31\n32\n33\n34\n35\n36\n37\n38\n39\n40\n41\n42\n43\n44\n45\n46\n47\n48\n49\n50\n51\n52\n53\n54\n55\n56\n5" -[19:28:28.471] turn f0ad36d1-d1e6-464b-b924-f808ba4eeb96 settled: completed after 4570 ms (first assistant text after 4570 ms) -{ - "latestTurn": { - "turnId": "f0ad36d1-d1e6-464b-b924-f808ba4eeb96", - "state": "completed", - "requestedAt": "2026-09-25T19:28:23.872Z", - "startedAt": "2026-09-25T19:28:23.872Z", - "completedAt": "2026-09-25T19:28:28.062Z", - "assistantMessageId": "assistant:f0ad36d1-d1e6-464b-b924-f808ba4eeb96" - }, - "session": { - "status": "stopped", - "activeTurnId": null, - "lastError": null - }, - "messageCount": 4, - "activityCount": 14, - "checkpoints": [ - { - "turnCount": 1, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWVhYWU4ZjVhLWFhOGUtNGY0ZC1iYzdhLTdiNmExNTNhNjEyNw/turn/1", - "files": [ - "notes.txt +1/-0" - ] - }, - { - "turnCount": 2, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWVhYWU4ZjVhLWFhOGUtNGY0ZC1iYzdhLTdiNmExNTNhNjEyNw/turn/2", - "files": [] - } - ], - "pendingApprovals": [] -} diff --git a/prototype-2/evidence/control-turn1.out b/prototype-2/evidence/control-turn1.out deleted file mode 100644 index 796f70c365b0..000000000000 --- a/prototype-2/evidence/control-turn1.out +++ /dev/null @@ -1,49 +0,0 @@ -[19:28:14.372] thread.turn.start accepted in 32 ms -[19:28:14.397] message user: "Append a new line 'edited by the remote runner' to the end of notes.txt in the current directory. Do not touch other files. Reply with just: done" -[19:28:16.883] activity tool.started: Tool call started -[19:28:17.295] activity tool.completed: Tool call -[19:28:17.295] activity tool.updated: Tool call -[19:28:17.295] activity context-window.updated: Context window updated -[19:28:18.527] activity tool.started: File change started -[19:28:19.346] activity tool.completed: File change -[19:28:19.347] activity context-window.updated: Context window updated -[19:28:20.164] activity tool.started: File change started -[19:28:20.983] activity tool.updated: File change -[19:28:20.983] activity approval.requested: File-change approval requested -[19:28:20.983] activity context-window.updated: Context window updated -[19:28:20.983] approving ac2b0e64-0b8a-49f5-a365-0802d8d4c90a (File-change approval requested: Edit: {"file_path":"/tmp/proto-runner/control-checkout/notes.txt","old_string":") -[19:28:20.996] approval dispatched in 13 ms -[19:28:21.406] activity approval.resolved: Approval resolved -[19:28:21.406] activity tool.completed: File change -[19:28:22.261] activity context-window.updated: Context window updated -[19:28:22.261] activity checkpoint.captured: Checkpoint captured -[19:28:22.261] message assistant: "done" -[19:28:22.261] turn 4e8d8ef0-ec2c-4dc9-a221-fe36aaacc587 settled: completed after 7889 ms (first assistant text after 7889 ms) -{ - "latestTurn": { - "turnId": "4e8d8ef0-ec2c-4dc9-a221-fe36aaacc587", - "state": "completed", - "requestedAt": "2026-09-25T19:28:14.346Z", - "startedAt": "2026-09-25T19:28:14.346Z", - "completedAt": "2026-09-25T19:28:21.959Z", - "assistantMessageId": "assistant:12925a11-744f-4c46-875f-b3e3a101cad6" - }, - "session": { - "status": "ready", - "activeTurnId": null, - "lastError": null - }, - "messageCount": 2, - "activityCount": 11, - "checkpoints": [ - { - "turnCount": 1, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWVhYWU4ZjVhLWFhOGUtNGY0ZC1iYzdhLTdiNmExNTNhNjEyNw/turn/1", - "files": [ - "notes.txt +1/-0" - ] - } - ], - "pendingApprovals": [] -} diff --git a/prototype-2/evidence/diff-0-1.out b/prototype-2/evidence/diff-0-1.out deleted file mode 100644 index 98e1dc6693a3..000000000000 --- a/prototype-2/evidence/diff-0-1.out +++ /dev/null @@ -1,7 +0,0 @@ -[19:17:31.083] getTurnDiff via hub in 121 ms -{ - "fromTurnCount": 0, - "toTurnCount": 1, - "threadId": "thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967", - "diff": "diff --git a/notes.txt b/notes.txt\nindex f31bc06..77957df 100644\n--- a/notes.txt\n+++ b/notes.txt\n@@ -1 +1,2 @@\n hello from the runner\n+edited by the remote runner\n" -} diff --git a/prototype-2/evidence/hub-delivery-log.txt b/prototype-2/evidence/hub-delivery-log.txt deleted file mode 100644 index fb5ccb390aed..000000000000 --- a/prototype-2/evidence/hub-delivery-log.txt +++ /dev/null @@ -1,350 +0,0 @@ -[19:15:29.359] INFO (#45): runner connected - bootId: 'boot-87901cd8-0c08-4338-9996-f7b99f3e14c9', - headSequence: 0, - runnerAckedSequence: 0, -[19:15:30.356] INFO (#203): runner event subscription opened - { afterSequence: 0 } -[19:20:05.283] INFO (#46): runner connected - bootId: 'boot-4e155bbc-f7b5-484c-8009-4922a3ae942f', - headSequence: 28, - runnerAckedSequence: 28, -[19:20:06.058] INFO (#90): runner event subscription opened - { afterSequence: 28 } -[19:22:29.856] INFO (#243): runner events acknowledged - throughSequence: 31, - deliveredCount: 3, - duplicateCount: 0, - connectCount: 1 -[19:22:30.414] INFO (#243): runner events acknowledged - throughSequence: 35, - deliveredCount: 7, - duplicateCount: 0, - connectCount: 1 -[19:22:32.993] INFO (#243): runner events acknowledged - throughSequence: 40, - deliveredCount: 12, - duplicateCount: 0, - connectCount: 1 -[19:23:22.101] INFO (#243): runner events acknowledged - throughSequence: 43, - deliveredCount: 15, - duplicateCount: 0, - connectCount: 1 -[19:23:22.673] INFO (#243): runner events acknowledged - throughSequence: 50, - deliveredCount: 22, - duplicateCount: 0, - connectCount: 1 -[19:23:24.226] INFO (#243): runner events acknowledged - throughSequence: 52, - deliveredCount: 24, - duplicateCount: 0, - connectCount: 1 -[19:23:24.782] INFO (#243): runner events acknowledged - throughSequence: 60, - deliveredCount: 32, - duplicateCount: 0, - connectCount: 1 -[19:23:25.836] INFO (#243): runner events acknowledged - throughSequence: 68, - deliveredCount: 40, - duplicateCount: 0, - connectCount: 1 -[19:23:26.390] INFO (#243): runner events acknowledged - throughSequence: 70, - deliveredCount: 42, - duplicateCount: 0, - connectCount: 1 -[19:23:54.470] INFO (#243): runner events acknowledged - throughSequence: 73, - deliveredCount: 45, - duplicateCount: 0, - connectCount: 1 -[19:23:55.524] INFO (#243): runner events acknowledged - throughSequence: 78, - deliveredCount: 52, - duplicateCount: 0, - connectCount: 1 -[19:23:56.077] INFO (#243): runner events acknowledged - throughSequence: 99, - deliveredCount: 73, - duplicateCount: 0, - connectCount: 1 -[19:23:56.733] INFO (#243): runner events acknowledged - throughSequence: 120, - deliveredCount: 93, - duplicateCount: 0, - connectCount: 1 -[19:23:57.387] INFO (#243): runner events acknowledged - throughSequence: 148, - deliveredCount: 122, - duplicateCount: 0, - connectCount: 1 -[19:23:57.943] INFO (#243): runner events acknowledged - throughSequence: 170, - deliveredCount: 143, - duplicateCount: 0, - connectCount: 1 -[19:23:58.498] INFO (#243): runner events acknowledged - throughSequence: 191, - deliveredCount: 164, - duplicateCount: 0, - connectCount: 1 -[19:23:59.054] INFO (#243): runner events acknowledged - throughSequence: 212, - deliveredCount: 186, - duplicateCount: 0, - connectCount: 1 -[19:23:59.607] INFO (#243): runner events acknowledged - throughSequence: 228, - deliveredCount: 200, - duplicateCount: 0, - connectCount: 1 -[19:24:09.684] INFO (#243): runner events acknowledged - throughSequence: 232, - deliveredCount: 204, - duplicateCount: 0, - connectCount: 1 -[19:24:11.239] INFO (#243): runner events acknowledged - throughSequence: 234, - deliveredCount: 206, - duplicateCount: 0, - connectCount: 1 -[19:24:11.793] INFO (#243): runner events acknowledged - throughSequence: 239, - deliveredCount: 211, - duplicateCount: 0, - connectCount: 1 -[19:24:12.848] INFO (#243): runner events acknowledged - throughSequence: 244, - deliveredCount: 216, - duplicateCount: 0, - connectCount: 1 -[19:24:23.417] INFO (#243): runner events acknowledged - throughSequence: 247, - deliveredCount: 219, - duplicateCount: 0, - connectCount: 1 -[19:24:24.470] INFO (#243): runner events acknowledged - throughSequence: 256, - deliveredCount: 230, - duplicateCount: 0, - connectCount: 1 -[19:24:25.022] INFO (#243): runner events acknowledged - throughSequence: 277, - deliveredCount: 251, - duplicateCount: 0, - connectCount: 1 -[19:24:25.576] INFO (#243): runner events acknowledged - throughSequence: 298, - deliveredCount: 272, - duplicateCount: 0, - connectCount: 1 -[19:24:26.128] INFO (#243): runner events acknowledged - throughSequence: 319, - deliveredCount: 293, - duplicateCount: 0, - connectCount: 1 -[19:24:26.682] INFO (#243): runner events acknowledged - throughSequence: 340, - deliveredCount: 314, - duplicateCount: 0, - connectCount: 1 -[19:24:27.234] INFO (#243): runner events acknowledged - throughSequence: 361, - deliveredCount: 335, - duplicateCount: 0, - connectCount: 1 -[19:24:27.787] INFO (#243): runner events acknowledged - throughSequence: 368, - deliveredCount: 340, - duplicateCount: 0, - connectCount: 1 -[19:24:52.865] INFO (#243): runner events acknowledged - throughSequence: 371, - deliveredCount: 343, - duplicateCount: 0, - connectCount: 1 -[19:24:53.920] INFO (#243): runner events acknowledged - throughSequence: 375, - deliveredCount: 347, - duplicateCount: 0, - connectCount: 1 -[19:24:56.975] INFO (#243): runner events acknowledged - throughSequence: 379, - deliveredCount: 351, - duplicateCount: 0, - connectCount: 1 -[19:25:26.826] INFO (#46): runner connected - bootId: 'boot-4e155bbc-f7b5-484c-8009-4922a3ae942f', - headSequence: 395, - runnerAckedSequence: 379, -[19:25:27.725] INFO (#90): runner event subscription opened - { afterSequence: 379 } -[19:25:28.285] INFO (#240): runner events acknowledged - throughSequence: 395, - deliveredCount: 16, - duplicateCount: 0, - connectCount: 1 -[19:25:28.865] INFO (#240): runner events acknowledged - throughSequence: 400, - deliveredCount: 21, - duplicateCount: 0, - connectCount: 1 -[19:25:29.927] INFO (#240): runner events acknowledged - throughSequence: 402, - deliveredCount: 23, - duplicateCount: 0, - connectCount: 1 -[19:25:30.555] INFO (#240): runner events acknowledged - throughSequence: 403, - deliveredCount: 26, - duplicateCount: 0, - connectCount: 1 -[19:25:31.117] INFO (#240): runner events acknowledged - throughSequence: 405, - deliveredCount: 26, - duplicateCount: 0, - connectCount: 1 -[19:25:53.280] WARN (#46): runner connection ended -[19:25:54.284] WARN (#46): runner connection ended -[19:25:55.288] WARN (#46): runner connection ended -[19:25:56.292] WARN (#46): runner connection ended -[19:25:57.311] INFO (#46): runner connected - bootId: 'boot-1b34a7ce-30b4-418a-8d97-7adb8eecf074', - headSequence: 405, - runnerAckedSequence: 405, -[19:25:57.311] INFO (#756): runner event subscription opened - { afterSequence: 405 } -[19:26:05.761] INFO (#240): runner events acknowledged - throughSequence: 408, - deliveredCount: 29, - duplicateCount: 0, - connectCount: 2 -[19:26:06.819] INFO (#240): runner events acknowledged - throughSequence: 412, - deliveredCount: 33, - duplicateCount: 0, - connectCount: 2 -[19:26:08.403] INFO (#240): runner events acknowledged - throughSequence: 418, - deliveredCount: 39, - duplicateCount: 0, - connectCount: 2 -[19:26:08.963] INFO (#240): runner events acknowledged - throughSequence: 420, - deliveredCount: 41, - duplicateCount: 0, - connectCount: 2 -[19:27:11.155] WARN (#46): runner connection ended -[19:27:12.178] INFO (#46): runner connected - bootId: 'boot-3d1e2e6c-b8d2-496f-8cdf-cb189cf657d2', - headSequence: 420, - runnerAckedSequence: 420, -[19:27:12.178] INFO (#1047): runner event subscription opened - { afterSequence: 420 } -[19:27:27.230] INFO (#240): runner events acknowledged - throughSequence: 423, - deliveredCount: 44, - duplicateCount: 0, - connectCount: 3 -[19:27:27.786] INFO (#240): runner events acknowledged - throughSequence: 427, - deliveredCount: 48, - duplicateCount: 0, - connectCount: 3 -[19:27:29.342] INFO (#240): runner events acknowledged - throughSequence: 431, - deliveredCount: 52, - duplicateCount: 0, - connectCount: 3 -[19:27:31.416] WARN (#46): runner connection ended -[19:27:32.419] WARN (#46): runner connection ended -[19:27:33.422] WARN (#46): runner connection ended -[19:27:34.441] INFO (#46): runner connected - bootId: 'boot-0d427cb0-d9a9-4c7a-9578-d5ca7b9d0bf4', - headSequence: 435, - runnerAckedSequence: 431, -[19:27:34.441] INFO (#1312): runner event subscription opened - { afterSequence: 431 } -[19:27:34.901] INFO (#240): runner events acknowledged - throughSequence: 435, - deliveredCount: 56, - duplicateCount: 0, - connectCount: 4 -[19:27:36.458] INFO (#240): runner events acknowledged - throughSequence: 438, - deliveredCount: 59, - duplicateCount: 0, - connectCount: 4 -[19:27:37.025] INFO (#240): runner events acknowledged - throughSequence: 440, - deliveredCount: 63, - duplicateCount: 0, - connectCount: 4 -[19:27:37.579] INFO (#240): runner events acknowledged - throughSequence: 442, - deliveredCount: 63, - duplicateCount: 0, - connectCount: 4 -[19:27:38.636] INFO (#240): runner events acknowledged - throughSequence: 449, - deliveredCount: 70, - duplicateCount: 0, - connectCount: 4 -[19:27:39.690] INFO (#240): runner events acknowledged - throughSequence: 450, - deliveredCount: 71, - duplicateCount: 0, - connectCount: 4 -[19:27:40.258] INFO (#240): runner events acknowledged - throughSequence: 451, - deliveredCount: 74, - duplicateCount: 0, - connectCount: 4 -[19:27:40.814] INFO (#240): runner events acknowledged - throughSequence: 458, - deliveredCount: 79, - duplicateCount: 0, - connectCount: 4 -[19:27:41.381] INFO (#240): runner events acknowledged - throughSequence: 460, - deliveredCount: 82, - duplicateCount: 0, - connectCount: 4 -[19:27:41.937] INFO (#240): runner events acknowledged - throughSequence: 463, - deliveredCount: 84, - duplicateCount: 0, - connectCount: 4 -[19:31:41.091] INFO (#46): runner connected - bootId: 'boot-e6ca177c-997c-49a2-aecf-1acdba8c445f', - headSequence: 464, - runnerAckedSequence: 463, -[19:31:41.895] INFO (#90): runner event subscription opened - { afterSequence: 463 } -[19:31:42.452] INFO (#242): runner events acknowledged - throughSequence: 464, - deliveredCount: 1, - duplicateCount: 0, - connectCount: 1 -[19:31:50.521] INFO (#242): runner events acknowledged - throughSequence: 467, - deliveredCount: 4, - duplicateCount: 0, - connectCount: 1 -[19:31:51.076] INFO (#242): runner events acknowledged - throughSequence: 471, - deliveredCount: 8, - duplicateCount: 0, - connectCount: 1 -[19:31:51.632] INFO (#242): runner events acknowledged - throughSequence: 472, - deliveredCount: 9, - duplicateCount: 0, - connectCount: 1 -[19:31:52.686] INFO (#242): runner events acknowledged - throughSequence: 477, - deliveredCount: 14, - duplicateCount: 0, - connectCount: 1 diff --git a/prototype-2/evidence/hub-restart-test.out b/prototype-2/evidence/hub-restart-test.out deleted file mode 100644 index 3aac3b705790..000000000000 --- a/prototype-2/evidence/hub-restart-test.out +++ /dev/null @@ -1,11 +0,0 @@ -outbox head before turn: 368 -[19:24:56.968] command started on runner; stopping hub -stopped pid 3600897 on :4411 in 107 ms -[19:24:57.126] hub down; outbox head 379 -[19:25:20.363] approval requested while hub down: -395 19:25:20.148 request.opened file_change_approval -[19:25:20.590] outbox head 395; restarting hub -ready in 8574 ms -hub pid 3609589 (start->ready 8675 ms) -scenario exit 0 -events produced by the runner while the hub was down: 379..405 diff --git a/prototype-2/evidence/hub-strace-all-lifetimes.txt b/prototype-2/evidence/hub-strace-all-lifetimes.txt deleted file mode 100644 index aab015070205..000000000000 --- a/prototype-2/evidence/hub-strace-all-lifetimes.txt +++ /dev/null @@ -1,32 +0,0 @@ -3594265 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594266, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3594265 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594267, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3594265 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594268, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3594265 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594269, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3594265 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594270, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3594186 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594265, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3594186 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3594279, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3594186 --- SIGTERM {si_signo=SIGTERM, si_code=SI_USER, si_pid=3600359, si_uid=1000} --- -3601072 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601073, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3601072 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601074, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3601072 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601075, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3601072 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601076, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3601072 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601077, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3600897 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601072, si_uid=1000, si_status=0, si_utime=0, si_stime=1 /* 0.01 s */} --- -3600897 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3601082, si_uid=1000, si_status=0, si_utime=0, si_stime=1 /* 0.01 s */} --- -3600897 --- SIGTERM {si_signo=SIGTERM, si_code=SI_USER, si_pid=3608476, si_uid=1000} --- -3609858 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609860, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3609858 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609862, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3609858 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609864, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3609858 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609865, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3609858 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609866, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3609589 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609858, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3609589 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3609885, si_uid=1000, si_status=0, si_utime=0, si_stime=1 /* 0.01 s */} --- -3609589 --- SIGTERM {si_signo=SIGTERM, si_code=SI_USER, si_pid=3616263, si_uid=1000} --- -3622473 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622476, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3622473 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622478, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3622473 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622480, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3622473 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622481, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3622473 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622483, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3622218 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622473, si_uid=1000, si_status=0, si_utime=0, si_stime=0} --- -3622218 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=3622493, si_uid=1000, si_status=0, si_utime=0, si_stime=1 /* 0.01 s */} --- -3622218 --- SIGTERM {si_signo=SIGTERM, si_code=SI_USER, si_pid=3623549, si_uid=1000} --- diff --git a/prototype-2/evidence/outbox-summary.txt b/prototype-2/evidence/outbox-summary.txt deleted file mode 100644 index edd189c15a6a..000000000000 --- a/prototype-2/evidence/outbox-summary.txt +++ /dev/null @@ -1,14 +0,0 @@ -Runner outbox (T3CODE_RUNNER_OUTBOX_KEEP=1 from the second runner boot on) -sequences 29 .. 478 = 450 events; runner boots: 5 -(session seq 29-478 boot#2 {"session.started":7,"session.configured":9,"session.state.changed":9,"thread.started":7,"session.exited":5,"task.completed":2,"task.updated":1} -646f7959 seq 32-39 boot#2 {"turn.started":1,"session.configured":1,"session.state.changed":1,"item.started":1,"item.completed":1,"thread.token-usage.updated":1,"turn.completed":1} -f4f2e0ef seq 46-48 boot#2 {"turn.started":1,"session.configured":1,"turn.completed":1} -a1aa7576 seq 51-70 boot#2 {"turn.started":1,"item.started":1,"item.updated":2,"thread.token-usage.updated":3,"account.rate-limits.updated":1,"task.started":1,"content.delta":7,"item.completed":2,"session.state.changed":1,"turn.completed":1} -91f83731 seq 71-228 boot#2 {"turn.started":1,"session.configured":1,"session.state.changed":1,"content.delta":151,"item.completed":1,"thread.token-usage.updated":2,"turn.completed":1} -505bb185 seq 233-244 boot#2 {"turn.started":1,"item.started":1,"item.updated":2,"thread.token-usage.updated":3,"item.completed":2,"session.state.changed":1,"content.delta":1,"turn.completed":1} -01b724b4 seq 245-367 boot#2 {"turn.started":1,"session.configured":1,"session.state.changed":1,"content.delta":117,"item.completed":1,"thread.token-usage.updated":1,"turn.completed":1} -e3457ec4 seq 372-405 boot#2 {"turn.started":1,"session.configured":1,"session.state.changed":4,"item.started":3,"item.updated":6,"thread.token-usage.updated":5,"account.rate-limits.updated":1,"task.started":1,"task.completed":1,"content.delta":3,"item.completed":4,"request.opened":1,"request.resolved":1,"turn.completed":1} -3aec0919 seq 409-420 boot#3 {"turn.started":1,"session.configured":1,"session.state.changed":1,"content.delta":3,"item.completed":1,"thread.token-usage.updated":2,"account.rate-limits.updated":1,"turn.completed":1} -b53e0583 seq 424-434 boot#4 {"turn.started":1,"session.configured":1,"session.state.changed":1,"item.started":1,"item.updated":1,"thread.token-usage.updated":2,"account.rate-limits.updated":1,"item.completed":1,"turn.completed":1} -5361dcda seq 439-463 boot#5 {"turn.started":1,"session.configured":1,"session.state.changed":3,"item.started":2,"item.updated":4,"thread.token-usage.updated":4,"account.rate-limits.updated":1,"item.completed":3,"request.opened":1,"request.resolved":1,"content.delta":2,"turn.completed":1} -dcc58fc1 seq 468-477 boot#6 {"turn.started":1,"session.configured":1,"session.state.changed":1,"account.rate-limits.updated":1,"content.delta":1,"item.completed":1,"thread.token-usage.updated":2,"turn.completed":1} diff --git a/prototype-2/evidence/runner-restart-midturn-test.out b/prototype-2/evidence/runner-restart-midturn-test.out deleted file mode 100644 index 341188389793..000000000000 --- a/prototype-2/evidence/runner-restart-midturn-test.out +++ /dev/null @@ -1,17 +0,0 @@ -outbox head before turn: 420 -[19:27:29.383] command running on runner; stopping runner -stopped pid 3613078 on :4412 in 412 ms -[19:27:29.812] events outboxed during drain: -432 19:27:29.407 item.completed command_execution -433 19:27:29.407 thread.token-usage.updated -434 19:27:29.407 turn.completed interrupted -435 19:27:29.408 session.exited -[19:27:31.860] starting runner -runner pid 3613921 -scenario exit 0 -[19:27:36.184] follow-up turn -follow-up exit 0 -hello from the runner -edited by the remote runner -after hub restart -after runner restart diff --git a/prototype-2/evidence/runner-restart-test.out b/prototype-2/evidence/runner-restart-test.out deleted file mode 100644 index 849b18cd2f1c..000000000000 --- a/prototype-2/evidence/runner-restart-test.out +++ /dev/null @@ -1,16 +0,0 @@ -[19:25:51.248] stopping runner -stopped pid 3600783 on :4412 in 107 ms -[19:25:54.376] starting runner -runner pid 3610610 -791- } -792:[19:25:56.292] WARN (#46): runner connection ended -793- { -794- cause: 'Cause([Fail(~effect/rpc/RpcClientError: SocketOpenError: An error occurred during Open)])' -795- } -796:[19:25:57.311] INFO (#46): runner connected -797- { -798- runnerId: 'runner-f44d1a25-700d-4f1d-859c-85956c39170f', -799- bootId: 'boot-1b34a7ce-30b4-418a-8d97-7adb8eecf074', --- -804:[19:25:57.311] INFO (#756): runner event subscription opened -805- { afterSequence: 405 } diff --git a/prototype-2/evidence/runner-rpc-log.txt b/prototype-2/evidence/runner-rpc-log.txt deleted file mode 100644 index 1e0cd9fbb118..000000000000 --- a/prototype-2/evidence/runner-rpc-log.txt +++ /dev/null @@ -1,258 +0,0 @@ -75:[19:22:29.478] INFO (#149): runner rpc startSession -76- { -77- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -78- cwd: '/tmp/proto-runner/checkout', -79- runtimeMode: 'approval-required', -80- resumeCursor: { -81- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -82- resume: '202a75b8-d54d-4c76-8507-3cfe0c931870', -83- resumeSessionAt: '1c0876b2-ef23-4bde-bd5c-f78bc34e859c', -89:[19:22:30.060] INFO (#188): runner rpc sendTurn -90- { -91- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -92- ok: true, -93- ms: 464 -94- } -95:[19:22:32.715] INFO (#236): runner rpc interruptTurn -96- { -97- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -98- turnId: undefined, -99- ok: true, -100- ms: 3 -101- } -102:[19:22:32.881] INFO (#270): runner rpc checkpoint.capture -103- { -104- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2', -105- ok: true, -106- ms: 42 -107- } -108:[19:22:33.015] INFO (#316): runner rpc checkpoint.capture -109- { -110- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2', -111- ok: true, -112- ms: 57 -113- } -114:[19:23:21.853] INFO (#371): runner rpc startSession -115- { -116- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -117- cwd: '/tmp/proto-runner/checkout', -118- runtimeMode: 'approval-required', -119- resumeCursor: { -120- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -121- resume: '202a75b8-d54d-4c76-8507-3cfe0c931870', -122- resumeSessionAt: '1c0876b2-ef23-4bde-bd5c-f78bc34e859c', -128:[19:23:22.380] INFO (#408): runner rpc sendTurn -129- { -130- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -131- ok: true, -132- ms: 457 -133- } -134:[19:23:22.588] INFO (#462): runner rpc checkpoint.capture -135- { -136- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/3', -137- ok: true, -138- ms: 57 -139- } -140:[19:23:25.992] INFO (#596): runner rpc checkpoint.capture -141- { -142- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/4', -143- ok: true, -144- ms: 34 -145- } -146:[19:23:53.948] INFO (#659): runner rpc sendTurn -147- { -148- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -149- ok: true, -150- ms: 20 -151- } -152:[19:23:59.614] INFO (#1057): runner rpc checkpoint.capture -153- { -154- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/5', -155- ok: true, -156- ms: 58 -157- } -158:[19:24:12.740] INFO (#1199): runner rpc checkpoint.capture -159- { -160- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/6', -161- ok: true, -162- ms: 36 -163- } -164:[19:24:23.191] INFO (#1259): runner rpc sendTurn -165- { -166- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -167- ok: true, -168- ms: 25 -169- } -170:[19:24:27.277] INFO (#1560): runner rpc interruptTurn -171- { -172- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -173- turnId: undefined, -174- ok: true, -175- ms: 2 -176- } -177:[19:24:27.443] INFO (#1594): runner rpc checkpoint.capture -178- { -179- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7', -180- ok: true, -181- ms: 34 -182- } -183:[19:24:27.518] INFO (#1640): runner rpc checkpoint.capture -184- { -185- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7', -186- ok: true, -187- ms: 34 -188- } -189:[19:24:52.741] INFO (#1690): runner rpc startSession -190- { -191- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -192- cwd: '/tmp/proto-runner/checkout', -193- runtimeMode: 'approval-required', -194- resumeCursor: { -195- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -196- resume: '202a75b8-d54d-4c76-8507-3cfe0c931870', -197- resumeSessionAt: '0d126ad2-ba4e-4ccb-962f-ce368ec2a5dc', -203:[19:24:53.474] INFO (#1727): runner rpc sendTurn -204- { -205- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -206- ok: true, -207- ms: 660 -208- } -209:[19:25:28.372] INFO (#1878): runner rpc respondToRequest -210- { -211- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -212- requestId: '1ec080d3-2abc-4a2a-948a-d0d9f4afe95e', -213- decision: 'accept', -214- ok: true, -215- ms: 0 -216- } -217:[19:25:30.821] INFO (#1936): runner rpc checkpoint.capture -218- { -219- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/8', -220- ok: true, -221- ms: 164 -222- } -223-(node:3610610) ExperimentalWarning: SQLite is an experimental feature and might change at any time -224-(Use `node --trace-warnings ...` to show where the warning was created) -225-[19:25:56.633] INFO (#7): runner outbox loaded -237:[19:26:05.065] INFO (#477): runner rpc startSession -238- { -239- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -240- cwd: '/tmp/proto-runner/checkout', -241- runtimeMode: 'approval-required', -242- resumeCursor: { -243- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -244- resume: '202a75b8-d54d-4c76-8507-3cfe0c931870', -245- resumeSessionAt: '3c54100f-d025-4ed4-9525-5f507a67c8bf', -251:[19:26:06.398] INFO (#514): runner rpc sendTurn -252- { -253- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -254- ok: true, -255- ms: 1045 -256- } -257:[19:26:08.915] INFO (#598): runner rpc checkpoint.capture -258- { -259- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/9', -260- ok: true, -261- ms: 74 -262- } -263-(node:3613078) ExperimentalWarning: SQLite is an experimental feature and might change at any time -264-(Use `node --trace-warnings ...` to show where the warning was created) -265-[19:27:11.790] INFO (#7): runner outbox loaded -277:[19:27:26.794] INFO (#493): runner rpc startSession -278- { -279- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -280- cwd: '/tmp/proto-runner/checkout', -281- runtimeMode: 'approval-required', -282- resumeCursor: { -283- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -284- resume: '202a75b8-d54d-4c76-8507-3cfe0c931870', -285- resumeSessionAt: '3c54100f-d025-4ed4-9525-5f507a67c8bf', -291:[19:27:27.344] INFO (#528): runner rpc sendTurn -292- { -293- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -294- ok: true, -295- ms: 489 -296- } -297-[19:27:29.708] INFO (#593): runner drained provider sessions into the outbox -298- { -299- runnerId: 'runner-f44d1a25-700d-4f1d-859c-85956c39170f', -320:[19:27:34.611] INFO (#525): runner rpc checkpoint.capture -321- { -322- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/10', -323- ok: true, -324- ms: 45 -325- } -326:[19:27:34.730] INFO (#572): runner rpc checkpoint.capture -327- { -328- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/10', -329- ok: true, -330- ms: 54 -331- } -332:[19:27:36.326] INFO (#617): runner rpc startSession -333- { -334- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -335- cwd: '/tmp/proto-runner/checkout', -336- runtimeMode: 'approval-required', -337- resumeCursor: { -338- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -339- resume: '202a75b8-d54d-4c76-8507-3cfe0c931870', -340- resumeSessionAt: '3c54100f-d025-4ed4-9525-5f507a67c8bf', -346:[19:27:36.868] INFO (#643): runner rpc sendTurn -347- { -348- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -349- ok: true, -350- ms: 485 -351- } -352:[19:27:40.470] INFO (#763): runner rpc respondToRequest -353- { -354- threadId: 'thread-c9fc73ef-a8fb-4e75-993f-2aa2987e0967', -355- requestId: 'fed8ec9c-d62b-4f8b-97dc-a1cf6c869b7f', -356- decision: 'accept', -357- ok: true, -358- ms: 0 -359- } -360:[19:27:41.549] INFO (#815): runner rpc checkpoint.capture -361- { -362- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/11', -363- ok: true, -364- ms: 32 -365- } -366-[19:28:57.900] INFO (#876): runner drained provider sessions into the outbox -367- { -368- runnerId: 'runner-f44d1a25-700d-4f1d-859c-85956c39170f', -389:[19:31:50.070] INFO (#571): runner rpc startSession -390- { -391- threadId: 'thread-title-f0b10a47-12ec-40e2-b958-683f1b38d6fa', -392- cwd: '/tmp/proto-runner/checkout', -393- runtimeMode: 'approval-required', -394- resumeCursor: null, -395- ok: true, -396- ms: 9 -397- } -398:[19:31:50.188] INFO (#606): runner rpc checkpoint.capture -399- { -400- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLXRpdGxlLWYwYjEwYTQ3LTEyZWMtNDBlMi1iOTU4LTY4M2YxYjM4ZDZmYQ/turn/0', -401- ok: true, -402- ms: 45 -403- } -404:[19:31:50.596] INFO (#616): runner rpc sendTurn -405- { -406- threadId: 'thread-title-f0b10a47-12ec-40e2-b958-683f1b38d6fa', -407- ok: true, -408- ms: 438 -409- } -410:[19:31:52.642] INFO (#718): runner rpc checkpoint.capture -411- { -412- checkpointRef: 'refs/t3/checkpoints/dGhyZWFkLXRpdGxlLWYwYjEwYTQ3LTEyZWMtNDBlMi1iOTU4LTY4M2YxYjM4ZDZmYQ/turn/1', -413- ok: true, -414- ms: 33 -415- } -416:[19:32:04.201] INFO (#560): runner rpc text.generateThreadTitle -417- { -418- cwd: '/tmp/proto-runner/checkout', -419- model: 'claude-haiku-4-5', -420- ok: true, -421- ms: 14161 -422- } -423-[19:32:21.384] INFO (#769): runner drained provider sessions into the outbox -424- { diff --git a/prototype-2/evidence/turn1.out b/prototype-2/evidence/turn1.out deleted file mode 100644 index 57d0e2da5803..000000000000 --- a/prototype-2/evidence/turn1.out +++ /dev/null @@ -1,45 +0,0 @@ -[19:17:17.733] thread.turn.start accepted in 40 ms -[19:17:17.754] message user: "Append a new line 'edited by the remote runner' to the end of notes.txt in the current directory. Do not touch other files. Reply with just: done" -[19:17:21.057] activity tool.started: Tool call started -[19:17:21.468] activity context-window.updated: Context window updated -[19:17:21.468] activity tool.completed: Tool call -[19:17:22.695] activity tool.started: File change started -[19:17:23.513] activity tool.updated: File change -[19:17:23.513] activity approval.requested: File-change approval requested -[19:17:23.513] activity context-window.updated: Context window updated -[19:17:23.513] approving de4acf9b-62b1-42ef-8302-ff2109bf7f7e (File-change approval requested: Edit: {"file_path":"/tmp/proto-runner/checkout/notes.txt","old_string":"hello fr) -[19:17:23.532] approval dispatched in 19 ms -[19:17:23.941] activity approval.resolved: Approval resolved -[19:17:23.941] activity tool.completed: File change -[19:17:24.782] activity context-window.updated: Context window updated -[19:17:24.782] message assistant: "done" -[19:17:25.194] activity checkpoint.captured: Checkpoint captured -[19:17:25.194] turn 8ff8671b-3ab7-4f43-8bc4-5b56d1983e2a settled: completed after 7460 ms (first assistant text after 7048 ms) -{ - "latestTurn": { - "turnId": "8ff8671b-3ab7-4f43-8bc4-5b56d1983e2a", - "state": "completed", - "requestedAt": "2026-09-25T19:17:17.702Z", - "startedAt": "2026-09-25T19:17:17.702Z", - "completedAt": "2026-09-25T19:17:24.588Z", - "assistantMessageId": "assistant:80ac5a42-e09e-42b6-b17d-cd7ce029b03b" - }, - "session": { - "status": "ready", - "activeTurnId": null, - "lastError": null - }, - "messageCount": 2, - "activityCount": 8, - "checkpoints": [ - { - "turnCount": 1, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/1", - "files": [ - "notes.txt +1/-0" - ] - } - ], - "pendingApprovals": [] -} diff --git a/prototype-2/evidence/turn10-continue.out b/prototype-2/evidence/turn10-continue.out deleted file mode 100644 index 965fac76080c..000000000000 --- a/prototype-2/evidence/turn10-continue.out +++ /dev/null @@ -1,110 +0,0 @@ -[19:27:36.288] thread.turn.start accepted in 24 ms -[19:27:36.318] message user: "Your previous task was cut off by a machine restart. Finish it now: make sure notes.txt ends with the line 'after runner restart' (add it only if missing). Repl" -[19:27:38.385] activity tool.started: Tool call started -[19:27:38.798] activity context-window.updated: Context window updated -[19:27:38.799] activity tool.completed: Tool call -[19:27:39.622] activity tool.started: File change started -[19:27:40.446] activity tool.updated: File change -[19:27:40.446] activity approval.requested: File-change approval requested -[19:27:40.446] activity context-window.updated: Context window updated -[19:27:40.447] approving fed8ec9c-d62b-4f8b-97dc-a1cf6c869b7f (File-change approval requested: Edit: {"file_path":"/tmp/proto-runner/checkout/notes.txt","old_string":"hello fr) -[19:27:40.464] approval dispatched in 17 ms -[19:27:40.878] activity approval.resolved: Approval resolved -[19:27:40.878] activity tool.completed: File change -[19:27:40.878] activity tool.updated: File change -[19:27:41.704] activity checkpoint.captured: Checkpoint captured -[19:27:41.704] activity context-window.updated: Context window updated -[19:27:41.704] message assistant: "done" -[19:27:41.704] turn 5361dcda-bf7d-45da-9a15-a3a4347b117e settled: completed after 5415 ms (first assistant text after 5415 ms) -{ - "latestTurn": { - "turnId": "5361dcda-bf7d-45da-9a15-a3a4347b117e", - "state": "completed", - "requestedAt": "2026-09-25T19:27:36.271Z", - "startedAt": "2026-09-25T19:27:36.271Z", - "completedAt": "2026-09-25T19:27:41.453Z", - "assistantMessageId": "assistant:4a8ac405-4358-44b2-bc48-e438e8d9e683" - }, - "session": { - "status": "ready", - "activeTurnId": null, - "lastError": null - }, - "messageCount": 17, - "activityCount": 61, - "checkpoints": [ - { - "turnCount": 1, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/1", - "files": [ - "notes.txt +1/-0" - ] - }, - { - "turnCount": 2, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2", - "files": [] - }, - { - "turnCount": 3, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/3", - "files": [] - }, - { - "turnCount": 4, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/4", - "files": [] - }, - { - "turnCount": 5, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/5", - "files": [] - }, - { - "turnCount": 6, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/6", - "files": [] - }, - { - "turnCount": 7, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7", - "files": [] - }, - { - "turnCount": 8, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/8", - "files": [ - "notes.txt +1/-0" - ] - }, - { - "turnCount": 9, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/9", - "files": [] - }, - { - "turnCount": 10, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/10", - "files": [] - }, - { - "turnCount": 11, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/11", - "files": [ - "notes.txt +1/-0" - ] - } - ], - "pendingApprovals": [] -} diff --git a/prototype-2/evidence/turn11-title.out b/prototype-2/evidence/turn11-title.out deleted file mode 100644 index 890037894e39..000000000000 --- a/prototype-2/evidence/turn11-title.out +++ /dev/null @@ -1,33 +0,0 @@ -[19:31:50.022] thread.turn.start accepted in 32 ms -[19:31:50.037] message user: "What is the capital of France? Reply with one word." -[19:31:52.517] activity context-window.updated: Context window updated -[19:31:52.517] message assistant: "Paris." -[19:31:52.929] activity checkpoint.captured: Checkpoint captured -[19:31:52.929] activity context-window.updated: Context window updated -[19:31:52.930] turn dcc58fc1-9dd8-4e48-9db9-a2ea5c469161 settled: completed after 2907 ms (first assistant text after 2494 ms) -{ - "latestTurn": { - "turnId": "dcc58fc1-9dd8-4e48-9db9-a2ea5c469161", - "state": "completed", - "requestedAt": "2026-09-25T19:31:49.996Z", - "startedAt": "2026-09-25T19:31:49.996Z", - "completedAt": "2026-09-25T19:31:52.541Z", - "assistantMessageId": "assistant:fdb5731a-3fff-4a3e-b7b5-eda167439390" - }, - "session": { - "status": "ready", - "activeTurnId": null, - "lastError": null - }, - "messageCount": 2, - "activityCount": 2, - "checkpoints": [ - { - "turnCount": 1, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLXRpdGxlLWYwYjEwYTQ3LTEyZWMtNDBlMi1iOTU4LTY4M2YxYjM4ZDZmYQ/turn/1", - "files": [] - } - ], - "pendingApprovals": [] -} diff --git a/prototype-2/evidence/turn6-interrupt.out b/prototype-2/evidence/turn6-interrupt.out deleted file mode 100644 index 2d6dec25f2b3..000000000000 --- a/prototype-2/evidence/turn6-interrupt.out +++ /dev/null @@ -1,72 +0,0 @@ -[19:24:23.128] thread.turn.start accepted in 26 ms -[19:24:23.147] message user: "Count from 1 to 1500, writing each number on its own line as plain text. Do not use any tools." -[19:24:27.260] interrupting turn 01b724b4-5435-4a80-ae29-49dbcacbabb0 -[19:24:27.685] activity context-window.updated: Context window updated -[19:24:27.685] activity checkpoint.captured: Checkpoint captured -[19:24:27.685] activity checkpoint.captured: Checkpoint captured -[19:24:27.685] message assistant: "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n21\n22\n23\n24\n25\n26\n27\n28\n29\n30\n31\n32\n33\n34\n35\n36\n37\n38\n39\n40\n41\n42\n43\n44\n45\n46\n47\n48\n49\n50\n51\n52\n53\n54\n55\n56\n5" -[19:24:27.685] turn 01b724b4-5435-4a80-ae29-49dbcacbabb0 settled: completed after 4556 ms (first assistant text after 4556 ms) -{ - "latestTurn": { - "turnId": "01b724b4-5435-4a80-ae29-49dbcacbabb0", - "state": "completed", - "requestedAt": "2026-09-25T19:24:23.108Z", - "startedAt": "2026-09-25T19:24:23.108Z", - "completedAt": "2026-09-25T19:24:27.276Z", - "assistantMessageId": "assistant:fd4dfdf7-ef55-4530-a742-4b1be6390a7c" - }, - "session": { - "status": "stopped", - "activeTurnId": null, - "lastError": null - }, - "messageCount": 10, - "activityCount": 33, - "checkpoints": [ - { - "turnCount": 1, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/1", - "files": [ - "notes.txt +1/-0" - ] - }, - { - "turnCount": 2, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2", - "files": [] - }, - { - "turnCount": 3, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/3", - "files": [] - }, - { - "turnCount": 4, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/4", - "files": [] - }, - { - "turnCount": 5, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/5", - "files": [] - }, - { - "turnCount": 6, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/6", - "files": [] - }, - { - "turnCount": 7, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7", - "files": [] - } - ], - "pendingApprovals": [] -} diff --git a/prototype-2/evidence/turn7-hub-restart.out b/prototype-2/evidence/turn7-hub-restart.out deleted file mode 100644 index 5a1c784ffcdc..000000000000 --- a/prototype-2/evidence/turn7-hub-restart.out +++ /dev/null @@ -1,124 +0,0 @@ -[19:24:52.710] thread.turn.start accepted in 25 ms -[19:24:52.746] message user: "Run the Bash command 'sleep 20' in the foreground (do not run it in the background). When it has finished, append the line 'after hub restart' to notes.txt. Rep" -[19:24:56.878] activity tool.started: Command run started -[19:24:57.281] hub unreachable: fetch failed -[19:24:58.283] hub unreachable: fetch failed -[19:24:59.285] hub unreachable: fetch failed -[19:25:00.286] hub unreachable: fetch failed -[19:25:01.291] hub unreachable: fetch failed -[19:25:02.293] hub unreachable: fetch failed -[19:25:03.296] hub unreachable: fetch failed -[19:25:04.298] hub unreachable: fetch failed -[19:25:05.300] hub unreachable: fetch failed -[19:25:06.302] hub unreachable: fetch failed -[19:25:07.303] hub unreachable: fetch failed -[19:25:08.304] hub unreachable: fetch failed -[19:25:09.306] hub unreachable: fetch failed -[19:25:10.308] hub unreachable: fetch failed -[19:25:11.310] hub unreachable: fetch failed -[19:25:12.312] hub unreachable: fetch failed -[19:25:13.314] hub unreachable: fetch failed -[19:25:14.316] hub unreachable: fetch failed -[19:25:15.318] hub unreachable: fetch failed -[19:25:16.320] hub unreachable: fetch failed -[19:25:17.321] hub unreachable: fetch failed -[19:25:18.323] hub unreachable: fetch failed -[19:25:19.325] hub unreachable: fetch failed -[19:25:20.327] hub unreachable: fetch failed -[19:25:21.328] hub unreachable: fetch failed -[19:25:22.334] hub unreachable: fetch failed -[19:25:23.336] hub unreachable: fetch failed -[19:25:24.338] hub unreachable: fetch failed -[19:25:25.340] hub unreachable: fetch failed -[19:25:26.342] hub unreachable: fetch failed -[19:25:27.398] activity tool.updated: Command run -[19:25:27.398] activity context-window.updated: Context window updated -[19:25:27.873] activity task.started: local_bash task started -[19:25:27.873] activity task.completed: Task completed -[19:25:28.306] activity tool.completed: Command run -[19:25:28.306] activity tool.started: Tool call started -[19:25:28.306] activity tool.completed: Tool call -[19:25:28.306] activity tool.started: File change started -[19:25:28.306] activity tool.updated: File change -[19:25:28.306] activity context-window.updated: Context window updated -[19:25:28.306] activity approval.requested: File-change approval requested -[19:25:28.306] approving 1ec080d3-2abc-4a2a-948a-d0d9f4afe95e (File-change approval requested: Edit: {"file_path":"/tmp/proto-runner/checkout/notes.txt","old_string":"hello fr) -[19:25:28.352] approval dispatched in 46 ms -[19:25:28.765] activity approval.resolved: Approval resolved -[19:25:28.766] activity tool.completed: File change -[19:25:30.046] activity context-window.updated: Context window updated -[19:25:30.046] message assistant: "done" -[19:25:30.957] activity context-window.updated: Context window updated -[19:25:30.957] turn e3457ec4-8d3f-493b-ae23-3d56fae7b69d settled: completed after 38246 ms (first assistant text after 37335 ms) -{ - "latestTurn": { - "turnId": "e3457ec4-8d3f-493b-ae23-3d56fae7b69d", - "state": "completed", - "requestedAt": "2026-09-25T19:24:52.691Z", - "startedAt": "2026-09-25T19:24:52.691Z", - "completedAt": "2026-09-25T19:25:30.482Z", - "assistantMessageId": "assistant:fc0426b0-ff22-4fb1-a3d6-aaa2e4d8f2a3" - }, - "session": { - "status": "ready", - "activeTurnId": null, - "lastError": null - }, - "messageCount": 12, - "activityCount": 45, - "checkpoints": [ - { - "turnCount": 1, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/1", - "files": [ - "notes.txt +1/-0" - ] - }, - { - "turnCount": 2, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2", - "files": [] - }, - { - "turnCount": 3, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/3", - "files": [] - }, - { - "turnCount": 4, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/4", - "files": [] - }, - { - "turnCount": 5, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/5", - "files": [] - }, - { - "turnCount": 6, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/6", - "files": [] - }, - { - "turnCount": 7, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7", - "files": [] - }, - { - "turnCount": 8, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/8", - "files": [ - "notes.txt +1/-0" - ] - } - ], - "pendingApprovals": [] -} diff --git a/prototype-2/evidence/turn8-after-runner-restart.out b/prototype-2/evidence/turn8-after-runner-restart.out deleted file mode 100644 index 0e0cd1e26f56..000000000000 --- a/prototype-2/evidence/turn8-after-runner-restart.out +++ /dev/null @@ -1,85 +0,0 @@ -[19:26:04.742] thread.turn.start accepted in 38 ms -[19:26:04.833] message user: "Without using any tools: which lines did you add to notes.txt earlier in this conversation? Reply with the exact lines only, one per line." -[19:26:08.271] activity context-window.updated: Context window updated -[19:26:08.271] message assistant: "edited by the remote runner\nafter hub restart" -[19:26:09.103] activity context-window.updated: Context window updated -[19:26:09.103] activity checkpoint.captured: Checkpoint captured -[19:26:09.103] turn 3aec0919-df29-4679-8037-0d3deaf7fea3 settled: completed after 4360 ms (first assistant text after 3528 ms) -{ - "latestTurn": { - "turnId": "3aec0919-df29-4679-8037-0d3deaf7fea3", - "state": "completed", - "requestedAt": "2026-09-25T19:26:04.710Z", - "startedAt": "2026-09-25T19:26:04.710Z", - "completedAt": "2026-09-25T19:26:08.669Z", - "assistantMessageId": "assistant:b663b4d0-215b-4770-8878-3a1d64c12568" - }, - "session": { - "status": "ready", - "activeTurnId": null, - "lastError": null - }, - "messageCount": 14, - "activityCount": 47, - "checkpoints": [ - { - "turnCount": 1, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/1", - "files": [ - "notes.txt +1/-0" - ] - }, - { - "turnCount": 2, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2", - "files": [] - }, - { - "turnCount": 3, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/3", - "files": [] - }, - { - "turnCount": 4, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/4", - "files": [] - }, - { - "turnCount": 5, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/5", - "files": [] - }, - { - "turnCount": 6, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/6", - "files": [] - }, - { - "turnCount": 7, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7", - "files": [] - }, - { - "turnCount": 8, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/8", - "files": [ - "notes.txt +1/-0" - ] - }, - { - "turnCount": 9, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/9", - "files": [] - } - ], - "pendingApprovals": [] -} diff --git a/prototype-2/evidence/turn9-runner-restart-midturn.out b/prototype-2/evidence/turn9-runner-restart-midturn.out deleted file mode 100644 index 3af00f2cc718..000000000000 --- a/prototype-2/evidence/turn9-runner-restart-midturn.out +++ /dev/null @@ -1,92 +0,0 @@ -[19:27:26.747] thread.turn.start accepted in 31 ms -[19:27:26.776] message user: "Run the Bash command 'sleep 15' in the foreground (do not run it in the background). When it has finished, append the line 'after runner restart' to notes.txt. " -[19:27:29.277] activity tool.started: Command run started -[19:27:29.278] activity tool.updated: Command run -[19:27:29.278] activity context-window.updated: Context window updated -[19:27:34.662] activity context-window.updated: Context window updated -[19:27:34.662] activity tool.completed: Command run -[19:27:34.662] turn b53e0583-1332-4761-b85c-163aedc3c1db settled: completed after 7914 ms (first assistant text after n/a ms) -{ - "latestTurn": { - "turnId": "b53e0583-1332-4761-b85c-163aedc3c1db", - "state": "completed", - "requestedAt": "2026-09-25T19:27:26.722Z", - "startedAt": "2026-09-25T19:27:26.722Z", - "completedAt": "2026-09-25T19:27:29.407Z", - "assistantMessageId": "assistant:b53e0583-1332-4761-b85c-163aedc3c1db" - }, - "session": { - "status": "stopped", - "activeTurnId": null, - "lastError": null - }, - "messageCount": 15, - "activityCount": 52, - "checkpoints": [ - { - "turnCount": 1, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/1", - "files": [ - "notes.txt +1/-0" - ] - }, - { - "turnCount": 2, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/2", - "files": [] - }, - { - "turnCount": 3, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/3", - "files": [] - }, - { - "turnCount": 4, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/4", - "files": [] - }, - { - "turnCount": 5, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/5", - "files": [] - }, - { - "turnCount": 6, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/6", - "files": [] - }, - { - "turnCount": 7, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/7", - "files": [] - }, - { - "turnCount": 8, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/8", - "files": [ - "notes.txt +1/-0" - ] - }, - { - "turnCount": 9, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/9", - "files": [] - }, - { - "turnCount": 10, - "status": "ready", - "ref": "refs/t3/checkpoints/dGhyZWFkLWM5ZmM3M2VmLWE4ZmItNGU3NS05OTNmLTJhYTI5ODdlMDk2Nw/turn/10", - "files": [] - } - ], - "pendingApprovals": [] -} diff --git a/prototype-2/hub-restart-test.sh b/prototype-2/hub-restart-test.sh deleted file mode 100755 index ff30c6f4120d..000000000000 --- a/prototype-2/hub-restart-test.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# (e1) Restart the hub while a turn runs on the runner. The approval request -# for the turn's file edit is produced while the hub is down. -set -uo pipefail -P2=/workspace/repos/t3code-proto-runner/prototype-2 -OUTBOX=/tmp/proto-runner/runner/userdata/runner/outbox.ndjson -head_seq() { tail -1 "$OUTBOX" | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>console.log(s.trim()?JSON.parse(s).sequence:0))'; } -START=$(head_seq); echo "outbox head before turn: $START" -node $P2/scenario.mjs turn "Run the Bash command 'sleep 20' in the foreground (do not run it in the background). When it has finished, append the line 'after hub restart' to notes.txt. Reply with just: done" --approve --timeout 240 > /tmp/proto-runner/logs/turn7-hub-restart.out 2>&1 & -SCEN=$! -for i in $(seq 1 120); do node $P2/outbox-tail.mjs $START item.started | grep -q command && break; sleep 0.5; done -echo "[$(date +%T.%3N)] command started on runner; stopping hub"; /tmp/proto-runner/ctl.sh stop-hub -DOWN=$(head_seq); echo "[$(date +%T.%3N)] hub down; outbox head $DOWN" -for i in $(seq 1 80); do node $P2/outbox-tail.mjs $DOWN request.opened | grep -q . && break; sleep 0.5; done -echo "[$(date +%T.%3N)] approval requested while hub down:"; node $P2/outbox-tail.mjs $DOWN request -echo "[$(date +%T.%3N)] outbox head $(head_seq); restarting hub"; /tmp/proto-runner/ctl.sh start-hub -wait $SCEN; echo "scenario exit $?" -echo "events produced by the runner while the hub was down: $DOWN..$(head_seq)" diff --git a/prototype-2/outbox-tail.mjs b/prototype-2/outbox-tail.mjs deleted file mode 100644 index 96f1fcaa2d35..000000000000 --- a/prototype-2/outbox-tail.mjs +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env node -// Prints outbox events after a sequence: node outbox-tail.mjs [typeFilter] -import * as fs from "node:fs"; -const path = process.env.OUTBOX ?? "/tmp/proto-runner/runner/userdata/runner/outbox.ndjson"; -const after = Number(process.argv[2] ?? 0); -const filter = process.argv[3]; -const lines = fs - .readFileSync(path, "utf8") - .trim() - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line)); -for (const entry of lines) { - if (entry.sequence <= after) continue; - if (filter && !entry.event.type.startsWith(filter)) continue; - const payload = entry.event.payload ?? {}; - console.log( - entry.sequence, - entry.event.createdAt.slice(11, 23), - entry.event.type, - payload.itemType ?? payload.requestType ?? payload.state ?? "", - ); -} diff --git a/prototype-2/runner-restart-midturn-test.sh b/prototype-2/runner-restart-midturn-test.sh deleted file mode 100755 index 7ea7e9557fee..000000000000 --- a/prototype-2/runner-restart-midturn-test.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Restart the runner while a turn is executing on it (graceful stop drains the -# provider session into the outbox), then continue the work in a new turn. -set -uo pipefail -P2=/workspace/repos/t3code-proto-runner/prototype-2 -OUTBOX=/tmp/proto-runner/runner/userdata/runner/outbox.ndjson -head_seq() { tail -1 "$OUTBOX" | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>console.log(s.trim()?JSON.parse(s).sequence:0))'; } -START=$(head_seq); echo "outbox head before turn: $START" -node $P2/scenario.mjs turn "Run the Bash command 'sleep 15' in the foreground (do not run it in the background). When it has finished, append the line 'after runner restart' to notes.txt. Reply with just: done" --approve --timeout 120 > /tmp/proto-runner/logs/turn9-runner-restart-midturn.out 2>&1 & -SCEN=$! -for i in $(seq 1 120); do node $P2/outbox-tail.mjs $START item.started | grep -q command && break; sleep 0.5; done -echo "[$(date +%T.%3N)] command running on runner; stopping runner"; /tmp/proto-runner/ctl.sh stop-runner -echo "[$(date +%T.%3N)] events outboxed during drain:"; node $P2/outbox-tail.mjs $START | tail -4 -sleep 2; echo "[$(date +%T.%3N)] starting runner"; /tmp/proto-runner/ctl.sh start-runner -wait $SCEN; echo "scenario exit $?" -echo "[$(date +%T.%3N)] follow-up turn" -node $P2/scenario.mjs turn "Your previous task was cut off by a machine restart. Finish it now: make sure notes.txt ends with the line 'after runner restart' (add it only if missing). Reply with just: done" --approve --timeout 120 > /tmp/proto-runner/logs/turn10-continue.out 2>&1 -echo "follow-up exit $?"; cat /tmp/proto-runner/checkout/notes.txt diff --git a/prototype-2/scenario.mjs b/prototype-2/scenario.mjs deleted file mode 100644 index 35479c021c53..000000000000 --- a/prototype-2/scenario.mjs +++ /dev/null @@ -1,335 +0,0 @@ -#!/usr/bin/env node -// Prototype 2 scenario driver. Talks only to the hub (:4411) over its public -// HTTP API and WebSocket RPC, exactly as a client would. -// -// node scenario.mjs setup -// node scenario.mjs turn "" [--approve] [--interrupt-after-tool] [--timeout 180] -// node scenario.mjs watch [--timeout 180] # follow the running turn without dispatching -// node scenario.mjs diff -// node scenario.mjs show -import * as fs from "node:fs"; -import * as crypto from "node:crypto"; - -const HUB = process.env.HUB_URL ?? "http://127.0.0.1:4411"; -const STATE_PATH = process.env.SCENARIO_STATE ?? "/tmp/proto-runner/scenario-state.json"; -const TOKEN = fs - .readFileSync(process.env.HUB_TOKEN_PATH ?? "/tmp/proto-runner/hub-token", "utf8") - .trim(); -const MODEL = { instanceId: "claudeAgent", model: "claude-haiku-4-5" }; - -const now = () => new Date().toISOString(); -const id = () => crypto.randomUUID(); -const stamp = () => new Date().toISOString().slice(11, 23); -const log = (...args) => console.log(`[${stamp()}]`, ...args); -const loadState = () => - fs.existsSync(STATE_PATH) ? JSON.parse(fs.readFileSync(STATE_PATH, "utf8")) : {}; -const saveState = (state) => fs.writeFileSync(STATE_PATH, JSON.stringify(state, null, 2)); -const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); - -async function http(method, path, body) { - const response = await fetch(`${HUB}${path}`, { - method, - headers: { - authorization: `Bearer ${TOKEN}`, - ...(body !== undefined ? { "content-type": "application/json" } : {}), - }, - ...(body !== undefined ? { body: JSON.stringify(body) } : {}), - }); - const text = await response.text(); - if (!response.ok) - throw new Error(`${method} ${path} -> ${response.status}: ${text.slice(0, 400)}`); - return text.length > 0 ? JSON.parse(text) : null; -} - -const dispatch = (command) => http("POST", "/api/orchestration/dispatch", command); -const threadSnapshot = (threadId) => http("GET", `/api/orchestration/threads/${threadId}`); - -/** One Effect RPC call over the hub's /ws endpoint using the JSON wire format. */ -async function rpc(tag, payload) { - const { ticket } = await http("POST", "/api/auth/websocket-ticket"); - const url = `${HUB.replace(/^http/, "ws")}/ws?wsTicket=${encodeURIComponent(ticket)}`; - const socket = new WebSocket(url); - return await new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error(`rpc ${tag} timed out`)), 30_000); - socket.addEventListener("open", () => { - socket.send(JSON.stringify({ _tag: "Request", id: "1", tag, payload, headers: [] })); - }); - socket.addEventListener("message", (message) => { - const decoded = JSON.parse(String(message.data)); - const messages = Array.isArray(decoded) ? decoded : [decoded]; - for (const entry of messages) { - if (entry._tag === "Ping") socket.send(JSON.stringify({ _tag: "Pong" })); - if (entry._tag === "Exit" && String(entry.requestId) === "1") { - clearTimeout(timer); - socket.close(); - if (entry.exit._tag === "Success") resolve(entry.exit.value); - else reject(new Error(`rpc ${tag} failed: ${JSON.stringify(entry.exit).slice(0, 600)}`)); - } - } - }); - socket.addEventListener("error", (error) => reject(error.error ?? new Error("socket error"))); - }); -} - -function summarize(thread) { - const pending = new Map(); - for (const activity of thread.activities) { - const requestId = activity.payload?.requestId; - if (activity.kind === "approval.requested") pending.set(requestId, activity); - if (activity.kind === "approval.resolved") pending.delete(requestId); - } - return { - latestTurn: thread.latestTurn, - session: thread.session - ? { - status: thread.session.status, - activeTurnId: thread.session.activeTurnId, - lastError: thread.session.lastError, - } - : null, - messageCount: thread.messages.length, - activityCount: thread.activities.length, - checkpoints: thread.checkpoints.map((c) => ({ - turnCount: c.checkpointTurnCount, - status: c.status, - ref: c.checkpointRef, - files: c.files.map((f) => `${f.path} +${f.additions}/-${f.deletions}`), - })), - pendingApprovals: [...pending.values()].map((a) => ({ - requestId: a.payload.requestId, - summary: a.summary, - detail: a.payload.detail, - })), - }; -} - -async function follow(state, options) { - const deadline = Date.now() + (options.timeoutSec ?? 180) * 1000; - const seenActivities = new Set(options.preSeenActivities ?? []); - const seenMessages = new Map(options.preSeenMessages ?? []); - let commandApprovedAt = null; - const approved = new Set(); - let interrupted = false; - let toolSeenAt = null; - const startedAt = Date.now(); - let firstAssistantAt = null; - let lastSummary = null; - while (Date.now() < deadline) { - let snapshot; - try { - snapshot = await threadSnapshot(state.threadId); - } catch (error) { - log("hub unreachable:", String(error.message ?? error).slice(0, 120)); - await sleep(1000); - continue; - } - const thread = snapshot.thread; - for (const activity of thread.activities) { - if (seenActivities.has(activity.id)) continue; - seenActivities.add(activity.id); - log(`activity ${activity.kind}: ${activity.summary}`); - if (activity.tone === "tool" && toolSeenAt === null) toolSeenAt = Date.now(); - } - for (const message of thread.messages) { - const previous = seenMessages.get(message.id); - if (previous !== message.text) { - seenMessages.set(message.id, message.text); - if (message.role === "assistant" && firstAssistantAt === null) - firstAssistantAt = Date.now(); - if ( - previous === undefined || - !message.streaming || - message.text.length - (previous?.length ?? 0) > 200 - ) { - log( - `message ${message.role}${message.streaming ? " (streaming)" : ""}: ${JSON.stringify(message.text.slice(0, 160))}`, - ); - } - } - } - const summary = summarize(thread); - lastSummary = summary; - if (options.approve) { - for (const pending of summary.pendingApprovals) { - if (approved.has(pending.requestId)) continue; - approved.add(pending.requestId); - log( - `approving ${pending.requestId} (${pending.summary}: ${String(pending.detail ?? "").slice(0, 80)})`, - ); - const approvalStart = Date.now(); - const sent = await dispatch({ - type: "thread.approval.respond", - commandId: id(), - threadId: state.threadId, - requestId: pending.requestId, - decision: "accept", - createdAt: now(), - }).then( - () => true, - (error) => { - log( - "approval dispatch failed, will retry:", - String(error.message ?? error).slice(0, 120), - ); - approved.delete(pending.requestId); - return false; - }, - ); - if (!sent) continue; - log(`approval dispatched in ${Date.now() - approvalStart} ms`); - if (/command/i.test(pending.summary)) commandApprovedAt = Date.now(); - } - } - // Interrupt once the approved command has been running for 3 s (or 15 s - // after the tool call streamed if no approval was needed). - const commandRunning = - (commandApprovedAt !== null && Date.now() - commandApprovedAt > 3000) || - (toolSeenAt !== null && Date.now() - toolSeenAt > 15000); - const textStreaming = - (firstAssistantAt !== null && Date.now() - firstAssistantAt > 2000) || - (options.interruptAfterMs !== undefined && Date.now() - startedAt > options.interruptAfterMs); - if ( - !interrupted && - ((options.interruptAfterTool && commandRunning) || - (options.interruptAfterText && textStreaming)) - ) { - interrupted = true; - const turnId = summary.latestTurn?.turnId; - log(`interrupting turn ${turnId}`); - await dispatch({ - type: "thread.turn.interrupt", - commandId: id(), - threadId: state.threadId, - ...(turnId ? { turnId } : {}), - createdAt: now(), - }); - } - const latest = summary.latestTurn; - if (latest && options.turnMessageId !== undefined) { - // wait until the turn for this message exists - } - if ( - latest && - latest.state !== "running" && - (options.expectNewTurn === undefined || latest.turnId !== options.expectNewTurn) - ) { - if ( - summary.pendingApprovals.length === 0 && - (thread.session?.activeTurnId ?? null) === null - ) { - log( - `turn ${latest.turnId} settled: ${latest.state} after ${Date.now() - startedAt} ms (first assistant text after ${firstAssistantAt ? firstAssistantAt - startedAt : "n/a"} ms)`, - ); - // Give the checkpoint reactor a moment, then report. - await sleep(1500); - const final = summarize((await threadSnapshot(state.threadId)).thread); - console.log(JSON.stringify(final, null, 2)); - return final; - } - } - await sleep(400); - } - log("timed out following the turn"); - console.log(JSON.stringify(lastSummary, null, 2)); - return lastSummary; -} - -const [command, ...rest] = process.argv.slice(2); -const flag = (name) => rest.includes(name); -const flagValue = (name, fallback) => { - const index = rest.indexOf(name); - return index >= 0 ? rest[index + 1] : fallback; -}; - -if (command === "setup") { - const state = { projectId: `proj-${id()}`, threadId: `thread-${id()}` }; - const t0 = Date.now(); - await dispatch({ - type: "project.create", - commandId: id(), - projectId: state.projectId, - title: "Proto runner checkout", - workspaceRoot: process.env.WORKSPACE_ROOT ?? "/tmp/proto-runner/checkout", - defaultModelSelection: MODEL, - createdAt: now(), - }); - log(`project.create accepted in ${Date.now() - t0} ms`); - const t1 = Date.now(); - await dispatch({ - type: "thread.create", - commandId: id(), - threadId: state.threadId, - projectId: state.projectId, - title: "Remote runner thread", - modelSelection: MODEL, - runtimeMode: "approval-required", - interactionMode: "default", - branch: null, - worktreePath: null, - createdAt: now(), - }); - log(`thread.create accepted in ${Date.now() - t1} ms`); - saveState(state); - console.log(JSON.stringify(state)); -} else if (command === "turn") { - const state = loadState(); - const prompt = rest[0]; - const beforeThread = (await threadSnapshot(state.threadId)).thread; - const before = beforeThread.latestTurn?.turnId; - const t0 = Date.now(); - await dispatch({ - type: "thread.turn.start", - commandId: id(), - threadId: state.threadId, - message: { messageId: `msg-${id()}`, role: "user", text: prompt, attachments: [] }, - runtimeMode: "approval-required", - interactionMode: "default", - createdAt: now(), - }); - log(`thread.turn.start accepted in ${Date.now() - t0} ms`); - await follow(state, { - approve: flag("--approve"), - interruptAfterTool: flag("--interrupt-after-tool"), - interruptAfterText: - flag("--interrupt-after-text") || flagValue("--interrupt-after-ms") !== undefined, - interruptAfterMs: - flagValue("--interrupt-after-ms") !== undefined - ? Number(flagValue("--interrupt-after-ms")) - : undefined, - timeoutSec: Number(flagValue("--timeout", "180")), - expectNewTurn: before, - preSeenActivities: beforeThread.activities.map((activity) => activity.id), - preSeenMessages: beforeThread.messages.map((message) => [message.id, message.text]), - }); -} else if (command === "watch") { - const state = loadState(); - const beforeThread = (await threadSnapshot(state.threadId)).thread; - await follow(state, { - approve: flag("--approve"), - interruptAfterTool: flag("--interrupt-after-tool"), - timeoutSec: Number(flagValue("--timeout", "180")), - preSeenActivities: flag("--all") ? [] : beforeThread.activities.map((activity) => activity.id), - preSeenMessages: flag("--all") - ? [] - : beforeThread.messages.map((message) => [message.id, message.text]), - }); -} else if (command === "diff") { - const state = loadState(); - const t0 = Date.now(); - const result = await rpc("orchestration.getTurnDiff", { - threadId: state.threadId, - fromTurnCount: Number(rest[0]), - toTurnCount: Number(rest[1]), - }); - log(`getTurnDiff via hub in ${Date.now() - t0} ms`); - console.log(JSON.stringify(result, null, 2)); -} else if (command === "show") { - const state = loadState(); - const snapshot = await threadSnapshot(state.threadId); - console.log(JSON.stringify(summarize(snapshot.thread), null, 2)); - for (const message of snapshot.thread.messages) - console.log(`${message.role}: ${JSON.stringify(message.text.slice(0, 300))}`); -} else { - console.error("usage: scenario.mjs setup|turn|watch|diff|show"); - process.exit(2); -} diff --git a/prototype-2/start-control.sh b/prototype-2/start-control.sh deleted file mode 100755 index 420487d4c7c7..000000000000 --- a/prototype-2/start-control.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# Control: the same server build in ordinary local mode (no runner), :4413. -set -euo pipefail -cd /tmp/proto-runner/control -export T3CODE_TELEMETRY_ENABLED=false T3CODE_NO_BROWSER=true T3CODE_AUTO_BOOTSTRAP_PROJECT_FROM_CWD=false -exec node /workspace/repos/t3code-proto-runner/apps/server/src/bin.ts \ - --base-dir /tmp/proto-runner/control --port 4413 >>/tmp/proto-runner/logs/control.log 2>&1 diff --git a/prototype-2/start-hub.sh b/prototype-2/start-hub.sh deleted file mode 100755 index 341ef9b45fca..000000000000 --- a/prototype-2/start-hub.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Starts the prototype hub on :4411. The hub runs inside a mount namespace in -# which /tmp/proto-runner/checkout is an empty, mode-000 tmpfs, under strace -# recording every syscall that names that path. -set -euo pipefail -cd /tmp/proto-runner/hub -export T3CODE_RUNNER_URL=ws://127.0.0.1:4412/runner/ws -export T3CODE_RUNNER_TOKEN="$(cat /tmp/proto-runner/runner-token)" -export T3CODE_TELEMETRY_ENABLED=false T3CODE_NO_BROWSER=true T3CODE_AUTO_BOOTSTRAP_PROJECT_FROM_CWD=false -exec strace -f -qq --seccomp-bpf -e trace=%file,%process,chdir,fchdir -P /tmp/proto-runner/checkout \ - -o "/tmp/proto-runner/logs/hub.strace.$(date +%s)" \ - bwrap --dev-bind / / --tmpfs /tmp/proto-runner/checkout --chmod 0000 /tmp/proto-runner/checkout --die-with-parent \ - node /workspace/repos/t3code-proto-runner/apps/server/src/bin.ts \ - --base-dir /tmp/proto-runner/hub --port 4411 >>/tmp/proto-runner/logs/hub.log 2>&1 diff --git a/prototype-2/start-runner.sh b/prototype-2/start-runner.sh deleted file mode 100755 index f78d86c0684e..000000000000 --- a/prototype-2/start-runner.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# Starts the prototype runner on :4412 with its own state dir; hosts real Claude. -set -euo pipefail -cd /tmp/proto-runner/checkout -export T3CODE_RUNNER_TOKEN="$(cat /tmp/proto-runner/runner-token)" -export T3CODE_TELEMETRY_ENABLED=false T3CODE_NO_BROWSER=true T3CODE_RUNNER_DRIVERS=claudeAgent T3CODE_RUNNER_OUTBOX_KEEP=1 -exec node /workspace/repos/t3code-proto-runner/apps/server/src/bin.ts runner \ - --base-dir /tmp/proto-runner/runner --port 4412 >>/tmp/proto-runner/logs/runner.log 2>&1 From babea580a38cdb4cf647bb8aad3ea684d704ee68 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 00:22:37 +0000 Subject: [PATCH 31/58] docs: describe thread machines as one integrated design Rewrite docs/internals/thread-machines.md after merging the hub persistence and runner work packages: one configuration section (database, tenant and secret key together, SQLite only for development, the machine directory requirement, standalone output unchanged), one hub persistence section (the tenant tables, thread-machine state in migration 050 with the single diff table, the `localOrHub` selection rule, attachments restored before shipping to runners), operations (migrations by range, roles and RLS behind a pooler, import and its verification script), composition, the remaining gaps, and how to run every test tier on SQLite and Postgres. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/internals/thread-machines.md | 176 ++++++++++++++++++++---------- 1 file changed, 117 insertions(+), 59 deletions(-) diff --git a/docs/internals/thread-machines.md b/docs/internals/thread-machines.md index 04f683851f45..0829014e914e 100644 --- a/docs/internals/thread-machines.md +++ b/docs/internals/thread-machines.md @@ -22,15 +22,17 @@ staging enables them. A hub never runs git, provider CLIs, terminals, or repository code, and never reads a checkout path. A runner never persists orchestration state. -### Hub configuration +## Configuration + +### Hub | Variable | Meaning | | ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `T3CODE_SERVER_MODE=hub` | Enables hub mode | -| `T3CODE_HUB_DATABASE_URL` | Postgres URL for the hub schema (a role without `BYPASSRLS`). Without it the hub persists to its SQLite state database, which only tests and development do | -| `T3CODE_HUB_DATABASE_ADMIN_URL` | Optional; used only to apply hub migrations. Defaults to the runtime URL | -| `T3CODE_HUB_TENANT_ID` | The Aldo user ID whose rows this process owns. One hub process serves one user. Required with a database URL | -| `T3CODE_HUB_SECRET_KEY` | Base64 32-byte key encrypting per-user secrets at rest in Postgres. Required with a database URL | +| `T3CODE_HUB_DATABASE_URL` | Postgres URL of the hub schema, for a role without `BYPASSRLS`. libpq's `sslrootcert=system` is accepted. Without it the hub persists to SQLite (tests, development) | +| `T3CODE_HUB_DATABASE_ADMIN_URL` | Optional; the schema owner, used only to apply hub migrations and grant the runtime role. Defaults to the runtime URL | +| `T3CODE_HUB_TENANT_ID` | The Aldo user ID whose rows this process owns; one hub process serves one user. Required with a database URL | +| `T3CODE_HUB_SECRET_KEY` | Base64 32-byte key encrypting the tenant's secrets at rest. Required with a database URL | | `T3CODE_HUB_MACHINES_URL` | Base URL of the machine directory (served by the platform host process) | | `T3CODE_HUB_MACHINES_TOKEN` | Bearer for the machine directory | | `T3CODE_HUB_PUBLIC_URL` | Base URL at which thread machines reach the hub. Runners are given `/mcp` as the `t3-code` MCP endpoint; without it only a runner on the hub's host can reach MCP | @@ -38,11 +40,15 @@ checkout path. A runner never persists orchestration state. | `T3CODE_RUNNER_URL` | Development only: a single `t3 runner` WebSocket URL used instead of the machine directory (static directory, always `running`) | | `T3CODE_RUNNER_TOKEN` | Development only: the bearer presented to that runner | -The existing managed variables (`T3CODE_MANAGED_DEVPC`, `WORKSPACE_GATEWAY_TOKEN`, port, -base dir) keep their meaning. The base dir holds only disposable caches and logs in hub -mode; losing it loses nothing. +A hub needs the machine directory (URL and token) or a development runner URL. The +database, tenant, and secret key go together: with them the hub is production-shaped and its +base directory is disposable (logs, caches, and the live process's runtime state file only); +without them it keeps a standalone-style state directory, which only tests and development +use. The existing managed variables (`T3CODE_MANAGED_DEVPC`, `WORKSPACE_GATEWAY_TOKEN`, port, +base dir) keep their meaning. In standalone mode none of these keys appear in the resolved +configuration, so it stays identical to upstream T3. -### Runner configuration +### Runner The runner is `t3 runner` with the usual base dir and port flags plus: @@ -249,28 +255,11 @@ checkout") and `thread-machine.failed` (tone `error`). A failed bootstrap delete thread as in standalone mode. Before every turn the checkout is prepared again (skipped when the same runner boot already did). -## Composition - -`server.ts` builds one runtime for every mode from the `ServerModeLayers` record; standalone -uses the local layers and a hub uses `hub/HubLayers.ts` for the checkout-bound groups -(checkpointing, git, VCS, terminals, workspace, repository identity, provider instances) -plus hub-only infrastructure and background work. Shared code sees hub mode only through -the `serverModeHooks.ts` references (deterministic ingestion ids, `HubThreadCheckouts`) and -`CheckoutGitProbe`, whose defaults are standalone behavior. `t3 runner` is composed in -`runner/RunnerServer.ts` without orchestration, projections or the client API. - -Not available on a hub (typed errors, never the hub's disk): filesystem browsing for the -project picker, repository clone and publish, and workspace-file, media and project-favicon -asset URLs. Still running where the hub runs, to move to runners or the credential service: -previews, provider sign-in (auth connector), provider maintenance, usage scanning, workflow -scripts, and pull-request listing (which needs provider credentials). - -## What the hub persists +## Hub persistence -A hub process serves exactly one tenant (`T3CODE_HUB_TENANT_ID`) and its base directory is -disposable: it holds only logs, caches, and the live process's runtime state file. Everything -a standalone server keeps in its state directory lives in Postgres instead, in tables whose -keys and indexes all start with `user_id`: +A hub process serves exactly one tenant. With a database, everything a standalone server +keeps in its state directory lives in Postgres, in tables whose keys and indexes all start +with `user_id`: | Standalone | Hub | | ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | @@ -280,55 +269,124 @@ keys and indexes all start with `user_id`: | `secrets/*.bin` | `hub_secrets`, AES-256-GCM with `T3CODE_HUB_SECRET_KEY`; the authenticated data binds tenant and secret name | | `attachments/` | `hub_attachments` (`bytea`); the local directory is a cache filled on lookup | | Repository identity from `git remote` on every read | `projection_projects.repository_identity_json`, recorded by `project.create` / `project.meta.update` | - -Machine-published themes stay local (a hub has no desktop to publish them). Thread-machine -state is hub migration 050: runner cursors (`hub_runner_cursors`), per-turn diffs -(`hub_checkpoint_turn_diffs`, the hub's only diff table; standalone `checkpoint_diff_blobs` -is never written and is not imported), and the last reported git status per thread -(`hub_thread_vcs_status`). - -In code, hub mode is a `HubDatabase` reference provided at the server and auth CLI roots -(`apps/server/src/persistence/Postgres/`). The SQLite persistence layer hands out its client, -each repository layer selects its Postgres port through `localOrHub`, and file-backed stores -branch on it. Standalone layers, types, and behavior are unchanged. +| (none) | Thread-machine state, migration 050: see below | + +Thread-machine state (`persistence/Services/HubThreadMachineState.ts`): + +- `hub_runner_cursors` (`RunnerCursorStore`): per thread, the runner outbox id, the last + reconciled boot, and the durable delivery sequence. +- `hub_checkpoint_turn_diffs` (`CheckpointTurnDiffStore`): per-turn patches in both + whitespace modes. It is the hub's only diff table; standalone `checkpoint_diff_blobs` is + never written, has no hub table, and is not imported. +- `hub_thread_vcs_status` (`ThreadVcsStatusStore`): the last git status a runner reported. + +Machine-published themes stay local (a hub has no desktop to publish them). + +In code, hub mode with a database is a `HubDatabase` reference (tenant id and the +tenant-scoped client) provided once at the server and auth CLI roots +(`persistence/Postgres/HubDatabase.ts`). There is one selection rule, `localOrHub(local, +hub)`: every repository layer, the thread-machine stores included, gets the tenant client as +its `SqlClient` and the tenant as `HubTenant` when the reference is set, and its SQLite +implementation otherwise. The SQLite persistence layer hands out the tenant client in hub +mode and refuses to open a local file when a database URL is configured; file-backed stores +(settings, keybindings, secrets, environment and anonymous ids, attachments) branch on the +same reference. A hub without a database therefore runs on SQLite end to end: the +thread-machine stores create their tables on first use (never in standalone databases), and +no repository identity is recorded, so machines start from an empty checkout. Standalone +layers, types, and behavior are unchanged. + +Attachments are persisted when they become durable (upload completed, pending upload +claimed by a turn, inline image written) and restored into the cache before any lookup by +id, including before the remote provider driver ships a turn's attachment bytes to the +runner, so a hub on a fresh base directory still sends them. ## Operations **Migrations.** Hub migrations are numbered entries in `apps/server/src/persistence/Postgres/migrations/index.ts`: 001–049 for hub persistence, -050–099 for thread-machine state. A hub applies pending ones at startup with +050–099 for thread-machine state (050 today). A hub applies pending ones at startup with `T3CODE_HUB_DATABASE_ADMIN_URL` (or the runtime URL), each in its own transaction under a transaction-scoped advisory lock, recorded in `hub_schema_migrations`. They are applied by id, -not list position, so separately owned ranges can land in any order. Keep them -expand/contract compatible: an older hub may still be running. +not list position, so separately owned ranges can land in any order. Every new tenant table +leads its keys and indexes with `user_id` and enables `hubTenantPolicyStatements` (001). +Keep migrations expand/contract compatible: an older hub may still be running. **Roles and row-level security.** Every tenant table has a forced policy comparing `user_id` with the transaction-local `hub.user_id`. The runtime client sets it with `SET LOCAL` at the -start of every transaction and wraps statements outside one in their own transaction (two -extra round trips), so it works behind a transaction pooler and never leaks between clients. -Without the setting a query sees nothing and cannot write. Use a runtime role without -`BYPASSRLS` (PlanetScale's default role has it) and pass the owner as the admin URL; after -migrating, the hub grants the runtime role DML on the schema's tables (read-only on the -migration history). The explicit `user_id` predicates remain the primary isolation; RLS is -the backstop. Each hub keeps a pool of at most four connections. +start of every transaction (`BEGIN; SET LOCAL hub.user_id = …` in one round trip) and wraps +statements outside one in their own transaction (two extra round trips), so it works behind a +transaction pooler such as PgBouncer and never leaks between clients. Without the setting a +query sees nothing and cannot write. Use a runtime role without `BYPASSRLS` (PlanetScale's +default role has it) and pass the owner as the admin URL; after migrating, the hub grants the +runtime role DML on the schema's tables (read-only on the migration history). The explicit +`user_id` predicates remain the primary isolation; RLS is the backstop. Each hub keeps a pool +of at most four connections, and the Postgres driver is loaded only when a hub database is +configured. **Import.** `t3 hub import ` copies a standalone state directory into the tenant named by the `T3CODE_HUB_*` environment in one transaction: the database (read-only, paged, at the current standalone migration), the documents, attachments within the upload limit, and provider environment secrets re-encrypted with the hub key. Auth sessions, pairing links, and other secrets stay behind. It refuses a tenant with data unless `--replace`, -which first deletes every row the tenant has (sessions and hub secrets included), records -repository identities for projects whose checkouts exist on the machine, and prints -counts only. +which first deletes every row the tenant has in any tenant table (sessions, hub secrets and +thread-machine state included), records repository identities for projects whose checkouts +exist on the machine, and prints counts only. To verify an import, run +`node apps/server/scripts/hub-import-compare.ts [thread-ids]` with the same +environment: it compares every snapshot read of the standalone database and the tenant and +prints only equal/different with counts. + +## Composition + +`server.ts` builds one runtime for every mode from the `ServerModeLayers` record; standalone +uses the local layers and a hub uses `hub/HubLayers.ts` for the checkout-bound groups +(checkpointing, git, VCS, terminals, workspace, provider instances), the recorded repository +identity (`HubRepositoryIdentityResolver`), and hub-only infrastructure (machine directory, +connection pool, session registry, event delivery, state stores, git status cache) and +background work. `HubDatabase.layerConfig` sits under the whole runtime. Shared code sees hub +mode only through the `serverModeHooks.ts` references (deterministic ingestion ids, +`HubThreadCheckouts`), `CheckoutGitProbe`, and `HubDatabase`, whose defaults are standalone +behavior. `t3 runner` is composed in `runner/RunnerServer.ts` without orchestration, +projections or the client API. + +## Gaps + +Not available on a hub (typed errors, never the hub's disk): filesystem browsing for the +project picker, repository clone and publish, and workspace-file, media and project-favicon +asset URLs. + +Still running where the hub runs, to move to runners or the credential service: previews, +provider sign-in (auth connector), provider maintenance, usage scanning, workflow scripts, +and pull-request listing (which needs provider credentials). + +Not built yet: + +- No client records a project's repository identity; the platform (or the import, from a + checkout) must dispatch `project.create` / `project.meta.update` with `repositoryIdentity`. + A hub without a database records none. +- A hub behind a runner's `firstRetainedSequence` (more than 200,000 unacknowledged events) + only logs the gap; resynchronizing from `readThread` is future work. +- Attachment bytes live in `bytea` within the upload limits; object storage can replace the + table behind the `HubAttachments` helpers. ## Testing - Unit: `apps/server/src/hub/*.test.ts` (wake semantics with a fake directory and fake - runner, delivery and reconciliation, routed services), - `apps/server/src/runner/*.test.ts` (outbox, checkout), - `packages/contracts/src/runner.test.ts`. + runner, delivery and reconciliation, routed services, the machine directory contract), + `apps/server/src/runner/*.test.ts` (outbox, checkout, handler guards), + `packages/contracts/src/runner.test.ts`, and the thread-machine stores on SQLite. +- Postgres: set `T3_HUB_TEST_DATABASE_URL` to a disposable database the tests may create + schemas in (for example a private loopback cluster as `postgres`). Each test gets its own + schema and, when the connecting role bypasses RLS on loopback, a runtime role without + `BYPASSRLS`, so row-level security is exercised. This enables the persistence tests + (`persistence/Postgres/*.test.ts`, including the thread-machine stores and migration 050), + the hub server boot (`server.hub.test.ts`), attachment restore before shipping to a + runner, and the loopback integration on Postgres. Without the variable they skip, and the + loopback runs on SQLite. - Loopback integration: `apps/server/integration/hubRunnerLoopback.integration.test.ts` runs `ProviderService` and the reactors on a hub against the real runner handlers. - End to end with a real provider: - `node apps/server/scripts/thread-machines-e2e.mjs [--keep] [--model ]` starts a - runner (4422) and a hub (4421) on loopback and drives the hub's public API. + `node apps/server/scripts/thread-machines-e2e.mjs [--keep] [--model ] +[--hub-database-url [--hub-database-admin-url ]]` starts a runner (4422) and a + hub (4421) on loopback and drives the hub's public API. With a database URL the hub runs + on Postgres for a fresh tenant seeded through `t3 hub import`, and restarts on an empty + base directory. From 16e3c1a62078664ed66049bba47583488913911d Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 00:32:21 +0000 Subject: [PATCH 32/58] feat(contracts): share thread machine state on thread shells and advertise hub mode Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/contracts/src/environment.ts | 4 ++++ packages/contracts/src/index.ts | 1 + packages/contracts/src/orchestration.ts | 3 +++ packages/contracts/src/runner.ts | 14 +++---------- packages/contracts/src/threadMachine.ts | 26 +++++++++++++++++++++++++ 5 files changed, 37 insertions(+), 11 deletions(-) create mode 100644 packages/contracts/src/threadMachine.ts diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index 34a864748f26..f0f96b842c52 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -101,6 +101,10 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({ this is false — no update would ever repaint it. Absent on older servers, which may still publish, so only an explicit false skips. */ agentActivityPublishing: Schema.optionalKey(Schema.Boolean), + /** Hub mode: every thread runs on its own on-demand machine, projects are + repositories or blank projects, and the server has no filesystem of its + own. Absent on standalone servers. */ + threadMachines: Schema.optionalKey(Schema.Boolean), }); export type ExecutionEnvironmentCapabilities = typeof ExecutionEnvironmentCapabilities.Type; diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index f6211e9a5faa..6c7363afda3c 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -21,6 +21,7 @@ export * from "./git.ts"; export * from "./vcs.ts"; export * from "./sourceControl.ts"; export * from "./pullRequest.ts"; +export * from "./threadMachine.ts"; export * from "./orchestration.ts"; export * from "./t3ProjectFile.ts"; export * from "./editor.ts"; diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts index 1e9c92e2ec3b..2ed2a7ac833f 100644 --- a/packages/contracts/src/orchestration.ts +++ b/packages/contracts/src/orchestration.ts @@ -23,6 +23,7 @@ import { TurnId, } from "./baseSchemas.ts"; import { ProviderInstanceId } from "./providerInstance.ts"; +import { ThreadMachineStatus } from "./threadMachine.ts"; export const ORCHESTRATION_WS_METHODS = { dispatchCommand: "orchestration.dispatchCommand", @@ -552,6 +553,8 @@ export const OrchestrationThreadShell = Schema.Struct({ * live work. Optional so old servers/clients interop; absent = none. */ backgroundLiveness: Schema.optional(Schema.NullOr(Schema.Literals(["working", "monitoring"]))), + /** Hub mode only: the thread machine's latest known state. Absent on standalone servers. */ + machine: Schema.optional(Schema.NullOr(ThreadMachineStatus)), /** * Current plan step while a turn runs, for the Working indicators * (sidebar row, in-chat working line). Cleared when the turn settles — diff --git a/packages/contracts/src/runner.ts b/packages/contracts/src/runner.ts index 788338454328..49e436827a36 100644 --- a/packages/contracts/src/runner.ts +++ b/packages/contracts/src/runner.ts @@ -90,6 +90,7 @@ import { ProviderTurnStartResult, } from "./provider.ts"; import { ProviderInstanceId } from "./providerInstance.ts"; +import { ThreadMachineState } from "./threadMachine.ts"; import { ProviderRuntimeEvent } from "./providerRuntime.ts"; import { ReviewDiffFileContentsInput, @@ -179,17 +180,8 @@ export function parseThreadCheckoutPath( // ── Thread machine directory ─────────────────────────────────────────── -/** Lifecycle state reported by the machine directory. */ -export const ThreadMachineState = Schema.Literals([ - "none", - "preparing", - "starting", - "running", - "paused", - "saved", - "failed", -]); -export type ThreadMachineState = typeof ThreadMachineState.Type; +/** Lifecycle state reported by the machine directory (shared with thread shells). */ +export { ThreadMachineState } from "./threadMachine.ts"; /** Repository a new thread machine checks out, from the project's identity. */ export const ThreadMachineRepository = Schema.Struct({ diff --git a/packages/contracts/src/threadMachine.ts b/packages/contracts/src/threadMachine.ts new file mode 100644 index 000000000000..7f4918021cfc --- /dev/null +++ b/packages/contracts/src/threadMachine.ts @@ -0,0 +1,26 @@ +import * as Schema from "effect/Schema"; + +import { IsoDateTime } from "./baseSchemas.ts"; + +/** + * Lifecycle of the machine a hub-mode thread runs on. Standalone servers never + * report it. See docs/internals/thread-machines.md. + */ +export const ThreadMachineState = Schema.Literals([ + "none", + "preparing", + "starting", + "running", + "paused", + "saved", + "failed", +]); +export type ThreadMachineState = typeof ThreadMachineState.Type; + +/** The latest machine state a hub knows for a thread, shown without waking it. */ +export const ThreadMachineStatus = Schema.Struct({ + state: ThreadMachineState, + detail: Schema.NullOr(Schema.String), + updatedAt: IsoDateTime, +}); +export type ThreadMachineStatus = typeof ThreadMachineStatus.Type; From a450bc4430ced184882813622cbc330abc2817c2 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 00:40:11 +0000 Subject: [PATCH 33/58] feat(server): advertise thread machines instead of pull requests on a hub Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/environment/ServerEnvironment.test.ts | 27 +++++++++++++++++++ .../src/environment/ServerEnvironment.ts | 5 +++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/apps/server/src/environment/ServerEnvironment.test.ts b/apps/server/src/environment/ServerEnvironment.test.ts index e2e096b13b89..15cb9bc6ec1b 100644 --- a/apps/server/src/environment/ServerEnvironment.test.ts +++ b/apps/server/src/environment/ServerEnvironment.test.ts @@ -173,6 +173,33 @@ it.layer(NodeServices.layer)("ServerEnvironmentLive", (it) => { expect(second.capabilities.primeAgentSubscriptionOAuth).toBeUndefined(); expect(second.capabilities.threadPullRequestLinking).toBe(true); expect(second.capabilities.agentActivityPublishing).toBe(false); + expect(second.capabilities.threadMachines).toBeUndefined(); + }), + ); + + it.effect("advertises thread machines instead of pull requests in hub mode", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-server-environment-hub-test-", + }); + const serverConfig = yield* makeServerConfig(baseDir); + yield* fileSystem.makeDirectory(serverConfig.stateDir, { recursive: true }); + const descriptor = yield* Effect.gen(function* () { + const serverEnvironment = yield* ServerEnvironment.ServerEnvironment; + return yield* serverEnvironment.getDescriptor; + }).pipe( + Effect.provide( + ServerEnvironment.layer.pipe( + Layer.provide(emptySecretStoreLayer), + Layer.provide(ServerConfig.layer({ ...serverConfig, serverMode: "hub", hub: {} })), + ), + ), + ); + + expect(descriptor.capabilities.threadMachines).toBe(true); + expect(descriptor.capabilities.pullRequests).toBeUndefined(); + expect(descriptor.capabilities.repositoryIdentity).toBe(true); }), ); diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index 337f745fd91e..2c538793a89e 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -231,6 +231,7 @@ export const make = Effect.gen(function* () { const cwdBaseName = path.basename(serverConfig.cwd).trim(); const label = yield* resolveServerEnvironmentLabel({ cwdBaseName }); const launcher = yield* resolveServiceLauncherMode(); + const hubMode = ServerConfig.serverModeOf(serverConfig) === "hub"; const serverSelfUpdate = serverConfig.managedDevPc ? null : resolveServerSelfUpdateCapability({ @@ -251,7 +252,9 @@ export const make = Effect.gen(function* () { connectionProbe: true, attachmentUploads: true, fileAttachments: { maxUploadBytes: PROVIDER_SEND_TURN_MAX_FILE_BYTES }, - pullRequests: true, + // A hub has no checkout or `gh` login of its own, so it advertises + // thread machines instead of the pull-request workspace. + ...(hubMode ? { threadMachines: true } : { pullRequests: true }), threadSettlement: true, threadAutoSettlement: true, threadSnooze: true, From 784ff5034d90849476c9bff1306597f40bb83204 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 00:47:53 +0000 Subject: [PATCH 34/58] feat(server): project thread machine state onto thread shells and record every transition The hub observes every machine directory response in ThreadMachineStates, persists the latest state per thread (hub migration 051, SQLite in development), and exposes it to thread shells through the ThreadMachineStatusReader hook, so the shell snapshot and every shell upsert carry machine: { state, detail, updatedAt }. Each change becomes a thread-machine.state activity { state, detail, bootId } (tone error when failed), which is the domain event that re-publishes the shell; it replaces the one-off thread-machine.starting activity. Machines believed awake without a hub connection are re-checked every minute without waking them. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../HubRunnerLoopbackHarness.integration.ts | 9 +- .../hubRunnerLoopback.integration.test.ts | 12 ++ apps/server/src/hub/HubLayers.ts | 16 +- apps/server/src/hub/HubRunnerLifecycle.ts | 42 ++++ apps/server/src/hub/HubThreadCheckouts.ts | 88 ++------ .../src/hub/ThreadMachineStates.test.ts | 112 ++++++++++ apps/server/src/hub/ThreadMachineStates.ts | 193 ++++++++++++++++++ apps/server/src/hub/threadMachineActivity.ts | 88 ++++++++ .../Layers/ProjectionSnapshotQuery.ts | 5 + .../Layers/HubThreadMachineState.ts | 52 ++++- .../Postgres/HubThreadMachineState.ts | 51 ++++- .../Postgres/ProjectionSnapshotQuery.ts | 5 + .../051_HubThreadMachineServices.ts | 40 ++++ .../persistence/Postgres/migrations/index.ts | 7 +- .../Services/HubThreadMachineState.ts | 28 ++- apps/server/src/serverModeHooks.ts | 25 +++ packages/contracts/src/runner.ts | 2 +- packages/contracts/src/threadMachine.ts | 25 +++ 18 files changed, 717 insertions(+), 83 deletions(-) create mode 100644 apps/server/src/hub/ThreadMachineStates.test.ts create mode 100644 apps/server/src/hub/ThreadMachineStates.ts create mode 100644 apps/server/src/hub/threadMachineActivity.ts create mode 100644 apps/server/src/persistence/Postgres/migrations/051_HubThreadMachineServices.ts diff --git a/apps/server/integration/HubRunnerLoopbackHarness.integration.ts b/apps/server/integration/HubRunnerLoopbackHarness.integration.ts index 3c8219fe5566..5c9d6e87e0bf 100644 --- a/apps/server/integration/HubRunnerLoopbackHarness.integration.ts +++ b/apps/server/integration/HubRunnerLoopbackHarness.integration.ts @@ -60,6 +60,7 @@ import { makeRemoteProviderDriver } from "../src/hub/RemoteProviderDriver.ts"; import * as RemoteSessionRegistry from "../src/hub/RemoteSessionRegistry.ts"; import { make as makePool, RunnerConnectionPool } from "../src/hub/RunnerConnectionPool.ts"; import * as RunnerEventDelivery from "../src/hub/RunnerEventDelivery.ts"; +import * as ThreadMachineStates from "../src/hub/ThreadMachineStates.ts"; import { serveRunner } from "../src/hub/testUtils/runnerServer.ts"; import { makeOrchestrationCommandDispatcher } from "../src/orchestration/CommandDispatcher.ts"; import { normalizeDispatchCommand } from "../src/orchestration/Normalizer.ts"; @@ -392,7 +393,13 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") makePool({ checkoutRoot, wakePollInterval: "5 millis", idleCheckInterval: "1 hour" }), ), ), - Layer.provideMerge(Layer.succeed(MachineDirectory, directory.directory)), + Layer.provideMerge( + ThreadMachineStates.observedMachineDirectoryLayer.pipe( + Layer.provide(Layer.succeed(MachineDirectory, directory.directory)), + ), + ), + Layer.provideMerge(ThreadMachineStates.readerLayer), + Layer.provideMerge(ThreadMachineStates.layer), Layer.provideMerge(HubLayers.makeHubStateStoresLayer(persistence)), ); const remoteAdapterRegistry = Layer.effect( diff --git a/apps/server/integration/hubRunnerLoopback.integration.test.ts b/apps/server/integration/hubRunnerLoopback.integration.test.ts index 40c24cf03fb6..32473f7ebe95 100644 --- a/apps/server/integration/hubRunnerLoopback.integration.test.ts +++ b/apps/server/integration/hubRunnerLoopback.integration.test.ts @@ -391,6 +391,18 @@ it.live("bootstraps a new thread on its machine: checkout path, branch, and prog thread.activities.map((activity) => activity.kind), "thread-machine.checkout.preparing", ); + // The wake is recorded as a machine state activity and on the shell. + const states = thread.activities + .filter((activity) => activity.kind === "thread-machine.state") + .map((activity) => (activity.payload as { readonly state: string }).state); + assert.include(states, "running"); + const shell = yield* harness.snapshotQuery().getThreadShellById(harness.threadId); + assert.equal(Option.getOrThrow(shell).machine?.state, "running"); + const snapshot = yield* harness.snapshotQuery().getShellSnapshot(); + assert.equal( + snapshot.threads.find((entry) => entry.id === harness.threadId)?.machine?.state, + "running", + ); const checkedOut = NodeFS.readFileSync( NodePath.join(harness.checkout, ".git", "HEAD"), "utf8", diff --git a/apps/server/src/hub/HubLayers.ts b/apps/server/src/hub/HubLayers.ts index b86f026a6907..068037456677 100644 --- a/apps/server/src/hub/HubLayers.ts +++ b/apps/server/src/hub/HubLayers.ts @@ -49,6 +49,7 @@ import { makeRemoteProviderDriver } from "./RemoteProviderDriver.ts"; import * as RemoteSessionRegistry from "./RemoteSessionRegistry.ts"; import * as RunnerConnectionPool from "./RunnerConnectionPool.ts"; import * as RunnerEventDelivery from "./RunnerEventDelivery.ts"; +import * as ThreadMachineStates from "./ThreadMachineStates.ts"; /** * Hub thread-machine state (runner cursors, per-turn diffs, git status): the @@ -65,10 +66,11 @@ export const makeHubStateStoresLayer = ( ); /** - * Hub-only services underneath the whole runtime: stores, machine directory, - * connection pool, session registry, event delivery and git status cache. - * `persistence` is the server's persistence layer (SQLite, or the hub's - * Postgres client in hub mode with a database). + * Hub-only services underneath the whole runtime: stores, machine states, + * machine directory (observed by the machine states), connection pool, + * session registry, event delivery and git status cache. `persistence` is + * the server's persistence layer (SQLite, or the hub's Postgres client in hub + * mode with a database). */ export const makeHubInfrastructureLayer = (parts: { readonly persistence: Layer.Layer; @@ -80,7 +82,11 @@ export const makeHubInfrastructureLayer = (parts: { ), ), Layer.provideMerge(RunnerConnectionPool.layer), - Layer.provideMerge(MachineDirectory.layer), + Layer.provideMerge( + ThreadMachineStates.observedMachineDirectoryLayer.pipe(Layer.provide(MachineDirectory.layer)), + ), + Layer.provideMerge(ThreadMachineStates.readerLayer), + Layer.provideMerge(ThreadMachineStates.layer), Layer.provideMerge(makeHubStateStoresLayer(parts.persistence)), ); diff --git a/apps/server/src/hub/HubRunnerLifecycle.ts b/apps/server/src/hub/HubRunnerLifecycle.ts index 28b9a8a3110c..2d5b63f91335 100644 --- a/apps/server/src/hub/HubRunnerLifecycle.ts +++ b/apps/server/src/hub/HubRunnerLifecycle.ts @@ -14,6 +14,11 @@ * machine directory requests. * - Releases a thread's machine when the thread is archived or deleted, and * drops its hub caches when it is deleted. + * - Records every machine state change as a `thread-machine.state` activity + * (which re-publishes the thread shell with its new `machine`), and checks + * machines the hub believes are awake but holds no connection to every + * `MACHINE_WATCH_INTERVAL`, so a machine the platform paused after an idle + * report shows as asleep. Checks never wake a machine. * * @module hub/HubRunnerLifecycle */ @@ -39,6 +44,11 @@ import { HubVcsStatusCache } from "./HubVcs.ts"; import { RemoteSessionRegistry } from "./RemoteSessionRegistry.ts"; import { RunnerConnectionPool } from "./RunnerConnectionPool.ts"; import { RunnerEventDelivery } from "./RunnerEventDelivery.ts"; +import { + makeThreadMachineActivityRecorder, + threadMachineStateActivity, +} from "./threadMachineActivity.ts"; +import { ThreadMachineStates } from "./ThreadMachineStates.ts"; /** Ingestion's stream consumers start at the same activation gate; let them subscribe first. */ const DELIVERY_START_DELAY = "250 millis"; @@ -46,6 +56,9 @@ const ACK_INTERVAL = "250 millis"; /** A safe point is persisted once it is this old, so its events have reached ingestion's queue. */ const ACK_SETTLE_MS = 250; const SHUTDOWN_FLUSH_TIMEOUT = "5 seconds"; +/** How often machines believed awake without a hub connection are re-checked. */ +const MACHINE_WATCH_INTERVAL = "60 seconds"; +const WATCHED_STATES: ReadonlySet = new Set(["preparing", "starting", "running"]); type ThreadLifecycleEvent = Extract< OrchestrationEvent, @@ -65,6 +78,8 @@ export const layer = Layer.effectDiscard( const cursors = yield* RunnerCursorStore; const diffs = yield* CheckpointTurnDiffStore; const vcsCache = yield* HubVcsStatusCache; + const machineStates = yield* ThreadMachineStates; + const recordActivity = yield* makeThreadMachineActivityRecorder; yield* pool.setContextResolver((threadId) => Effect.gen(function* () { @@ -129,6 +144,7 @@ export const layer = Layer.effectDiscard( yield* vcsCache.remove(threadId); yield* cursors.remove(threadId).pipe(Effect.ignore); yield* diffs.removeThread(threadId).pipe(Effect.ignore); + yield* machineStates.remove(threadId); } }).pipe( Effect.catchCause((cause) => @@ -148,5 +164,31 @@ export const layer = Layer.effectDiscard( : Effect.void, ), ); + + // Machine state transitions become thread activities; the activity's + // domain event re-publishes the shell with the new `machine`. + yield* forkParked( + Stream.runForEach(machineStates.changes, (change) => + Effect.gen(function* () { + const thread = yield* projections + .getThreadShellById(change.threadId) + .pipe(Effect.orElseSucceed(() => Option.none())); + // The provider sign-in machine and deleted threads have no thread. + if (Option.isNone(thread)) return; + yield* recordActivity(change.threadId, threadMachineStateActivity(change.current)); + }), + ), + ); + + // Machines the hub believes awake but holds no connection to may have been + // paused by the platform; a status read (which never wakes) records that. + const watchMachines = Effect.gen(function* () { + for (const [threadId, entry] of yield* machineStates.list) { + if (!WATCHED_STATES.has(entry.state)) continue; + if (Option.isSome(yield* pool.current(threadId))) continue; + yield* pool.machineStatus(threadId).pipe(Effect.ignore); + } + }); + yield* forkParked(watchMachines.pipe(Effect.delay(MACHINE_WATCH_INTERVAL), Effect.forever)); }), ); diff --git a/apps/server/src/hub/HubThreadCheckouts.ts b/apps/server/src/hub/HubThreadCheckouts.ts index 0263f7cfb309..14ad4354a99e 100644 --- a/apps/server/src/hub/HubThreadCheckouts.ts +++ b/apps/server/src/hub/HubThreadCheckouts.ts @@ -9,90 +9,47 @@ * branch) ensures and wakes the machine through the directory, passing the * project's repository identity, then asks the runner to prepare the * checkout (clone or fetch with the machine's git credentials, create or - * switch the branch). Progress and failures become thread activities: - * `thread-machine.starting`, `thread-machine.checkout.preparing`, - * `thread-machine.failed`. + * switch the branch). Machine progress is recorded by `ThreadMachineStates` + * (`thread-machine.state` activities); checkout progress and failures are + * `thread-machine.checkout.preparing` and `thread-machine.failed`. + * - A base ref of `HEAD` (the sentinel for "the repository's default + * branch") is resolved by the runner from `origin/HEAD`; the directory is + * asked for the default ref. * - Before every turn the checkout is prepared again; the runner call is * idempotent, and skipped when this runner boot already prepared it. * * @module hub/HubThreadCheckouts */ import { - CommandId, - EventId, OrchestrationDispatchCommandError, type ProjectId, + THREAD_MACHINE_ACTIVITY_KINDS, type ThreadId, } from "@t3tools/contracts"; import { + DEFAULT_BRANCH_BASE_REF, projectVirtualRoot, type ThreadMachineRepository, - type ThreadMachineStatus, } from "@t3tools/contracts/runner"; -import * as Crypto from "effect/Crypto"; -import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts"; import { ProjectionSnapshotQuery } from "../orchestration/Services/ProjectionSnapshotQuery.ts"; import { RepositoryIdentityResolver } from "../project/RepositoryIdentityResolver.ts"; import { HubThreadCheckouts, type HubThreadCheckoutsShape } from "../serverModeHooks.ts"; import { describeCause } from "./hubRouting.ts"; import { RunnerConnectionPool, type ThreadMachineContext } from "./RunnerConnectionPool.ts"; - -export const THREAD_MACHINE_ACTIVITY = { - starting: "thread-machine.starting", - preparingCheckout: "thread-machine.checkout.preparing", - failed: "thread-machine.failed", -} as const; +import { makeThreadMachineActivityRecorder } from "./threadMachineActivity.ts"; export const makeHubThreadCheckouts = Effect.gen(function* () { const pool = yield* RunnerConnectionPool; - const engine = yield* OrchestrationEngineService; const projections = yield* ProjectionSnapshotQuery; const repositoryIdentity = yield* RepositoryIdentityResolver; - const crypto = yield* Crypto.Crypto; /** Runner boot that last prepared each thread's checkout. */ const preparedOnBoot = new Map(); - const appendActivity = ( - threadId: ThreadId, - activity: { - readonly kind: string; - readonly summary: string; - readonly tone: "info" | "error"; - readonly payload: Record; - }, - ) => - Effect.gen(function* () { - const createdAt = DateTime.formatIso(yield* DateTime.now); - const uuid = yield* crypto.randomUUIDv4; - yield* engine.dispatch({ - type: "thread.activity.append", - commandId: CommandId.make(`server:thread-machine-activity:${uuid}`), - threadId, - activity: { - id: EventId.make(uuid), - tone: activity.tone, - kind: activity.kind, - summary: activity.summary, - payload: activity.payload, - turnId: null, - createdAt, - }, - createdAt, - }); - }).pipe( - Effect.catchCause((cause) => - Effect.logWarning("failed to record thread machine activity", { - threadId, - kind: activity.kind, - cause: String(cause).slice(0, 300), - }), - ), - ); + const appendActivity = yield* makeThreadMachineActivityRecorder; /** Project, repository and branch for the directory and the runner. */ const machineContext = ( @@ -118,8 +75,12 @@ export const makeHubThreadCheckouts = Effect.gen(function* () { ) : undefined; const identity = project ? yield* repositoryIdentity.resolve(project.workspaceRoot) : null; + // The default-branch sentinel asks the directory for the default ref. const repository: ThreadMachineRepository | null = identity - ? { url: identity.locator.remoteUrl, ref: baseRef } + ? { + url: identity.locator.remoteUrl, + ref: baseRef === DEFAULT_BRANCH_BASE_REF ? null : baseRef, + } : null; return { projectId: resolvedProjectId ?? null, @@ -139,22 +100,9 @@ export const makeHubThreadCheckouts = Effect.gen(function* () { const { threadId } = input; const checkout = pool.checkoutFor(threadId); const context = yield* machineContext(threadId, input.projectId, input.branch, input.baseRef); - let announcedStart = false; - const onWakeProgress = (status: ThreadMachineStatus) => - announcedStart - ? Effect.void - : Effect.suspend(() => { - announcedStart = true; - return appendActivity(threadId, { - kind: THREAD_MACHINE_ACTIVITY.starting, - summary: "Starting machine", - tone: "info", - payload: { state: status.state, detail: status.detail ?? null }, - }); - }); return yield* pool.use( threadId, - { wake: true, operation: "checkout.prepare", context, onWakeProgress }, + { wake: true, operation: "checkout.prepare", context }, (connection) => Effect.gen(function* () { if (!input.announce && preparedOnBoot.get(threadId) === connection.hello.bootId) { @@ -162,7 +110,7 @@ export const makeHubThreadCheckouts = Effect.gen(function* () { } if (input.announce) { yield* appendActivity(threadId, { - kind: THREAD_MACHINE_ACTIVITY.preparingCheckout, + kind: THREAD_MACHINE_ACTIVITY_KINDS.checkoutPreparing, summary: "Preparing checkout", tone: "info", payload: { @@ -186,7 +134,7 @@ export const makeHubThreadCheckouts = Effect.gen(function* () { }).pipe( Effect.tapError((error) => appendActivity(input.threadId, { - kind: THREAD_MACHINE_ACTIVITY.failed, + kind: THREAD_MACHINE_ACTIVITY_KINDS.failed, summary: "Thread machine is unavailable", tone: "error", payload: { detail: describeCause(error) }, diff --git a/apps/server/src/hub/ThreadMachineStates.test.ts b/apps/server/src/hub/ThreadMachineStates.test.ts new file mode 100644 index 000000000000..c4e116bbbee5 --- /dev/null +++ b/apps/server/src/hub/ThreadMachineStates.test.ts @@ -0,0 +1,112 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { it } from "@effect/vitest"; +import { ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import { describe, expect } from "vite-plus/test"; + +import { HubThreadMachineStateSqliteLive } from "../persistence/Layers/HubThreadMachineState.ts"; +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; +import { ThreadMachineStatusReader } from "../serverModeHooks.ts"; +import { MachineDirectory, makeFakeMachineDirectory } from "./MachineDirectory.ts"; +import * as ThreadMachineStates from "./ThreadMachineStates.ts"; +import { threadMachineStateActivity } from "./threadMachineActivity.ts"; + +const threadId = ThreadId.make("thread-states"); + +const StoreLayer = HubThreadMachineStateSqliteLive.pipe( + Layer.provideMerge(SqlitePersistenceMemory), + Layer.provideMerge(NodeServices.layer), +); + +describe("thread machine states", () => { + it.effect("records transitions once, keeps progress, and survives a restart", () => + Effect.gen(function* () { + const states = yield* ThreadMachineStates.make; + const changes = yield* states.changes.pipe( + Stream.take(5), + Stream.runCollect, + Effect.forkChild, + ); + yield* Effect.yieldNow; + + yield* states.observe(threadId, { state: "preparing", detail: "Building…" }); + yield* states.observe(threadId, { state: "preparing", detail: "Building…" }); + yield* states.observe(threadId, { state: "preparing", detail: "Waiting for a machine…" }); + yield* states.observe(threadId, { state: "running", bootId: "boot-1", detail: "up" }); + yield* states.observe(threadId, { state: "running", bootId: "boot-1", detail: "still up" }); + yield* states.observe(threadId, { state: "paused", detail: null }); + yield* states.observe(threadId, { state: "running", bootId: "boot-2" }); + + const recorded = [...(yield* Fiber.join(changes))].map((change) => [ + change.current.state, + change.current.detail, + change.current.bootId, + ]); + expect(recorded).toEqual([ + ["preparing", "Building…", null], + ["preparing", "Waiting for a machine…", null], + ["running", "up", "boot-1"], + ["paused", null, "boot-1"], + ["running", null, "boot-2"], + ]); + expect(states.get(threadId)?.state).toBe("running"); + + // A new instance on the same store starts from the persisted state. + const reloaded = yield* ThreadMachineStates.make; + expect(reloaded.get(threadId)).toMatchObject({ state: "running", bootId: "boot-2" }); + yield* reloaded.remove(threadId); + expect((yield* ThreadMachineStates.make).get(threadId)).toBeNull(); + }).pipe(Effect.scoped, Effect.provide(StoreLayer)), + ); + + it.effect("observes directory responses and exposes them to thread shells", () => + Effect.gen(function* () { + const fake = yield* makeFakeMachineDirectory({ + initial: [[threadId, { state: "paused", bootId: "boot-a" }]], + onWake: () => ({ state: "running", runnerUrl: "ws://127.0.0.1:1", bootId: "boot-a" }), + }); + const layer = Layer.mergeAll( + ThreadMachineStates.observedMachineDirectoryLayer, + ThreadMachineStates.readerLayer, + ).pipe( + Layer.provideMerge(ThreadMachineStates.layer), + Layer.provide(Layer.succeed(MachineDirectory, fake.directory)), + ); + yield* Effect.gen(function* () { + const directory = yield* MachineDirectory; + const reader = yield* ThreadMachineStatusReader; + expect(reader.get(threadId)).toBeNull(); + + yield* directory.status(threadId); + expect(reader.get(threadId)).toMatchObject({ state: "paused", detail: null }); + + yield* directory.ensure(threadId, { + projectId: null, + repository: null, + branch: null, + checkout: "/workspace/t/thread-states", + wake: true, + }); + expect(reader.get(threadId)?.state).toBe("running"); + + yield* directory.release(threadId); + expect(reader.get(threadId)?.state).toBe("none"); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(StoreLayer)), + ); + + it("describes each state as an activity with an error tone only for failures", () => { + expect(threadMachineStateActivity({ state: "running", detail: null, bootId: "b" })).toEqual({ + kind: "thread-machine.state", + summary: "Machine running", + tone: "info", + payload: { state: "running", detail: null, bootId: "b" }, + }); + expect( + threadMachineStateActivity({ state: "failed", detail: "No capacity", bootId: null }).tone, + ).toBe("error"); + }); +}); diff --git a/apps/server/src/hub/ThreadMachineStates.ts b/apps/server/src/hub/ThreadMachineStates.ts new file mode 100644 index 000000000000..7df1544aeb0c --- /dev/null +++ b/apps/server/src/hub/ThreadMachineStates.ts @@ -0,0 +1,193 @@ +/** + * ThreadMachineStates - the latest machine state the hub knows per thread. + * + * Every machine directory response (ensure, status, wake polls) passes + * through `observe`, and a released machine is recorded as `none`. A change of + * state or boot, or new progress while a machine is coming up or failed, is + * persisted (`ThreadMachineStatusStore`, so a restarted hub still shows it) + * and published on `changes`. `HubRunnerLifecycle` turns each change into a + * `thread-machine.state` activity; that domain event is what makes the shell + * stream re-read the thread, and thread shells read `machine` from here + * (`ThreadMachineStatusReader`), so the snapshot and every shell upsert carry + * the new state. + * + * @module hub/ThreadMachineStates + */ +import { + type ThreadId, + type ThreadMachineState, + type ThreadMachineStatus as ThreadShellMachineStatus, +} from "@t3tools/contracts"; +import type { ThreadMachineStatus as DirectoryMachineStatus } from "@t3tools/contracts/runner"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; + +import { + ThreadMachineStatusStore, + type ThreadMachineStatusRow, +} from "../persistence/Services/HubThreadMachineState.ts"; +import { ThreadMachineStatusReader } from "../serverModeHooks.ts"; +import { MachineDirectory } from "./MachineDirectory.ts"; + +export interface ThreadMachineStateEntry { + readonly state: ThreadMachineState; + readonly detail: string | null; + readonly bootId: string | null; + readonly updatedAt: string; +} + +export interface ThreadMachineStateChange { + readonly threadId: ThreadId; + readonly previous: ThreadMachineStateEntry | null; + readonly current: ThreadMachineStateEntry; +} + +export interface ThreadMachineStatesShape { + /** The latest known state; synchronous so shell mapping can read it. */ + readonly get: (threadId: ThreadId) => ThreadMachineStateEntry | null; + readonly list: Effect.Effect>; + /** Records a directory response; publishes a change when it is one. */ + readonly observe: (threadId: ThreadId, status: DirectoryMachineStatus) => Effect.Effect; + /** Forgets a deleted thread. */ + readonly remove: (threadId: ThreadId) => Effect.Effect; + readonly changes: Stream.Stream; +} + +export class ThreadMachineStates extends Context.Service< + ThreadMachineStates, + ThreadMachineStatesShape +>()("t3/hub/ThreadMachineStates") {} + +/** States in which progress detail is worth recording as it changes. */ +const PROGRESS_STATES: ReadonlySet = new Set([ + "preparing", + "starting", + "failed", +]); + +/** Whether `next` differs from `previous` in a way thread activity should record. */ +export const isThreadMachineStateChange = ( + previous: ThreadMachineStateEntry | null, + next: Omit, +): boolean => + previous === null || + previous.state !== next.state || + (next.bootId !== null && previous.bootId !== next.bootId) || + (PROGRESS_STATES.has(next.state) && previous.detail !== next.detail); + +export const make = Effect.gen(function* () { + const store = yield* ThreadMachineStatusStore; + const entries = new Map(); + for (const row of yield* store.list().pipe( + Effect.catch((error) => + Effect.logWarning("thread machine states could not be loaded", { + detail: error.message, + }).pipe(Effect.as([] as ReadonlyArray)), + ), + )) { + entries.set(row.threadId, { + state: row.state, + detail: row.detail, + bootId: row.bootId, + updatedAt: row.updatedAt, + }); + } + const changes = yield* PubSub.unbounded(); + const lock = yield* Semaphore.make(1); + + const observe: ThreadMachineStatesShape["observe"] = (threadId, status) => + lock.withPermit( + Effect.gen(function* () { + const previous = entries.get(threadId) ?? null; + const next = { + state: status.state, + detail: status.detail ?? null, + // A machine that is not running keeps the boot it last ran. + bootId: status.bootId ?? (status.state === "running" ? null : (previous?.bootId ?? null)), + }; + if (!isThreadMachineStateChange(previous, next)) return; + const current: ThreadMachineStateEntry = { + ...next, + updatedAt: DateTime.formatIso(yield* DateTime.now), + }; + entries.set(threadId, current); + yield* store.put({ threadId, ...current }).pipe( + Effect.catch((error) => + Effect.logWarning("thread machine state was not persisted", { + threadId, + detail: error.message, + }), + ), + ); + yield* PubSub.publish(changes, { threadId, previous, current }); + }), + ); + + return ThreadMachineStates.of({ + get: (threadId) => entries.get(threadId) ?? null, + list: Effect.sync(() => [...entries]), + observe, + remove: (threadId) => + lock.withPermit( + Effect.sync(() => entries.delete(threadId)).pipe( + Effect.andThen(store.remove(threadId)), + Effect.catch((error) => + Effect.logWarning("thread machine state was not removed", { + threadId, + detail: error.message, + }), + ), + ), + ), + changes: Stream.fromPubSub(changes), + }); +}); + +export const layer = Layer.effect(ThreadMachineStates, make); + +/** The shell field for a thread: its latest state, or null when no machine is known. */ +export const toShellMachineStatus = ( + entry: ThreadMachineStateEntry | null, +): ThreadShellMachineStatus | null => + entry === null ? null : { state: entry.state, detail: entry.detail, updatedAt: entry.updatedAt }; + +/** Thread shells read machine state from this hub service. */ +export const readerLayer = Layer.effect( + ThreadMachineStatusReader, + Effect.map(ThreadMachineStates, (states) => ({ + get: (threadId: ThreadId) => toShellMachineStatus(states.get(threadId)), + })), +); + +/** + * The machine directory with every response recorded in `ThreadMachineStates`. + * Provide the underlying directory to this layer. + */ +export const observedMachineDirectoryLayer = Layer.effect( + MachineDirectory, + Effect.gen(function* () { + const directory = yield* MachineDirectory; + const states = yield* ThreadMachineStates; + const record = (threadId: ThreadId) => (status: DirectoryMachineStatus) => + states.observe(threadId, status); + return MachineDirectory.of({ + ...directory, + ensure: (threadId, request) => + directory.ensure(threadId, request).pipe(Effect.tap(record(threadId))), + status: (threadId) => directory.status(threadId).pipe(Effect.tap(record(threadId))), + release: (threadId) => + directory + .release(threadId) + .pipe( + Effect.tap(() => + states.observe(threadId, { state: "none", detail: "The machine was released." }), + ), + ), + }); + }), +); diff --git a/apps/server/src/hub/threadMachineActivity.ts b/apps/server/src/hub/threadMachineActivity.ts new file mode 100644 index 000000000000..2f06fbb17e9d --- /dev/null +++ b/apps/server/src/hub/threadMachineActivity.ts @@ -0,0 +1,88 @@ +/** + * Thread activities a hub records about thread machines. + * + * Kinds (see `THREAD_MACHINE_ACTIVITY_KINDS` in contracts): + * - `thread-machine.state` on every machine state transition, payload + * `{ state, detail, bootId }`, tone `error` for `failed`. + * - `thread-machine.checkout.preparing` when bootstrap prepares the checkout. + * - `thread-machine.failed` when bootstrap or checkout preparation fails. + * + * @module hub/threadMachineActivity + */ +import { + CommandId, + EventId, + THREAD_MACHINE_ACTIVITY_KINDS, + type ThreadId, + type ThreadMachineState, + type ThreadMachineStateActivityPayload, +} from "@t3tools/contracts"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; + +import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts"; + +export interface ThreadMachineActivity { + readonly kind: string; + readonly summary: string; + readonly tone: "info" | "error"; + readonly payload: Record; +} + +const STATE_SUMMARIES: Record = { + none: "No machine", + preparing: "Preparing machine", + starting: "Starting machine", + running: "Machine running", + paused: "Machine asleep", + saved: "Machine saved", + failed: "Machine failed", +}; + +/** The `thread-machine.state` activity for a machine state. */ +export const threadMachineStateActivity = ( + payload: ThreadMachineStateActivityPayload, +): ThreadMachineActivity => ({ + kind: THREAD_MACHINE_ACTIVITY_KINDS.state, + summary: STATE_SUMMARIES[payload.state], + tone: payload.state === "failed" ? "error" : "info", + payload: { state: payload.state, detail: payload.detail, bootId: payload.bootId }, +}); + +/** + * A function appending activities to threads; failures are logged, never + * raised. Resolve it once where the orchestration engine is available. + */ +export const makeThreadMachineActivityRecorder = Effect.gen(function* () { + const engine = yield* OrchestrationEngineService; + const crypto = yield* Crypto.Crypto; + return (threadId: ThreadId, activity: ThreadMachineActivity): Effect.Effect => + Effect.gen(function* () { + const createdAt = DateTime.formatIso(yield* DateTime.now); + const uuid = yield* crypto.randomUUIDv4; + yield* engine.dispatch({ + type: "thread.activity.append", + commandId: CommandId.make(`server:thread-machine-activity:${uuid}`), + threadId, + activity: { + id: EventId.make(uuid), + tone: activity.tone, + kind: activity.kind, + summary: activity.summary, + payload: activity.payload, + turnId: null, + createdAt, + }, + createdAt, + }); + }).pipe( + Effect.catch((error) => + Effect.logWarning("failed to record thread machine activity", { + threadId, + kind: activity.kind, + detail: String(error.message).slice(0, 300), + }), + ), + ); +}); diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index fb1b727cc030..3f46a5c48fc3 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -46,6 +46,7 @@ import { import { ProjectionCheckpoint } from "../../persistence/Services/ProjectionCheckpoints.ts"; import { ThreadBackgroundLivenessService } from "../ThreadBackgroundLiveness.ts"; import { ThreadPlanProgressService } from "../ThreadPlanProgress.ts"; +import { ThreadMachineStatusReader, threadMachineShellField } from "../../serverModeHooks.ts"; import { ProjectionProject } from "../../persistence/Services/ProjectionProjects.ts"; import { ProjectionState } from "../../persistence/Services/ProjectionState.ts"; import { ProjectionThreadActivity } from "../../persistence/Services/ProjectionThreadActivities.ts"; @@ -386,6 +387,7 @@ function toPersistenceSqlOrDecodeError(sqlOperation: string, decodeOperation: st const makeProjectionSnapshotQuery = Effect.gen(function* () { const threadBackgroundLiveness = yield* ThreadBackgroundLivenessService; const threadPlanProgress = yield* ThreadPlanProgressService; + const threadMachineStatus = yield* ThreadMachineStatusReader; const sql = yield* SqlClient.SqlClient; const repositoryIdentityResolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; const repositoryIdentityResolutionConcurrency = 4; @@ -2248,6 +2250,7 @@ pending_approval_requests AS ( row.threadId, ), planProgress: threadPlanProgress.getThreadPlanProgress(row.threadId), + ...threadMachineShellField(threadMachineStatus, row.threadId), } satisfies OrchestrationThreadShell) : Result.failVoid, ), @@ -2397,6 +2400,7 @@ pending_approval_requests AS ( row.threadId, ), planProgress: threadPlanProgress.getThreadPlanProgress(row.threadId), + ...threadMachineShellField(threadMachineStatus, row.threadId), }), ), updatedAt: updatedAt ?? "1970-01-01T00:00:00.000Z", @@ -2680,6 +2684,7 @@ pending_approval_requests AS ( threadRow.value.threadId, ), planProgress: threadPlanProgress.getThreadPlanProgress(threadRow.value.threadId), + ...threadMachineShellField(threadMachineStatus, threadRow.value.threadId), } satisfies OrchestrationThreadShell); }); diff --git a/apps/server/src/persistence/Layers/HubThreadMachineState.ts b/apps/server/src/persistence/Layers/HubThreadMachineState.ts index 644b4fddddee..82502f409555 100644 --- a/apps/server/src/persistence/Layers/HubThreadMachineState.ts +++ b/apps/server/src/persistence/Layers/HubThreadMachineState.ts @@ -4,7 +4,7 @@ * Used only in hub mode without `T3CODE_HUB_DATABASE_URL` (tests and local * development). The tables are created on first use instead of through a * numbered migration so standalone databases never gain hub-only tables; the - * production schema is hub Postgres migration 050. + * production schema is hub Postgres migrations 050 and 051. * * @module HubThreadMachineStateSqlite */ @@ -23,6 +23,8 @@ import { CheckpointTurnDiffStore, RunnerCursor, RunnerCursorStore, + ThreadMachineStatusRow, + ThreadMachineStatusStore, ThreadVcsStatus, ThreadVcsStatusStore, } from "../Services/HubThreadMachineState.ts"; @@ -50,6 +52,13 @@ const HUB_SQLITE_TABLES = [ remote_json TEXT, updated_at TEXT NOT NULL )`, + `CREATE TABLE IF NOT EXISTS hub_thread_machine_status ( + thread_id TEXT PRIMARY KEY, + state TEXT NOT NULL, + detail TEXT, + boot_id TEXT, + updated_at TEXT NOT NULL + )`, ] as const; const ensureTables = Effect.gen(function* () { @@ -214,9 +223,48 @@ export const makeThreadVcsStatusStore = Effect.gen(function* () { }); }); -/** All three repositories over the configured SQLite client. */ +export const makeThreadMachineStatusStore = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* ensureTables; + + const selectAll = SqlSchema.findAll({ + Request: Schema.Void, + Result: ThreadMachineStatusRow, + execute: () => sql` + SELECT thread_id AS "threadId", state, detail, boot_id AS "bootId", + updated_at AS "updatedAt" + FROM hub_thread_machine_status + `, + }); + const upsert = SqlSchema.void({ + Request: ThreadMachineStatusRow, + execute: (row) => sql` + INSERT INTO hub_thread_machine_status (thread_id, state, detail, boot_id, updated_at) + VALUES (${row.threadId}, ${row.state}, ${row.detail}, ${row.bootId}, ${row.updatedAt}) + ON CONFLICT (thread_id) DO UPDATE SET + state = excluded.state, + detail = excluded.detail, + boot_id = excluded.boot_id, + updated_at = excluded.updated_at + `, + }); + + return ThreadMachineStatusStore.of({ + list: () => + selectAll(undefined).pipe(Effect.mapError(sqlOrDecode("ThreadMachineStatusStore.list"))), + put: (row) => upsert(row).pipe(Effect.mapError(sqlOrDecode("ThreadMachineStatusStore.put"))), + remove: (threadId) => + sql`DELETE FROM hub_thread_machine_status WHERE thread_id = ${threadId}`.pipe( + Effect.asVoid, + Effect.mapError(toPersistenceSqlError("ThreadMachineStatusStore.remove")), + ), + }); +}); + +/** Every repository over the configured SQLite client. */ export const HubThreadMachineStateSqliteLive = Layer.mergeAll( Layer.effect(RunnerCursorStore, makeRunnerCursorStore), Layer.effect(CheckpointTurnDiffStore, makeCheckpointTurnDiffStore), Layer.effect(ThreadVcsStatusStore, makeThreadVcsStatusStore), + Layer.effect(ThreadMachineStatusStore, makeThreadMachineStatusStore), ); diff --git a/apps/server/src/persistence/Postgres/HubThreadMachineState.ts b/apps/server/src/persistence/Postgres/HubThreadMachineState.ts index 97b5ca669c5c..2a321ed13f1e 100644 --- a/apps/server/src/persistence/Postgres/HubThreadMachineState.ts +++ b/apps/server/src/persistence/Postgres/HubThreadMachineState.ts @@ -1,6 +1,6 @@ /** * Postgres implementations of the hub thread-machine repositories - * (tables from hub migration 050). Every statement is scoped to the + * (tables from hub migrations 050 and 051). Every statement is scoped to the * `HubTenant` user, like the rest of the hub schema. * * @module HubThreadMachineStatePostgres @@ -20,6 +20,8 @@ import { CheckpointTurnDiffStore, RunnerCursor, RunnerCursorStore, + ThreadMachineStatusRow, + ThreadMachineStatusStore, ThreadVcsStatus, ThreadVcsStatusStore, } from "../Services/HubThreadMachineState.ts"; @@ -186,9 +188,54 @@ export const makeThreadVcsStatusStore = Effect.gen(function* () { }); }); -/** All three repositories; requires the hub Postgres `SqlClient` and `HubTenant`. */ +export const makeThreadMachineStatusStore = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const selectAll = SqlSchema.findAll({ + Request: Schema.Void, + Result: ThreadMachineStatusRow, + execute: () => sql` + SELECT thread_id AS "threadId", state, detail, boot_id AS "bootId", + updated_at AS "updatedAt" + FROM hub_thread_machine_status WHERE user_id = ${userId} + `, + }); + const upsert = SqlSchema.void({ + Request: ThreadMachineStatusRow, + execute: (row) => sql` + INSERT INTO hub_thread_machine_status ( + user_id, thread_id, state, detail, boot_id, updated_at + ) + VALUES ( + ${userId}, ${row.threadId}, ${row.state}, ${row.detail}, ${row.bootId}, ${row.updatedAt} + ) + ON CONFLICT (user_id, thread_id) DO UPDATE SET + state = excluded.state, + detail = excluded.detail, + boot_id = excluded.boot_id, + updated_at = excluded.updated_at + `, + }); + + return ThreadMachineStatusStore.of({ + list: () => + selectAll(undefined).pipe(Effect.mapError(sqlOrDecode("ThreadMachineStatusStore.list"))), + put: (row) => upsert(row).pipe(Effect.mapError(sqlOrDecode("ThreadMachineStatusStore.put"))), + remove: (threadId) => + sql` + DELETE FROM hub_thread_machine_status WHERE user_id = ${userId} AND thread_id = ${threadId} + `.pipe( + Effect.asVoid, + Effect.mapError(toPersistenceSqlError("ThreadMachineStatusStore.remove")), + ), + }); +}); + +/** Every repository; requires the hub Postgres `SqlClient` and `HubTenant`. */ export const HubThreadMachineStatePostgresLive = Layer.mergeAll( Layer.effect(RunnerCursorStore, makeRunnerCursorStore), Layer.effect(CheckpointTurnDiffStore, makeCheckpointTurnDiffStore), Layer.effect(ThreadVcsStatusStore, makeThreadVcsStatusStore), + Layer.effect(ThreadMachineStatusStore, makeThreadMachineStatusStore), ); diff --git a/apps/server/src/persistence/Postgres/ProjectionSnapshotQuery.ts b/apps/server/src/persistence/Postgres/ProjectionSnapshotQuery.ts index 1fc964abd8b4..5f7de72000f9 100644 --- a/apps/server/src/persistence/Postgres/ProjectionSnapshotQuery.ts +++ b/apps/server/src/persistence/Postgres/ProjectionSnapshotQuery.ts @@ -47,6 +47,7 @@ import { import { ProjectionCheckpoint } from "../Services/ProjectionCheckpoints.ts"; import { ThreadBackgroundLivenessService } from "../../orchestration/ThreadBackgroundLiveness.ts"; import { ThreadPlanProgressService } from "../../orchestration/ThreadPlanProgress.ts"; +import { ThreadMachineStatusReader, threadMachineShellField } from "../../serverModeHooks.ts"; import { ProjectionProject } from "../Services/ProjectionProjects.ts"; import { ProjectionState } from "../Services/ProjectionState.ts"; import { ProjectionThreadActivity } from "../Services/ProjectionThreadActivities.ts"; @@ -386,6 +387,7 @@ function toPersistenceSqlOrDecodeError(sqlOperation: string, decodeOperation: st const makeProjectionSnapshotQuery = Effect.gen(function* () { const threadBackgroundLiveness = yield* ThreadBackgroundLivenessService; const threadPlanProgress = yield* ThreadPlanProgressService; + const threadMachineStatus = yield* ThreadMachineStatusReader; const sql = yield* SqlClient.SqlClient; const { userId } = yield* HubTenant; // A hub has no checkout to run `git remote` in: projects carry the identity @@ -2304,6 +2306,7 @@ pending_approval_requests AS ( row.threadId, ), planProgress: threadPlanProgress.getThreadPlanProgress(row.threadId), + ...threadMachineShellField(threadMachineStatus, row.threadId), } satisfies OrchestrationThreadShell) : Result.failVoid, ), @@ -2451,6 +2454,7 @@ pending_approval_requests AS ( row.threadId, ), planProgress: threadPlanProgress.getThreadPlanProgress(row.threadId), + ...threadMachineShellField(threadMachineStatus, row.threadId), }), ), updatedAt: updatedAt ?? "1970-01-01T00:00:00.000Z", @@ -2732,6 +2736,7 @@ pending_approval_requests AS ( threadRow.value.threadId, ), planProgress: threadPlanProgress.getThreadPlanProgress(threadRow.value.threadId), + ...threadMachineShellField(threadMachineStatus, threadRow.value.threadId), } satisfies OrchestrationThreadShell); }); diff --git a/apps/server/src/persistence/Postgres/migrations/051_HubThreadMachineServices.ts b/apps/server/src/persistence/Postgres/migrations/051_HubThreadMachineServices.ts new file mode 100644 index 000000000000..379f691c046a --- /dev/null +++ b/apps/server/src/persistence/Postgres/migrations/051_HubThreadMachineServices.ts @@ -0,0 +1,40 @@ +/** + * Hub migration 051: state behind the hub's thread-machine services. + * + * - `hub_thread_machine_status`: the last machine state the directory reported + * per thread, so thread shells show it without asking the directory again. + * + * Every table leads its key with `user_id` and carries the forced tenant policy + * from the baseline (`hubTenantPolicyStatements`). The statements are + * idempotent. + * + * @module 051_HubThreadMachineServices + */ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { hubTenantPolicyStatements } from "./001_HubBaseline.ts"; + +const c = `COLLATE "C"`; + +export const HUB_THREAD_MACHINE_SERVICE_TABLES = ["hub_thread_machine_status"] as const; + +const statements: ReadonlyArray = [ + `CREATE TABLE IF NOT EXISTS hub_thread_machine_status ( + user_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + state text ${c} NOT NULL, + detail text, + boot_id text ${c}, + updated_at text ${c} NOT NULL, + PRIMARY KEY (user_id, thread_id) + )`, + ...HUB_THREAD_MACHINE_SERVICE_TABLES.flatMap(hubTenantPolicyStatements), +]; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + for (const statement of statements) { + yield* sql.unsafe(statement); + } +}); diff --git a/apps/server/src/persistence/Postgres/migrations/index.ts b/apps/server/src/persistence/Postgres/migrations/index.ts index edb40f28cebd..e052933a9940 100644 --- a/apps/server/src/persistence/Postgres/migrations/index.ts +++ b/apps/server/src/persistence/Postgres/migrations/index.ts @@ -10,15 +10,20 @@ import { orderHubMigrations, type HubMigrationEntry } from "../HubMigrator.ts"; import Migration001 from "./001_HubBaseline.ts"; import Migration050 from "./050_HubThreadMachineState.ts"; +import Migration051 from "./051_HubThreadMachineServices.ts"; /** 001–049: hub persistence (orchestration, auth, settings, secrets, attachments). */ const hubPersistenceMigrations: ReadonlyArray = [ { id: 1, name: "HubBaseline", migration: Migration001 }, ]; -/** 050–099: thread-machine state (runner cursors, per-turn diffs, git status). */ +/** + * 050–099: thread-machine state (runner cursors, per-turn diffs, git status, + * machine states, provider snapshots, MCP credentials). + */ const threadMachineMigrations: ReadonlyArray = [ { id: 50, name: "HubThreadMachineState", migration: Migration050 }, + { id: 51, name: "HubThreadMachineServices", migration: Migration051 }, ]; export const hubMigrations = orderHubMigrations([ diff --git a/apps/server/src/persistence/Services/HubThreadMachineState.ts b/apps/server/src/persistence/Services/HubThreadMachineState.ts index d44d3f58cebc..55d7d8c90ad8 100644 --- a/apps/server/src/persistence/Services/HubThreadMachineState.ts +++ b/apps/server/src/persistence/Services/HubThreadMachineState.ts @@ -10,9 +10,11 @@ * - `CheckpointTurnDiffStore`: the patch between two checkpoints of a thread, * captured while the machine was awake. * - `ThreadVcsStatusStore`: the last git status a runner reported per thread. + * - `ThreadMachineStatusStore`: the last machine state the directory reported + * per thread, shown on thread shells without asking the directory again. * * Postgres implementations live in `persistence/Postgres/HubThreadMachineState.ts` - * (hub migration 050); the SQLite implementations back hub mode without a + * (hub migrations 050 and 051); the SQLite implementations back hub mode without a * database URL (tests and local development) and are never built in * standalone mode. * @@ -22,6 +24,7 @@ import { IsoDateTime, NonNegativeInt, ThreadId, + ThreadMachineState, TrimmedNonEmptyString, VcsStatusLocalResult, VcsStatusRemoteResult, @@ -114,3 +117,26 @@ export class ThreadVcsStatusStore extends Context.Service< ThreadVcsStatusStore, ThreadVcsStatusStoreShape >()("t3/persistence/Services/HubThreadMachineState/ThreadVcsStatusStore") {} + +export const ThreadMachineStatusRow = Schema.Struct({ + threadId: ThreadId, + state: ThreadMachineState, + detail: Schema.NullOr(Schema.String), + bootId: Schema.NullOr(Schema.String), + updatedAt: IsoDateTime, +}); +export type ThreadMachineStatusRow = typeof ThreadMachineStatusRow.Type; + +export interface ThreadMachineStatusStoreShape { + readonly list: () => Effect.Effect< + ReadonlyArray, + ProjectionRepositoryError + >; + readonly put: (row: ThreadMachineStatusRow) => Effect.Effect; + readonly remove: (threadId: ThreadId) => Effect.Effect; +} + +export class ThreadMachineStatusStore extends Context.Service< + ThreadMachineStatusStore, + ThreadMachineStatusStoreShape +>()("t3/persistence/Services/HubThreadMachineState/ThreadMachineStatusStore") {} diff --git a/apps/server/src/serverModeHooks.ts b/apps/server/src/serverModeHooks.ts index ed2580c7e975..85907c7672ec 100644 --- a/apps/server/src/serverModeHooks.ts +++ b/apps/server/src/serverModeHooks.ts @@ -13,6 +13,7 @@ import type { OrchestrationDispatchCommandError, ProjectId, ThreadId, + ThreadMachineStatus, } from "@t3tools/contracts"; import * as Context from "effect/Context"; import type * as Effect from "effect/Effect"; @@ -69,3 +70,27 @@ export class HubThreadCheckouts extends Context.Reference null }, ) {} + +export interface ThreadMachineStatusReaderShape { + /** + * The latest machine state a hub knows for a thread, read synchronously + * while thread shells are mapped. `null` means no machine is known; + * `undefined` (standalone) omits the shell field entirely. + */ + readonly get: (threadId: ThreadId) => ThreadMachineStatus | null | undefined; +} + +/** Thread-machine state for thread shells; standalone servers never report one. */ +export class ThreadMachineStatusReader extends Context.Reference( + "t3/serverModeHooks/ThreadMachineStatusReader", + { defaultValue: () => ({ get: () => undefined }) }, +) {} + +/** Spreads a thread's machine state into a shell; nothing in standalone mode. */ +export const threadMachineShellField = ( + reader: ThreadMachineStatusReaderShape, + threadId: ThreadId, +): { readonly machine?: ThreadMachineStatus | null } => { + const machine = reader.get(threadId); + return machine === undefined ? {} : { machine }; +}; diff --git a/packages/contracts/src/runner.ts b/packages/contracts/src/runner.ts index 49e436827a36..81e0d67cbf28 100644 --- a/packages/contracts/src/runner.ts +++ b/packages/contracts/src/runner.ts @@ -181,7 +181,7 @@ export function parseThreadCheckoutPath( // ── Thread machine directory ─────────────────────────────────────────── /** Lifecycle state reported by the machine directory (shared with thread shells). */ -export { ThreadMachineState } from "./threadMachine.ts"; +export { DEFAULT_BRANCH_BASE_REF, ThreadMachineState } from "./threadMachine.ts"; /** Repository a new thread machine checks out, from the project's identity. */ export const ThreadMachineRepository = Schema.Struct({ diff --git a/packages/contracts/src/threadMachine.ts b/packages/contracts/src/threadMachine.ts index 7f4918021cfc..09c477859387 100644 --- a/packages/contracts/src/threadMachine.ts +++ b/packages/contracts/src/threadMachine.ts @@ -24,3 +24,28 @@ export const ThreadMachineStatus = Schema.Struct({ updatedAt: IsoDateTime, }); export type ThreadMachineStatus = typeof ThreadMachineStatus.Type; + +/** Thread activity kinds a hub records for its thread machines. */ +export const THREAD_MACHINE_ACTIVITY_KINDS = { + /** Every machine state transition; payload `ThreadMachineStateActivityPayload`. */ + state: "thread-machine.state", + /** The runner is cloning or preparing the thread's checkout. */ + checkoutPreparing: "thread-machine.checkout.preparing", + /** Bootstrap or checkout preparation failed; tone `error`, payload `{ detail }`. */ + failed: "thread-machine.failed", +} as const; + +/** Payload of a `thread-machine.state` activity (tone `error` when `failed`). */ +export const ThreadMachineStateActivityPayload = Schema.Struct({ + state: ThreadMachineState, + detail: Schema.NullOr(Schema.String), + bootId: Schema.NullOr(Schema.String), +}); +export type ThreadMachineStateActivityPayload = typeof ThreadMachineStateActivityPayload.Type; + +/** + * Hub mode: a `prepareWorktree.baseBranch` of `"HEAD"` means "the repository's + * default branch". The thread's runner resolves it from `origin/HEAD`, so a + * client that has not picked a base ref never needs a branch list first. + */ +export const DEFAULT_BRANCH_BASE_REF = "HEAD"; From d2dd3c51394c42aeced1cdd21dff0d30e4d2fd6f Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 00:51:47 +0000 Subject: [PATCH 35/58] feat(server): add threadMachines.wake and threadMachines.pause for hub clients wake resumes or recreates a thread's machine through the connection pool and answers within two seconds with the state reached so far; pause closes the runner connection and reports the machine idle unless a turn runs. Both answer ThreadMachineControlError (unsupported on standalone servers, not-found, busy, unavailable). Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/auth/RpcAuthorization.ts | 2 + apps/server/src/hub/HubLayers.ts | 4 +- apps/server/src/hub/RunnerConnectionPool.ts | 32 +++-- .../src/hub/ThreadMachineControls.test.ts | 121 ++++++++++++++++++ apps/server/src/hub/ThreadMachineControls.ts | 109 ++++++++++++++++ apps/server/src/server.test.ts | 20 +++ apps/server/src/serverModeHooks.ts | 18 +++ apps/server/src/ws.ts | 27 ++++ packages/contracts/src/rpc.ts | 33 +++++ packages/contracts/src/threadMachine.ts | 41 +++++- 10 files changed, 397 insertions(+), 10 deletions(-) create mode 100644 apps/server/src/hub/ThreadMachineControls.test.ts create mode 100644 apps/server/src/hub/ThreadMachineControls.ts diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index fdaa2444f384..3575ae6fa1e0 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -50,6 +50,8 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.serverGetResourceTelemetryHistory]: AuthOrchestrationReadScope, [WS_METHODS.serverRetryResourceTelemetry]: AuthOrchestrationOperateScope, [WS_METHODS.serverGetUsageSummary]: AuthOrchestrationReadScope, + [WS_METHODS.threadMachinesWake]: AuthOrchestrationOperateScope, + [WS_METHODS.threadMachinesPause]: AuthOrchestrationOperateScope, [WS_METHODS.serverSignalProcess]: AuthOrchestrationOperateScope, [WS_METHODS.serverReportClientActivity]: AuthOrchestrationReadScope, [WS_METHODS.serverReportHostPowerState]: AuthOrchestrationOperateScope, diff --git a/apps/server/src/hub/HubLayers.ts b/apps/server/src/hub/HubLayers.ts index 068037456677..29489f3e63ba 100644 --- a/apps/server/src/hub/HubLayers.ts +++ b/apps/server/src/hub/HubLayers.ts @@ -49,6 +49,7 @@ import { makeRemoteProviderDriver } from "./RemoteProviderDriver.ts"; import * as RemoteSessionRegistry from "./RemoteSessionRegistry.ts"; import * as RunnerConnectionPool from "./RunnerConnectionPool.ts"; import * as RunnerEventDelivery from "./RunnerEventDelivery.ts"; +import * as ThreadMachineControls from "./ThreadMachineControls.ts"; import * as ThreadMachineStates from "./ThreadMachineStates.ts"; /** @@ -145,11 +146,12 @@ export const hubSourceControlRepositoryServiceLayer = Layer.effect( }), ); -/** Hooks consumed by the orchestration reactors and command dispatch. */ +/** Hooks consumed by the orchestration reactors, command dispatch and the client API. */ export const hubReactorHooksLayer = Layer.mergeAll( Layer.succeed(DeterministicIngestionCommandIds, true), hubCheckoutGitProbeLayer, HubThreadCheckouts.layer, + ThreadMachineControls.layer, ); /** Background work that needs the reactors: delivery acks, resume, machine release. */ diff --git a/apps/server/src/hub/RunnerConnectionPool.ts b/apps/server/src/hub/RunnerConnectionPool.ts index bee3dcc62d86..ae5cf12f4a05 100644 --- a/apps/server/src/hub/RunnerConnectionPool.ts +++ b/apps/server/src/hub/RunnerConnectionPool.ts @@ -129,6 +129,11 @@ export interface RunnerConnectionPoolShape { readonly setContextResolver: (resolver: ThreadMachineContextResolver) => Effect.Effect; /** Closes the connection and retires the machine. */ readonly release: (threadId: ThreadId) => Effect.Effect; + /** + * Closes the connection now and tells the directory the machine is idle, + * unless a busy mark (a running turn) holds it. Never wakes a machine. + */ + readonly idle: (threadId: ThreadId) => Effect.Effect<"idle" | "busy">; /** * Closes connections idle for `idleTimeout` and notifies the directory. * Runs every `idleCheckInterval`; exposed so callers can sweep on demand. @@ -611,6 +616,16 @@ export const make = (options: RunnerConnectionPoolOptions = {}) => }), ); + const reportIdle = (threadId: ThreadId) => + directory.idle(threadId).pipe( + Effect.catch((error) => + Effect.logWarning("machine directory idle notification failed", { + threadId, + detail: error.message, + }), + ), + ); + // Idle connections close and tell the platform the machine may sleep. const sweepIdle = Effect.gen(function* () { const now = yield* Clock.currentTimeMillis; @@ -621,14 +636,7 @@ export const make = (options: RunnerConnectionPoolOptions = {}) => if (now - entry.lastActivityAt < Duration.toMillis(idleTimeout)) continue; yield* Effect.logInfo("closing idle runner connection", { threadId }); yield* closeSlot(threadId, slot, true); - yield* directory.idle(threadId).pipe( - Effect.catch((error) => - Effect.logWarning("machine directory idle notification failed", { - threadId, - detail: error.message, - }), - ), - ); + yield* reportIdle(threadId); } }); yield* sweepIdle.pipe( @@ -702,6 +710,14 @@ export const make = (options: RunnerConnectionPoolOptions = {}) => Effect.sync(() => { contextResolver = resolver; }), + idle: (threadId) => + Effect.gen(function* () { + if ((activity.get(threadId)?.busy.size ?? 0) > 0) return "busy" as const; + const slot = slots.get(threadId); + if (slot) yield* closeSlot(threadId, slot, true); + yield* reportIdle(threadId); + return "idle" as const; + }), release: (threadId) => Effect.gen(function* () { const slot = slots.get(threadId); diff --git a/apps/server/src/hub/ThreadMachineControls.test.ts b/apps/server/src/hub/ThreadMachineControls.test.ts new file mode 100644 index 000000000000..ac6c24a19e3b --- /dev/null +++ b/apps/server/src/hub/ThreadMachineControls.test.ts @@ -0,0 +1,121 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { it } from "@effect/vitest"; +import { type OrchestrationThreadShell, ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import { describe, expect } from "vite-plus/test"; + +import { ProjectionSnapshotQuery } from "../orchestration/Services/ProjectionSnapshotQuery.ts"; +import { HubThreadMachineStateSqliteLive } from "../persistence/Layers/HubThreadMachineState.ts"; +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; +import { ThreadMachineControls } from "../serverModeHooks.ts"; +import { + type FakeMachine, + MachineDirectory, + makeFakeMachineDirectory, +} from "./MachineDirectory.ts"; +import { make as makePool, RunnerConnectionPool } from "./RunnerConnectionPool.ts"; +import * as ThreadMachineControlsLayer from "./ThreadMachineControls.ts"; +import * as ThreadMachineStates from "./ThreadMachineStates.ts"; +import { fakeRunner, fakeRunnerHello, serveRunner } from "./testUtils/runnerServer.ts"; + +const threadId = ThreadId.make("thread-controls"); + +const StoreLayer = HubThreadMachineStateSqliteLive.pipe( + Layer.provideMerge(SqlitePersistenceMemory), + Layer.provideMerge(NodeServices.layer), +); + +const projectionsWith = (threads: ReadonlyArray) => + Layer.succeed( + ProjectionSnapshotQuery, + ProjectionSnapshotQuery.of({ + getThreadShellById: (id: ThreadId) => + Effect.succeed( + threads.includes(id) + ? Option.some({ id } as unknown as OrchestrationThreadShell) + : Option.none(), + ), + } as unknown as ProjectionSnapshotQuery["Service"]), + ); + +const setup = (onWake: (runnerUrl: string) => FakeMachine) => + Effect.gen(function* () { + const runner = yield* serveRunner( + fakeRunner({ + "runner.hello": (input) => Effect.succeed(fakeRunnerHello({ threadId: input.threadId })), + }), + ); + const fake = yield* makeFakeMachineDirectory({ + initial: [[threadId, { state: "paused", bootId: "boot-1" }]], + onWake: () => onWake(runner.url), + }); + const directoryLayer = ThreadMachineStates.observedMachineDirectoryLayer.pipe( + Layer.provide(Layer.succeed(MachineDirectory, fake.directory)), + Layer.provideMerge(ThreadMachineStates.layer), + ); + const context = yield* Layer.build( + ThreadMachineControlsLayer.layer.pipe( + Layer.provideMerge( + Layer.effect( + RunnerConnectionPool, + makePool({ wakePollInterval: "5 millis", idleCheckInterval: "1 hour" }), + ), + ), + Layer.provideMerge(directoryLayer), + Layer.provide(projectionsWith([threadId])), + ), + ); + const controls = yield* ThreadMachineControls.pipe(Effect.provide(context)); + const pool = yield* RunnerConnectionPool.pipe(Effect.provide(context)); + const calls = fake.calls.pipe( + Effect.map((entries) => + entries.map((call) => (call.wake ? `${call.method}+wake` : call.method)), + ), + ); + return { controls: controls!, pool, calls }; + }); + +describe("thread machine controls", () => { + it.live("wakes a sleeping machine and releases it again", () => + Effect.gen(function* () { + const { controls, pool, calls } = yield* setup((url) => ({ + state: "running", + runnerUrl: url, + bootId: "boot-1", + })); + const woken = yield* controls.wake(threadId); + expect(woken?.state).toBe("running"); + expect(Option.isSome(yield* pool.current(threadId))).toBe(true); + + const paused = yield* controls.pause(threadId); + expect(paused?.state).toBe("running"); + expect(Option.isNone(yield* pool.current(threadId))).toBe(true); + expect(yield* calls).toEqual(["ensure+wake", "idle"]); + }).pipe(Effect.scoped, Effect.provide(StoreLayer)), + ); + + it.live("refuses to release a machine while a turn runs", () => + Effect.gen(function* () { + const { controls, pool } = yield* setup((url) => ({ state: "running", runnerUrl: url })); + yield* pool.setBusy(threadId, "turn", true); + const error = yield* controls.pause(threadId).pipe(Effect.flip); + expect(error.reason).toBe("busy"); + }).pipe(Effect.scoped, Effect.provide(StoreLayer)), + ); + + it.live("reports failures and unknown threads with typed reasons", () => + Effect.gen(function* () { + const { controls } = yield* setup(() => ({ state: "failed", detail: "No capacity" })); + const failed = yield* controls.wake(threadId).pipe(Effect.flip); + expect(failed).toMatchObject({ reason: "unavailable", state: "failed" }); + const missing = yield* controls.wake(ThreadId.make("thread-missing")).pipe(Effect.flip); + expect(missing.reason).toBe("not-found"); + const signIn = yield* controls + .pause(ThreadId.make("aldo-provider-sign-in")) + .pipe(Effect.flip); + expect(signIn.reason).toBe("not-found"); + }).pipe(Effect.scoped, Effect.provide(StoreLayer)), + ); +}); diff --git a/apps/server/src/hub/ThreadMachineControls.ts b/apps/server/src/hub/ThreadMachineControls.ts new file mode 100644 index 000000000000..0dabc4383b34 --- /dev/null +++ b/apps/server/src/hub/ThreadMachineControls.ts @@ -0,0 +1,109 @@ +/** + * Client controls for thread machines (`threadMachines.wake`, + * `threadMachines.pause`). + * + * - `wake` resumes or recreates the machine through the connection pool (the + * same path a turn takes, so the runner is connected and its handlers run + * once it is up). It waits at most `WAKE_REPLY_WAIT` for the result and + * otherwise returns the state the directory reported so far; the rest of + * the wake continues in the background and is visible on the thread shell. + * - `pause` releases the hub's hold: it closes the runner connection and + * reports the machine idle, after which the platform pauses it. A running + * turn refuses with `busy`. + * + * Only existing, unarchived threads are controlled; the provider sign-in + * machine is managed by the sign-in flow. + * + * @module hub/ThreadMachineControls + */ +import { + PROVIDER_SIGN_IN_THREAD_ID, + type ThreadId, + ThreadMachineControlError, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; + +import { ProjectionSnapshotQuery } from "../orchestration/Services/ProjectionSnapshotQuery.ts"; +import { ThreadMachineControls, type ThreadMachineControlsShape } from "../serverModeHooks.ts"; +import { RunnerConnectionPool } from "./RunnerConnectionPool.ts"; +import { ThreadMachineStates, toShellMachineStatus } from "./ThreadMachineStates.ts"; + +/** How long `wake` waits for the machine before answering with its progress. */ +const WAKE_REPLY_WAIT = "2 seconds"; + +export const make = Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + const states = yield* ThreadMachineStates; + const projections = yield* ProjectionSnapshotQuery; + const scope = yield* Effect.scope; + + const requireThread = (operation: string, threadId: ThreadId) => + Effect.gen(function* () { + const thread = + threadId === PROVIDER_SIGN_IN_THREAD_ID + ? Option.none() + : yield* projections + .getThreadShellById(threadId) + .pipe(Effect.orElseSucceed(() => Option.none())); + if (Option.isNone(thread)) { + return yield* new ThreadMachineControlError({ + operation, + reason: "not-found", + detail: `Thread ${threadId} does not exist or is archived.`, + }); + } + }); + + const current = (threadId: ThreadId) => toShellMachineStatus(states.get(threadId)); + + const wake: ThreadMachineControlsShape["wake"] = (threadId) => + Effect.gen(function* () { + yield* requireThread("threadMachines.wake", threadId); + const fiber = yield* pool + .use(threadId, { wake: true, operation: "threadMachines.wake" }, () => Effect.void) + .pipe( + Effect.tapError((error) => + Effect.logInfo("thread machine wake did not complete", { + threadId, + reason: error.reason, + detail: error.detail, + }), + ), + Effect.forkIn(scope), + ); + const settled = yield* Fiber.await(fiber).pipe(Effect.timeoutOption(WAKE_REPLY_WAIT)); + if (Option.isSome(settled) && Exit.isFailure(settled.value)) { + const failure = Exit.findErrorOption(settled.value); + if (Option.isSome(failure)) { + return yield* new ThreadMachineControlError({ + operation: "threadMachines.wake", + reason: "unavailable", + ...(failure.value.state !== undefined ? { state: failure.value.state } : {}), + detail: failure.value.detail, + }); + } + } + return current(threadId); + }); + + const pause: ThreadMachineControlsShape["pause"] = (threadId) => + Effect.gen(function* () { + yield* requireThread("threadMachines.pause", threadId); + if ((yield* pool.idle(threadId)) === "busy") { + return yield* new ThreadMachineControlError({ + operation: "threadMachines.pause", + reason: "busy", + detail: "A turn is running on this thread's machine.", + }); + } + return current(threadId); + }); + + return { wake, pause } satisfies ThreadMachineControlsShape; +}); + +export const layer = Layer.effect(ThreadMachineControls, make); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 57429be7169c..b6cdd6c12c75 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -4762,6 +4762,26 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("answers thread machine controls as unsupported on a standalone server", () => + Effect.gen(function* () { + yield* buildAppUnderTest(); + const wsUrl = yield* getWsServerUrl("/ws"); + const threadId = ThreadId.make("thread-standalone"); + const [wake, pause] = yield* Effect.scoped( + withWsRpcClient(wsUrl, (client) => + Effect.all([ + client[WS_METHODS.threadMachinesWake]({ threadId }).pipe(Effect.flip), + client[WS_METHODS.threadMachinesPause]({ threadId }).pipe(Effect.flip), + ]), + ), + ); + for (const error of [wake, pause]) { + assert.equal(error._tag, "ThreadMachineControlError"); + assert.equal(error._tag === "ThreadMachineControlError" && error.reason, "unsupported"); + } + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("uploads image bytes through a signed URL issued by websocket rpc", () => Effect.gen(function* () { const config = yield* buildAppUnderTest(); diff --git a/apps/server/src/serverModeHooks.ts b/apps/server/src/serverModeHooks.ts index 85907c7672ec..71b0f369bb47 100644 --- a/apps/server/src/serverModeHooks.ts +++ b/apps/server/src/serverModeHooks.ts @@ -13,6 +13,7 @@ import type { OrchestrationDispatchCommandError, ProjectId, ThreadId, + ThreadMachineControlError, ThreadMachineStatus, } from "@t3tools/contracts"; import * as Context from "effect/Context"; @@ -94,3 +95,20 @@ export const threadMachineShellField = ( const machine = reader.get(threadId); return machine === undefined ? {} : { machine }; }; + +export interface ThreadMachineControlsShape { + /** Resumes or recreates the thread's machine; returns once the directory accepted. */ + readonly wake: ( + threadId: ThreadId, + ) => Effect.Effect; + /** Releases the hub's hold on the machine so the platform may pause it. */ + readonly pause: ( + threadId: ThreadId, + ) => Effect.Effect; +} + +/** `threadMachines.wake` / `threadMachines.pause`; `null` (standalone) answers `unsupported`. */ +export class ThreadMachineControls extends Context.Reference( + "t3/serverModeHooks/ThreadMachineControls", + { defaultValue: () => null }, +) {} diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 8dea97052191..7e2e7df9f713 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -58,6 +58,7 @@ import { RpcClientId, EnvironmentAuthorizationError, ThreadId, + ThreadMachineControlError, type TerminalAttachStreamEvent, type TerminalError, type TerminalEvent, @@ -129,6 +130,7 @@ import * as AzureDevOpsCli from "./sourceControl/AzureDevOpsCli.ts"; import * as BitbucketApi from "./sourceControl/BitbucketApi.ts"; import * as GitHubCli from "./sourceControl/GitHubCli.ts"; import * as GitLabCli from "./sourceControl/GitLabCli.ts"; +import { ThreadMachineControls } from "./serverModeHooks.ts"; import * as AuthConnectorManager from "./authConnector/AuthConnectorManager.ts"; import * as SourceControlProviderRegistry from "./sourceControl/SourceControlProviderRegistry.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; @@ -422,6 +424,14 @@ function readClientAnalyticsProps(request: HttpServerRequest.HttpServerRequest) }; } +/** Thread-machine controls exist only on a hub (the `threadMachines` capability). */ +const unsupportedThreadMachineControl = (operation: string) => + new ThreadMachineControlError({ + operation, + reason: "unsupported", + detail: "This server does not run threads on thread machines.", + }); + const makeWsRpcLayer = ( currentSession: EnvironmentAuth.AuthenticatedSession, clientOrigin: OrchestrationClientOrigin, @@ -474,6 +484,7 @@ const makeWsRpcLayer = ( const providerMaintenanceRunner = yield* ProviderMaintenanceRunner.ProviderMaintenanceRunner; const serverSelfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; const config = yield* ServerConfig.ServerConfig; + const threadMachineControls = yield* ThreadMachineControls; const lifecycleEvents = yield* ServerLifecycleEvents.ServerLifecycleEvents; const serverSettings = yield* ServerSettings.ServerSettingsService; const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; @@ -1414,6 +1425,22 @@ const makeWsRpcLayer = ( observeRpcEffect(WS_METHODS.serverGetUsageSummary, usage.readSummary(input), { "rpc.aggregate": "server", }), + [WS_METHODS.threadMachinesWake]: ({ threadId }) => + observeRpcEffect( + WS_METHODS.threadMachinesWake, + threadMachineControls === null + ? Effect.fail(unsupportedThreadMachineControl("threadMachines.wake")) + : threadMachineControls.wake(threadId), + { "rpc.aggregate": "thread-machine" }, + ), + [WS_METHODS.threadMachinesPause]: ({ threadId }) => + observeRpcEffect( + WS_METHODS.threadMachinesPause, + threadMachineControls === null + ? Effect.fail(unsupportedThreadMachineControl("threadMachines.pause")) + : threadMachineControls.pause(threadId), + { "rpc.aggregate": "thread-machine" }, + ), [WS_METHODS.serverRetryResourceTelemetry]: (_input) => observeRpcEffect(WS_METHODS.serverRetryResourceTelemetry, resourceTelemetry.retry, { "rpc.aggregate": "server", diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 4100cba6fbaf..b70eb62af195 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -213,6 +213,11 @@ import { SourceControlRepositoryListResult, SourceControlRepositoryLookupInput, } from "./sourceControl.ts"; +import { + ThreadMachineControlError, + ThreadMachineControlInput, + ThreadMachineStatus, +} from "./threadMachine.ts"; import { VcsError } from "./vcs.ts"; export const WS_METHODS = { @@ -305,6 +310,10 @@ export const WS_METHODS = { serverGetBackgroundPolicy: "server.getBackgroundPolicy", serverGetUsageSummary: "server.getUsageSummary", + // Thread machine methods (hub mode; see the threadMachines capability) + threadMachinesWake: "threadMachines.wake", + threadMachinesPause: "threadMachines.pause", + // Cloud environment methods cloudGetRelayClientStatus: "cloud.getRelayClientStatus", cloudInstallRelayClient: "cloud.installRelayClient", @@ -485,6 +494,28 @@ export const WsServerRetryResourceTelemetryRpc = Rpc.make(WS_METHODS.serverRetry error: EnvironmentAuthorizationError, }); +/** + * Hub mode: resumes or recreates a thread's machine. Returns once the + * directory accepted the request (usually still `starting`); progress arrives + * on the thread shell's `machine` and as `thread-machine.state` activities. + */ +export const WsThreadMachinesWakeRpc = Rpc.make(WS_METHODS.threadMachinesWake, { + payload: ThreadMachineControlInput, + success: Schema.NullOr(ThreadMachineStatus), + error: Schema.Union([ThreadMachineControlError, EnvironmentAuthorizationError]), +}); + +/** + * Hub mode: releases the hub's hold on a thread's machine (closes its runner + * connection and reports it idle); the platform pauses it after its idle + * delay. Refused with `busy` while a turn runs. + */ +export const WsThreadMachinesPauseRpc = Rpc.make(WS_METHODS.threadMachinesPause, { + payload: ThreadMachineControlInput, + success: Schema.NullOr(ThreadMachineStatus), + error: Schema.Union([ThreadMachineControlError, EnvironmentAuthorizationError]), +}); + export const WsServerGetUsageSummaryRpc = Rpc.make(WS_METHODS.serverGetUsageSummary, { payload: UsageSummaryInput, success: UsageSummary, @@ -1095,6 +1126,8 @@ export const WsRpcGroup = RpcGroup.make( WsServerGetResourceTelemetryHistoryRpc, WsServerRetryResourceTelemetryRpc, WsServerGetUsageSummaryRpc, + WsThreadMachinesWakeRpc, + WsThreadMachinesPauseRpc, WsServerSignalProcessRpc, WsServerReportClientActivityRpc, WsServerReportHostPowerStateRpc, diff --git a/packages/contracts/src/threadMachine.ts b/packages/contracts/src/threadMachine.ts index 09c477859387..8b9130c3ff20 100644 --- a/packages/contracts/src/threadMachine.ts +++ b/packages/contracts/src/threadMachine.ts @@ -1,6 +1,6 @@ import * as Schema from "effect/Schema"; -import { IsoDateTime } from "./baseSchemas.ts"; +import { IsoDateTime, ThreadId } from "./baseSchemas.ts"; /** * Lifecycle of the machine a hub-mode thread runs on. Standalone servers never @@ -49,3 +49,42 @@ export type ThreadMachineStateActivityPayload = typeof ThreadMachineStateActivit * client that has not picked a base ref never needs a branch list first. */ export const DEFAULT_BRANCH_BASE_REF = "HEAD"; + +/** + * The reserved thread (and project) whose machine runs provider sign-in in hub + * mode. It has no repository, never runs agent sessions, and never appears in + * the thread list. + */ +export const PROVIDER_SIGN_IN_THREAD_ID = "aldo-provider-sign-in"; + +// ── Client controls (hub mode) ───────────────────────────────────────── + +/** `threadMachines.wake` / `threadMachines.pause` payload. */ +export const ThreadMachineControlInput = Schema.Struct({ threadId: ThreadId }); +export type ThreadMachineControlInput = typeof ThreadMachineControlInput.Type; + +export const ThreadMachineControlErrorReason = Schema.Literals([ + /** The server is not a hub (no `threadMachines` capability). */ + "unsupported", + /** The thread does not exist, is archived, or is reserved. */ + "not-found", + /** A turn is running on the machine; it cannot be released now. */ + "busy", + /** The machine directory or the machine failed; `state` and `detail` say how. */ + "unavailable", +]); +export type ThreadMachineControlErrorReason = typeof ThreadMachineControlErrorReason.Type; + +export class ThreadMachineControlError extends Schema.TaggedErrorClass()( + "ThreadMachineControlError", + { + operation: Schema.String, + reason: ThreadMachineControlErrorReason, + state: Schema.optional(ThreadMachineState), + detail: Schema.String, + }, +) { + override get message(): string { + return this.detail; + } +} From 5acc1ecec4da70826bd3cc91a11889a9887a5327 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 00:52:28 +0000 Subject: [PATCH 36/58] feat(runner): resolve the HEAD base ref to the repository's default branch A prepareWorktree.baseBranch of HEAD (DEFAULT_BRANCH_BASE_REF) now means the repository's default branch: the runner branches from origin/HEAD, recording it with git remote set-head --auto when the checkout lacks it, and the hub asks the machine directory for the default ref. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/runner/RunnerCheckout.test.ts | 45 +++++++++++++++++++ apps/server/src/runner/RunnerCheckout.ts | 31 +++++++++++-- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/apps/server/src/runner/RunnerCheckout.test.ts b/apps/server/src/runner/RunnerCheckout.test.ts index cf817ce6c1e0..a20a515da648 100644 --- a/apps/server/src/runner/RunnerCheckout.test.ts +++ b/apps/server/src/runner/RunnerCheckout.test.ts @@ -113,6 +113,51 @@ describe("RunnerCheckout", () => { }).pipe(Effect.provide(TestLayer)), ); + it.effect("branches from the repository's default branch for the HEAD base ref", () => + Effect.gen(function* () { + const origin = makeOrigin(); + // The origin's default branch is main; the clone checks out a work branch first. + git(origin.root, ["--git-dir", "origin.git", "symbolic-ref", "HEAD", "refs/heads/main"]); + const checkout = NodePath.join(origin.root, "t", threadId); + const runner = yield* RunnerCheckout; + yield* runner.prepare({ + threadId, + checkout, + repository: { url: origin.url, ref: null }, + branch: "feature/existing", + baseRef: null, + }); + + const fromDefault = yield* runner.prepare({ + threadId, + checkout, + repository: { url: origin.url, ref: null }, + branch: "t3/from-default", + baseRef: "HEAD", + }); + expect(fromDefault.branch).toBe("t3/from-default"); + expect(fromDefault.headCommit).toBe(origin.mainHead); + + // A checkout without a recorded origin/HEAD learns it from the remote. + git(checkout, ["remote", "set-head", "origin", "--delete"]); + git(origin.root, [ + "--git-dir", + "origin.git", + "symbolic-ref", + "HEAD", + "refs/heads/feature/existing", + ]); + const learned = yield* runner.prepare({ + threadId, + checkout, + repository: { url: origin.url, ref: null }, + branch: "t3/from-learned-default", + baseRef: "HEAD", + }); + expect(learned.headCommit).toBe(origin.featureHead); + }).pipe(Effect.provide(TestLayer)), + ); + it.effect("initializes a blank project without a repository", () => Effect.gen(function* () { const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "runner-checkout-blank-")); diff --git a/apps/server/src/runner/RunnerCheckout.ts b/apps/server/src/runner/RunnerCheckout.ts index 61e8bf50003a..eb9beaa2d12f 100644 --- a/apps/server/src/runner/RunnerCheckout.ts +++ b/apps/server/src/runner/RunnerCheckout.ts @@ -10,6 +10,9 @@ * 3. When `branch` is not checked out: switch to it if it exists locally; * otherwise create it from `origin/` (a branch pushed from an * earlier machine), else from `baseRef` (remote-tracking first), else HEAD. + * A `baseRef` of `HEAD` (`DEFAULT_BRANCH_BASE_REF`) means the repository's + * default branch: `origin/HEAD`, which `git remote set-head --auto` fills + * in when a clone did not record it. * * Network access happens only when the checkout is cloned or a branch has to * be created, so steady-state calls run two or three local git commands. @@ -17,6 +20,7 @@ * @module runner/RunnerCheckout */ import { + DEFAULT_BRANCH_BASE_REF, RunnerCheckoutError, type RunnerPrepareCheckoutInput, type RunnerPrepareCheckoutResult, @@ -129,6 +133,23 @@ export const make = Effect.gen(function* () { Effect.map((sha) => sha !== null), ); + /** `origin`'s default branch as a remote-tracking ref, recording it when missing. */ + const defaultBranchRefs = (withOrigin: boolean) => + Effect.gen(function* () { + if (!withOrigin) return []; + const readDefault = probe(checkout, [ + "symbolic-ref", + "--quiet", + "refs/remotes/origin/HEAD", + ]); + const recorded = + (yield* readDefault) ?? + (yield* probe(checkout, ["remote", "set-head", "origin", "--auto"]).pipe( + Effect.andThen(readDefault), + )); + return recorded === null ? [] : [recorded]; + }); + if (input.branch !== null && (yield* currentBranch()) !== input.branch) { const branch = input.branch; if (yield* refExists(`refs/heads/${branch}`)) { @@ -137,11 +158,15 @@ export const make = Effect.gen(function* () { if (hasOrigin && !created) { yield* run("fetch", checkout, ["fetch", "--prune", "origin"], FETCH_TIMEOUT_MS); } + const baseCandidates = + input.baseRef === null + ? [] + : input.baseRef === DEFAULT_BRANCH_BASE_REF + ? yield* defaultBranchRefs(hasOrigin) + : [...(hasOrigin ? [`refs/remotes/origin/${input.baseRef}`] : []), input.baseRef]; const candidates = [ ...(hasOrigin ? [`refs/remotes/origin/${branch}`] : []), - ...(input.baseRef !== null - ? [...(hasOrigin ? [`refs/remotes/origin/${input.baseRef}`] : []), input.baseRef] - : []), + ...baseCandidates, "HEAD", ]; let startPoint: string | null = null; From 10ceee2c63505022bce6a6f75a393318095c7944 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 00:58:01 +0000 Subject: [PATCH 37/58] feat(server): list a hub project's branches from the platform without a machine vcs.listRefs on a virtual project root (/workspace/p/) is served from GET {T3CODE_HUB_MACHINES_URL}/repositories/refs for the project's recorded repository, mapped to the ordinary result with the default branch marked isDefault and cached for 30 s. Blank projects list nothing, non-GitHub repositories an empty list, and missing App access a GitCommandError that says so. A sleeping thread's checkout gets the same listing with its branch marked current instead of an asleep error, so listing never wakes a machine. The machine directory also gains the provider-homes list and sign-out calls. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/hub/HubRepositoryRefs.test.ts | 184 ++++++++++++++++++ apps/server/src/hub/HubRepositoryRefs.ts | 151 ++++++++++++++ apps/server/src/hub/HubVcs.ts | 34 +++- apps/server/src/hub/MachineDirectory.test.ts | 48 +++++ apps/server/src/hub/MachineDirectory.ts | 147 +++++++++++++- packages/contracts/src/runner.ts | 58 ++++++ 6 files changed, 613 insertions(+), 9 deletions(-) create mode 100644 apps/server/src/hub/HubRepositoryRefs.test.ts create mode 100644 apps/server/src/hub/HubRepositoryRefs.ts diff --git a/apps/server/src/hub/HubRepositoryRefs.test.ts b/apps/server/src/hub/HubRepositoryRefs.test.ts new file mode 100644 index 000000000000..3dfd4be50550 --- /dev/null +++ b/apps/server/src/hub/HubRepositoryRefs.test.ts @@ -0,0 +1,184 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { it } from "@effect/vitest"; +import { + type OrchestrationThreadShell, + ProjectId, + type RepositoryIdentity, + ThreadId, +} from "@t3tools/contracts"; +import { projectVirtualRoot, threadCheckoutPath } from "@t3tools/contracts/runner"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import { describe, expect } from "vite-plus/test"; + +import * as GitWorkflowService from "../git/GitWorkflowService.ts"; +import { ProjectionSnapshotQuery } from "../orchestration/Services/ProjectionSnapshotQuery.ts"; +import { HubThreadMachineStateSqliteLive } from "../persistence/Layers/HubThreadMachineState.ts"; +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; +import { RepositoryIdentityResolver } from "../project/RepositoryIdentityResolver.ts"; +import { toListRefsResult } from "./HubRepositoryRefs.ts"; +import { hubGitWorkflowServiceLayer, hubVcsStatusCacheLayer } from "./HubVcs.ts"; +import { MachineDirectory, makeFakeMachineDirectory } from "./MachineDirectory.ts"; +import { make as makePool, RunnerConnectionPool } from "./RunnerConnectionPool.ts"; + +const REPO = "https://github.com/acme/widgets"; +const repoProject = ProjectId.make("project-repo"); +const blankProject = ProjectId.make("project-blank"); +const lockedProject = ProjectId.make("project-locked"); +const gitlabProject = ProjectId.make("project-gitlab"); +const sleepingThread = ThreadId.make("thread-sleeping"); + +const listing = { + defaultBranch: "main", + refs: [ + { name: "main", sha: "a1" }, + { name: "develop", sha: "b2" }, + { name: "feature/login", sha: "c3" }, + ], + truncated: false, +}; + +const identity = (url: string): RepositoryIdentity => ({ + canonicalKey: url, + locator: { source: "git-remote", remoteName: "origin", remoteUrl: url }, +}); + +const identities: Record = { + [projectVirtualRoot(repoProject)]: identity(REPO), + [projectVirtualRoot(lockedProject)]: identity("https://github.com/acme/locked"), + [projectVirtualRoot(gitlabProject)]: identity("https://gitlab.com/acme/widgets"), + [threadCheckoutPath(sleepingThread)]: identity(REPO), +}; + +const setup = Effect.gen(function* () { + const fake = yield* makeFakeMachineDirectory({ + initial: [[sleepingThread, { state: "paused" }]], + repositories: { + [REPO]: listing, + "https://github.com/acme/locked": { status: 403, code: "REPOSITORY_ACCESS_REQUIRED" }, + "https://gitlab.com/acme/widgets": { status: 422, code: "REPOSITORY_UNSUPPORTED" }, + }, + }); + const context = yield* Layer.build( + hubGitWorkflowServiceLayer.pipe( + Layer.provideMerge(hubVcsStatusCacheLayer), + Layer.provideMerge( + Layer.effect( + RunnerConnectionPool, + makePool({ wakePollInterval: "5 millis", idleCheckInterval: "1 hour" }), + ), + ), + Layer.provideMerge(Layer.succeed(MachineDirectory, fake.directory)), + Layer.provide( + Layer.succeed(RepositoryIdentityResolver, { + resolve: (cwd: string) => Effect.succeed(identities[cwd] ?? null), + }), + ), + Layer.provide( + Layer.succeed( + ProjectionSnapshotQuery, + ProjectionSnapshotQuery.of({ + getThreadShellById: (threadId: ThreadId) => + Effect.succeed( + threadId === sleepingThread + ? Option.some({ + id: threadId, + branch: "t3/unpushed", + } as unknown as OrchestrationThreadShell) + : Option.none(), + ), + } as unknown as ProjectionSnapshotQuery["Service"]), + ), + ), + ), + ); + const workflow = yield* GitWorkflowService.GitWorkflowService.pipe(Effect.provide(context)); + return { workflow, fake }; +}); + +const TestLayer = HubThreadMachineStateSqliteLive.pipe( + Layer.provideMerge(SqlitePersistenceMemory), + Layer.provideMerge(NodeServices.layer), +); + +describe("hub repository refs", () => { + it("maps a platform listing to listRefs results", () => { + const all = toListRefsResult(listing, { cwd: "/workspace/p/x" }, null); + expect(all.refs.map((ref) => [ref.name, ref.isDefault, ref.current])).toEqual([ + ["main", true, false], + ["develop", false, false], + ["feature/login", false, false], + ]); + expect(all).toMatchObject({ isRepo: true, hasPrimaryRemote: true, totalCount: 3 }); + + const current = toListRefsResult(listing, { cwd: "/workspace/t/x" }, "t3/unpushed"); + expect(current.refs.slice(0, 2).map((ref) => [ref.name, ref.current])).toEqual([ + ["t3/unpushed", true], + ["main", false], + ]); + + const page = toListRefsResult( + listing, + { cwd: "/workspace/p/x", query: "E", limit: 1, cursor: 1 }, + null, + ); + expect(page.refs.map((ref) => ref.name)).toEqual(["feature/login"]); + expect(page).toMatchObject({ totalCount: 2, nextCursor: null }); + + const remote = toListRefsResult(listing, { cwd: "/workspace/p/x", refKind: "remote" }, "x"); + expect(remote.refs[0]).toMatchObject({ + name: "origin/main", + isRemote: true, + remoteName: "origin", + isDefault: true, + }); + }); + + it.effect("lists project branches from the platform, cached, without any machine", () => + Effect.gen(function* () { + const { workflow, fake } = yield* setup; + const cwd = projectVirtualRoot(repoProject); + const first = yield* workflow.listRefs({ cwd }); + expect(first.refs.find((ref) => ref.isDefault)?.name).toBe("main"); + yield* workflow.listRefs({ cwd, query: "dev" }); + yield* workflow.listRefs({ cwd, refresh: true }); + expect(yield* fake.platformCalls).toEqual([ + `repositoryRefs ${REPO}`, + `repositoryRefs ${REPO}`, + ]); + expect(yield* fake.calls).toEqual([]); + }).pipe(Effect.scoped, Effect.provide(TestLayer)), + ); + + it.effect("answers blank, unsupported and unauthorized repositories", () => + Effect.gen(function* () { + const { workflow } = yield* setup; + expect(yield* workflow.listRefs({ cwd: projectVirtualRoot(blankProject) })).toEqual({ + refs: [], + isRepo: false, + hasPrimaryRemote: false, + nextCursor: null, + totalCount: 0, + }); + expect( + (yield* workflow.listRefs({ cwd: projectVirtualRoot(gitlabProject) })).totalCount, + ).toBe(0); + const locked = yield* workflow + .listRefs({ cwd: projectVirtualRoot(lockedProject) }) + .pipe(Effect.flip); + expect(locked._tag).toBe("GitCommandError"); + expect(locked.message).toContain("Aldo GitHub App"); + }).pipe(Effect.scoped, Effect.provide(TestLayer)), + ); + + it.effect("lists a sleeping thread's branches without waking its machine", () => + Effect.gen(function* () { + const { workflow, fake } = yield* setup; + const result = yield* workflow.listRefs({ cwd: threadCheckoutPath(sleepingThread) }); + expect(result.refs[0]).toMatchObject({ name: "t3/unpushed", current: true }); + expect(result.refs.some((ref) => ref.name === "main" && ref.isDefault)).toBe(true); + expect((yield* fake.calls).map((call) => call.method)).toEqual(["status"]); + }).pipe(Effect.scoped, Effect.provide(TestLayer)), + ); +}); diff --git a/apps/server/src/hub/HubRepositoryRefs.ts b/apps/server/src/hub/HubRepositoryRefs.ts new file mode 100644 index 000000000000..cce8400f1029 --- /dev/null +++ b/apps/server/src/hub/HubRepositoryRefs.ts @@ -0,0 +1,151 @@ +/** + * Branch lists without a machine. + * + * A hub project's root (`/workspace/p/`) has no checkout, so + * `vcs.listRefs` on it (drafts, the base-branch picker) is answered from the + * platform: `GET {T3CODE_HUB_MACHINES_URL}/repositories/refs?url=` + * for the project's recorded repository, mapped to the ordinary listRefs + * result (the default branch marked `isDefault`). A blank project has no + * repository and lists nothing. A sleeping thread's checkout gets the same + * listing with the thread's branch marked current, so listing never wakes a + * machine. + * + * Listings are cached per repository for `CACHE_TTL_MS` (the platform caches + * for 60 s too); `refresh: true` bypasses the cache. + * + * @module hub/HubRepositoryRefs + */ +import { + GitCommandError, + type VcsListRefsInput, + type VcsListRefsResult, + type VcsRef, +} from "@t3tools/contracts"; +import type { RepositoryRefsResponse } from "@t3tools/contracts/runner"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; + +import { RepositoryIdentityResolver } from "../project/RepositoryIdentityResolver.ts"; +import { MachineDirectory, type MachineDirectoryError } from "./MachineDirectory.ts"; + +const CACHE_TTL_MS = 30_000; +const DEFAULT_LIMIT = 100; +const REMOTE_NAME = "origin"; + +/** User-facing detail for the platform's repository-refs error codes. */ +const PLATFORM_ERROR_DETAILS: Record = { + REPOSITORY_ACCESS_REQUIRED: + "Connect this repository to the Aldo GitHub App to list its branches.", + REPOSITORY_NOT_FOUND: "The repository was not found, or the Aldo GitHub App cannot see it.", + NOT_HUB_HOSTED: "This workspace does not run threads on thread machines.", + REPOSITORY_REFS_UNAVAILABLE: "Branches are temporarily unavailable. Try again shortly.", +}; + +const emptyResult = (isRepo: boolean): VcsListRefsResult => ({ + refs: [], + isRepo, + hasPrimaryRemote: isRepo, + nextCursor: null, + totalCount: 0, +}); + +/** + * Maps a platform listing to a listRefs result: branch names as local refs + * (or `origin/` remote refs for `refKind: "remote"`), `current` first, + * then the default branch, filtered by `query` and paginated like a checkout. + */ +export const toListRefsResult = ( + response: RepositoryRefsResponse, + input: VcsListRefsInput, + current: string | null, +): VcsListRefsResult => { + const remote = input.refKind === "remote"; + const names = response.refs.map((ref) => ref.name); + if (!remote && current !== null && !names.includes(current)) names.unshift(current); + const refs: Array = names.map((name) => + remote + ? { + name: `${REMOTE_NAME}/${name}`, + isRemote: true, + remoteName: REMOTE_NAME, + current: false, + isDefault: name === response.defaultBranch, + worktreePath: null, + } + : { + name, + current: name === current, + isDefault: name === response.defaultBranch, + worktreePath: null, + }, + ); + const priority = (ref: VcsRef) => (ref.current ? 0 : ref.isDefault ? 1 : 2); + const query = input.query?.toLowerCase(); + const matching = refs + .toSorted((left, right) => priority(left) - priority(right)) + .filter((ref) => query === undefined || ref.name.toLowerCase().includes(query)); + const cursor = input.cursor ?? 0; + const page = matching.slice(cursor, cursor + (input.limit ?? DEFAULT_LIMIT)); + return { + refs: page, + isRepo: true, + hasPrimaryRemote: true, + nextCursor: cursor + page.length < matching.length ? cursor + page.length : null, + totalCount: matching.length, + }; +}; + +export const makeRepositoryRefsLister = Effect.gen(function* () { + const directory = yield* MachineDirectory; + const repositoryIdentity = yield* RepositoryIdentityResolver; + const cache = new Map< + string, + { readonly at: number; readonly response: RepositoryRefsResponse } + >(); + + const toGitCommandError = (cwd: string) => (error: MachineDirectoryError) => + new GitCommandError({ + operation: "listRefs", + command: "repositories/refs", + cwd, + detail: + (error.code !== undefined ? PLATFORM_ERROR_DETAILS[error.code] : undefined) ?? + `Branches could not be listed: ${error.message}`, + cause: error, + }); + + const fetchRefs = (url: string, refresh: boolean, cwd: string) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + const cached = cache.get(url); + if (!refresh && cached && now - cached.at < CACHE_TTL_MS) return cached.response; + const response = yield* directory + .repositoryRefs(url) + .pipe(Effect.mapError(toGitCommandError(cwd))); + cache.set(url, { at: now, response }); + return response; + }); + + /** + * Branches of the repository recorded for `input.cwd` (a project root, or a + * thread checkout through its project), with `current` marked. Never wakes a + * machine. Repositories the platform cannot list (not GitHub) list nothing. + */ + return (input: VcsListRefsInput, current: string | null) => + Effect.gen(function* () { + const identity = yield* repositoryIdentity.resolve(input.cwd); + if (identity === null) { + return current === null + ? emptyResult(false) + : toListRefsResult({ defaultBranch: null, refs: [] }, input, current); + } + return yield* fetchRefs(identity.locator.remoteUrl, input.refresh === true, input.cwd).pipe( + Effect.catchIf( + (error) => + (error.cause as MachineDirectoryError | undefined)?.code === "REPOSITORY_UNSUPPORTED", + () => Effect.succeed({ defaultBranch: null, refs: [] } as RepositoryRefsResponse), + ), + Effect.map((response) => toListRefsResult(response, input, current)), + ); + }); +}); diff --git a/apps/server/src/hub/HubVcs.ts b/apps/server/src/hub/HubVcs.ts index 77bb3a214968..2eafe30942f2 100644 --- a/apps/server/src/hub/HubVcs.ts +++ b/apps/server/src/hub/HubVcs.ts @@ -11,7 +11,9 @@ * PR thread preparation) run on the thread's runner and wake its machine. * Read-only git and review calls never wake a machine; on a sleeping machine * they fail with the service's error type caused by a `ThreadMachineUnavailableError` - * whose reason is `asleep`. + * whose reason is `asleep`. Branch lists are the exception: a project root and + * a sleeping thread's checkout list the repository's branches from the + * platform (`HubRepositoryRefs`). * * @module hub/HubVcs */ @@ -25,7 +27,7 @@ import { type VcsStatusResult, type VcsStatusStreamEvent, } from "@t3tools/contracts"; -import { ThreadMachineUnavailableError } from "@t3tools/contracts/runner"; +import { parseProjectVirtualRoot, ThreadMachineUnavailableError } from "@t3tools/contracts/runner"; import { mergeGitStatusParts } from "@t3tools/shared/git"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; @@ -38,6 +40,7 @@ import * as Stream from "effect/Stream"; import * as GitManager from "../git/GitManager.ts"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; +import { ProjectionSnapshotQuery } from "../orchestration/Services/ProjectionSnapshotQuery.ts"; import { ThreadVcsStatusStore } from "../persistence/Services/HubThreadMachineState.ts"; import * as ReviewService from "../review/ReviewService.ts"; import * as VcsProvisioningService from "../vcs/VcsProvisioningService.ts"; @@ -51,6 +54,7 @@ import { toGitManagerServiceError, toVcsError, } from "./hubRouting.ts"; +import { makeRepositoryRefsLister } from "./HubRepositoryRefs.ts"; import { RunnerConnectionPool } from "./RunnerConnectionPool.ts"; const isThreadMachineUnavailable: (error: unknown) => error is ThreadMachineUnavailableError = @@ -296,6 +300,13 @@ export const hubGitWorkflowServiceLayer = Layer.effect( Effect.gen(function* () { const pool = yield* RunnerConnectionPool; const cache = yield* HubVcsStatusCache; + const projections = yield* ProjectionSnapshotQuery; + const listRepositoryRefs = yield* makeRepositoryRefsLister; + const threadBranch = (threadId: ThreadId) => + projections.getThreadShellById(threadId).pipe( + Effect.map((thread) => (Option.isSome(thread) ? thread.value.branch : null)), + Effect.orElseSucceed(() => null), + ); const wake = (operation: string) => ({ wake: true, operation: `git.${operation}` }); const read = (operation: string) => ({ wake: false, operation: `git.${operation}` }); const cachedFallback = ( @@ -401,9 +412,22 @@ export const hubGitWorkflowServiceLayer = Layer.effect( client["runner.git.preparePullRequestThread"](input), ).pipe(Effect.mapError(toGitManagerServiceError("preparePullRequestThread", input.cwd))), listRefs: (input) => - onCwdRunner(pool, input.cwd, read("listRefs"), (client) => - client["runner.git.listRefs"](input), - ).pipe(Effect.mapError(toGitCommandError("listRefs", input.cwd))), + parseProjectVirtualRoot(input.cwd) !== null + ? listRepositoryRefs(input, null) + : onCwdRunner(pool, input.cwd, read("listRefs"), (client) => + client["runner.git.listRefs"](input), + ).pipe( + // A sleeping machine's checkout lists the repository's branches instead. + Effect.catchIf( + (error) => isThreadMachineUnavailable(error) && error.reason === "asleep", + () => + threadOfCwd(pool, input.cwd, "listRefs").pipe( + Effect.flatMap(threadBranch), + Effect.flatMap((branch) => listRepositoryRefs(input, branch)), + ), + ), + Effect.mapError(toGitCommandError("listRefs", input.cwd)), + ), createWorktree: (input) => onCwdRunner(pool, input.cwd, wake("createWorktree"), (client) => client["runner.git.createWorktree"](input), diff --git a/apps/server/src/hub/MachineDirectory.test.ts b/apps/server/src/hub/MachineDirectory.test.ts index 354b5eab93b4..3c8431e561bd 100644 --- a/apps/server/src/hub/MachineDirectory.test.ts +++ b/apps/server/src/hub/MachineDirectory.test.ts @@ -134,4 +134,52 @@ describe("HTTP machine directory", () => { }).pipe(Effect.provide(FetchHttpClient.layer)), ), ); + + it.live("lists repository refs and provider homes and signs a provider out", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* serveDirectory((request) => { + if (request.url.startsWith("/api/repositories/refs")) { + return request.url.includes("locked") + ? { status: 403, body: { error: "REPOSITORY_ACCESS_REQUIRED" } } + : { + status: 200, + body: { + defaultBranch: "main", + refs: [{ name: "main", sha: "abc" }], + truncated: false, + }, + }; + } + if (request.method === "DELETE") { + return { status: 200, body: { provider: "claude", deleted: true } }; + } + return { + status: 200, + body: { providers: [{ provider: "claude", version: 3, updatedAt: "t" }] }, + }; + }); + const directory = yield* makeHttpMachineDirectory({ baseUrl: server.url, token: "t" }); + + const refs = yield* directory.repositoryRefs("https://github.com/acme/app"); + expect(refs.defaultBranch).toBe("main"); + const locked = yield* directory + .repositoryRefs("https://github.com/acme/locked") + .pipe(Effect.flip); + expect(locked).toMatchObject({ status: 403, code: "REPOSITORY_ACCESS_REQUIRED" }); + expect((yield* directory.providerHomes).providers[0]?.version).toBe(3); + expect(yield* directory.deleteProviderHome("claude")).toEqual({ + provider: "claude", + deleted: true, + }); + + expect(server.requests.map((request) => [request.method, request.url])).toEqual([ + ["GET", "/api/repositories/refs?url=https%3A%2F%2Fgithub.com%2Facme%2Fapp"], + ["GET", "/api/repositories/refs?url=https%3A%2F%2Fgithub.com%2Facme%2Flocked"], + ["GET", "/api/provider-homes"], + ["DELETE", "/api/provider-homes/claude"], + ]); + }).pipe(Effect.provide(FetchHttpClient.layer)), + ), + ); }); diff --git a/apps/server/src/hub/MachineDirectory.ts b/apps/server/src/hub/MachineDirectory.ts index d892e42a90e9..9ea13e240d29 100644 --- a/apps/server/src/hub/MachineDirectory.ts +++ b/apps/server/src/hub/MachineDirectory.ts @@ -10,6 +10,12 @@ * POST /threads/{threadId}/machine/idle nothing on the hub needs the machine * POST /threads/{threadId}/machine/release the thread was archived or deleted * + * The same base URL serves the platform's machine-free lookups: + * + * GET /repositories/refs?url= branches (GitHub App repositories) + * GET /provider-homes stored provider sign-ins (metadata) + * DELETE /provider-homes/{provider} sign a provider out everywhere + * * Implementations: * - `layerHttp`: the documented HTTP contract (production). * - `makeStaticMachineDirectory`: one runner at `T3CODE_RUNNER_URL` that is @@ -20,6 +26,10 @@ */ import type { ThreadId } from "@t3tools/contracts"; import { + ProviderHomeDeleteResponse, + type ProviderHomeId, + ProviderHomesResponse, + RepositoryRefsResponse, type ThreadMachineEnsureRequest, type ThreadMachineState, ThreadMachineStatus, @@ -28,6 +38,7 @@ import * as Context from "effect/Context"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; @@ -38,17 +49,38 @@ export class MachineDirectoryError extends Schema.TaggedErrorClass { + const parsed = decodeErrorBody(body); + if (Option.isNone(parsed)) return undefined; + const code = parsed.value.error ?? parsed.value.code; + return typeof code === "string" && code.length > 0 ? code : undefined; +}; + const isMachineDirectoryError = Schema.is(MachineDirectoryError); export interface MachineDirectoryShape { @@ -65,6 +97,16 @@ export interface MachineDirectoryShape { readonly idle: (threadId: ThreadId) => Effect.Effect; /** Retires the machine; the thread was archived or deleted. */ readonly release: (threadId: ThreadId) => Effect.Effect; + /** A repository's branches, without any machine. */ + readonly repositoryRefs: ( + url: string, + ) => Effect.Effect; + /** Stored provider sign-ins (versions only). */ + readonly providerHomes: Effect.Effect; + /** Deletes a provider's stored sign-in; running machines sign out within seconds. */ + readonly deleteProviderHome: ( + provider: ProviderHomeId, + ) => Effect.Effect; } export class MachineDirectory extends Context.Service()( @@ -85,7 +127,7 @@ export const makeHttpMachineDirectory = (options: { const url = (threadId: ThreadId, suffix = "") => `${base}/threads/${encodeURIComponent(threadId)}/machine${suffix}`; - const toDirectoryError = (operation: string, threadId: ThreadId) => (cause: unknown) => + const toDirectoryError = (operation: string, threadId: ThreadId | "") => (cause: unknown) => isMachineDirectoryError(cause) ? cause : new MachineDirectoryError({ @@ -99,7 +141,7 @@ export const makeHttpMachineDirectory = (options: { const send = ( operation: string, - threadId: ThreadId, + threadId: ThreadId | "", request: Effect.Effect, decode: (response: HttpClientResponse.HttpClientResponse) => Effect.Effect, ): Effect.Effect => @@ -111,10 +153,12 @@ export const makeHttpMachineDirectory = (options: { ); if (response.status < 200 || response.status >= 300) { const body = yield* response.text.pipe(Effect.orElseSucceed(() => "")); + const code = errorCodeOf(body); return yield* new MachineDirectoryError({ operation, threadId, status: response.status, + ...(code !== undefined ? { code } : {}), detail: body.slice(0, 500) || "request failed", }); } @@ -164,6 +208,32 @@ export const makeHttpMachineDirectory = (options: { HttpClientRequest.post(url(threadId, "/release")).pipe(HttpClientRequest.bodyJson({})), () => Effect.void, ), + repositoryRefs: (repositoryUrl) => + send( + "repositoryRefs", + "", + Effect.succeed( + HttpClientRequest.get( + `${base}/repositories/refs?url=${encodeURIComponent(repositoryUrl)}`, + ), + ), + HttpClientResponse.schemaBodyJson(RepositoryRefsResponse), + ), + providerHomes: send( + "providerHomes", + "", + Effect.succeed(HttpClientRequest.get(`${base}/provider-homes`)), + HttpClientResponse.schemaBodyJson(ProviderHomesResponse), + ), + deleteProviderHome: (provider) => + send( + "deleteProviderHome", + "", + Effect.succeed( + HttpClientRequest.delete(`${base}/provider-homes/${encodeURIComponent(provider)}`), + ), + HttpClientResponse.schemaBodyJson(ProviderHomeDeleteResponse), + ), }); }); @@ -181,6 +251,16 @@ export const makeStaticMachineDirectory = (options: { status: () => Effect.succeed(staticStatus(options)), idle: () => Effect.void, release: () => Effect.void, + repositoryRefs: () => + Effect.fail( + new MachineDirectoryError({ + operation: "repositoryRefs", + threadId: "", + detail: "The development runner directory has no platform to list repository refs.", + }), + ), + providerHomes: Effect.succeed({ providers: [] }), + deleteProviderHome: (provider) => Effect.succeed({ provider, deleted: false }), }); const staticStatus = (options: { @@ -209,17 +289,29 @@ export interface FakeMachineDirectoryCall { readonly wake?: boolean; } +/** A fake repository listing, or the platform error it answers with. */ +export type FakeRepositoryRefs = + | RepositoryRefsResponse + | { readonly status: number; readonly code: string }; + /** * Scriptable directory for tests. `onWake` decides what an ensure-with-wake - * does to a machine (by default it becomes `running`). + * does to a machine (by default it becomes `running`). `repositories` answers + * repository ref listings by URL; provider homes start empty and change with + * `setProviderHome` and `deleteProviderHome`. */ export const makeFakeMachineDirectory = (options?: { readonly initial?: ReadonlyArray; readonly onWake?: (threadId: ThreadId, machine: FakeMachine | undefined) => FakeMachine; + readonly repositories?: Readonly>; }) => Effect.gen(function* () { const machines = yield* Ref.make(new Map(options?.initial ?? [])); const calls = yield* Ref.make>([]); + const platformCalls = yield* Ref.make>([]); + const providerHomes = yield* Ref.make(new Map()); + const recordPlatform = (call: string) => + Ref.update(platformCalls, (current) => [...current, call]); const record = (call: FakeMachineDirectoryCall) => Ref.update(calls, (current) => [...current, call]); const toStatus = (machine: FakeMachine | undefined): ThreadMachineStatus => @@ -262,10 +354,57 @@ export const makeFakeMachineDirectory = (options?: { }), ), ), + repositoryRefs: (url) => + Effect.gen(function* () { + yield* recordPlatform(`repositoryRefs ${url}`); + const answer = options?.repositories?.[url]; + if (answer === undefined) { + return yield* new MachineDirectoryError({ + operation: "repositoryRefs", + threadId: "", + status: 404, + code: "REPOSITORY_NOT_FOUND", + detail: `{"error":"REPOSITORY_NOT_FOUND"}`, + }); + } + if ("status" in answer) { + return yield* new MachineDirectoryError({ + operation: "repositoryRefs", + threadId: "", + status: answer.status, + code: answer.code, + detail: `{"error":"${answer.code}"}`, + }); + } + return answer; + }), + providerHomes: recordPlatform("providerHomes").pipe( + Effect.andThen(Ref.get(providerHomes)), + Effect.map((homes) => ({ + providers: [...homes].map(([provider, version]) => ({ + provider, + version, + updatedAt: null, + })), + })), + ), + deleteProviderHome: (provider) => + recordPlatform(`deleteProviderHome ${provider}`).pipe( + Effect.andThen( + Ref.modify(providerHomes, (homes) => { + const next = new Map(homes); + const deleted = next.delete(provider); + return [{ provider, deleted }, next] as const; + }), + ), + ), }); return { directory, calls: Ref.get(calls), + platformCalls: Ref.get(platformCalls), + setProviderHome: (provider: string, version: number) => + Ref.update(providerHomes, (homes) => new Map(homes).set(provider, version)), set: (threadId: ThreadId, machine: FakeMachine) => Ref.update(machines, (map) => new Map(map).set(threadId, machine)), get: (threadId: ThreadId) => Ref.get(machines).pipe(Effect.map((map) => map.get(threadId))), diff --git a/packages/contracts/src/runner.ts b/packages/contracts/src/runner.ts index 81e0d67cbf28..a34f42e19949 100644 --- a/packages/contracts/src/runner.ts +++ b/packages/contracts/src/runner.ts @@ -155,6 +155,19 @@ export function projectVirtualRoot(projectId: string): string { return `${PROJECT_VIRTUAL_ROOT}/${encodePathSegment(projectId)}`; } +/** The project whose virtual root is exactly `path`, or null. */ +export function parseProjectVirtualRoot(path: string): ProjectId | null { + const prefix = `${PROJECT_VIRTUAL_ROOT}/`; + if (!path.startsWith(prefix)) return null; + const segment = trimTrailingSlashes(path.slice(prefix.length)); + if (segment.length === 0 || segment.includes("/")) return null; + const projectId = decodePathSegment(segment); + if (projectId === null || projectId.trim().length === 0 || projectId !== projectId.trim()) { + return null; + } + return ProjectId.make(projectId); +} + /** * The thread whose checkout contains `path`, or null when the path is not * inside `/`. Paths inside the checkout (`//src`) @@ -216,6 +229,51 @@ export const ThreadMachineStatus = Schema.Struct({ }); export type ThreadMachineStatus = typeof ThreadMachineStatus.Type; +/** + * `GET {machines}/repositories/refs?url=` — a repository's branches without a + * machine (GitHub App repositories only; default branch first, at most 500). + */ +export const RepositoryRefsResponse = Schema.Struct({ + defaultBranch: Schema.NullOr(Schema.String), + refs: Schema.Array(Schema.Struct({ name: TrimmedNonEmptyString, sha: Schema.String })), + truncated: Schema.optional(Schema.Boolean), +}); +export type RepositoryRefsResponse = typeof RepositoryRefsResponse.Type; + +/** + * Provider ids of the platform's provider-home archives (the files a provider + * CLI keeps its sign-in in), restored onto every thread machine. + */ +export const ProviderHomeId = Schema.Literals([ + "claude", + "codex", + "opencode", + "grok", + "cursor", + "prime", + "muse", +]); +export type ProviderHomeId = typeof ProviderHomeId.Type; + +/** `GET {machines}/provider-homes` — stored sign-in metadata; never the files. */ +export const ProviderHomesResponse = Schema.Struct({ + providers: Schema.Array( + Schema.Struct({ + provider: Schema.String, + version: Schema.Number, + updatedAt: Schema.optional(Schema.NullOr(Schema.String)), + }), + ), +}); +export type ProviderHomesResponse = typeof ProviderHomesResponse.Type; + +/** `DELETE {machines}/provider-homes/{provider}` — sign out everywhere. */ +export const ProviderHomeDeleteResponse = Schema.Struct({ + provider: Schema.String, + deleted: Schema.Boolean, +}); +export type ProviderHomeDeleteResponse = typeof ProviderHomeDeleteResponse.Type; + export const ThreadMachineUnavailableReason = Schema.Literals([ /** The machine is paused, saved, or not created, and the call must not wake it. */ "asleep", From 8196fdfd9591b26f671e21faf8ae155ba81c5742 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 01:04:17 +0000 Subject: [PATCH 38/58] feat(server): keep provider snapshots on a hub before and between runner reports The remote provider driver now persists the last snapshot each runner reports per provider instance (hub migration 051, SQLite in development) and starts from it after a restart. Before any runner reported an instance it shows the driver's pending snapshot marked installed and ready with auth unknown (models from the driver or the bundled manifest), so the composer can select the provider and the first turn can start a machine. The hub's settings stay authoritative for identity and enablement, and update advisories are dropped because a hub cannot update CLIs on machines. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../HubRunnerLoopbackHarness.integration.ts | 3 +- apps/server/src/hub/HubLayers.ts | 5 +- .../src/hub/HubProviderSnapshots.test.ts | 164 +++++++++++++ apps/server/src/hub/HubProviderSnapshots.ts | 231 ++++++++++++++++++ .../src/hub/RemoteProviderDriver.test.ts | 10 +- apps/server/src/hub/RemoteProviderDriver.ts | 73 ++++-- .../Layers/HubThreadMachineState.ts | 50 +++- .../Postgres/HubThreadMachineState.ts | 45 +++- .../051_HubThreadMachineServices.ts | 14 +- .../Services/HubThreadMachineState.ts | 21 ++ 10 files changed, 582 insertions(+), 34 deletions(-) create mode 100644 apps/server/src/hub/HubProviderSnapshots.test.ts create mode 100644 apps/server/src/hub/HubProviderSnapshots.ts diff --git a/apps/server/integration/HubRunnerLoopbackHarness.integration.ts b/apps/server/integration/HubRunnerLoopbackHarness.integration.ts index 5c9d6e87e0bf..c55057324ddc 100644 --- a/apps/server/integration/HubRunnerLoopbackHarness.integration.ts +++ b/apps/server/integration/HubRunnerLoopbackHarness.integration.ts @@ -61,6 +61,7 @@ import * as RemoteSessionRegistry from "../src/hub/RemoteSessionRegistry.ts"; import { make as makePool, RunnerConnectionPool } from "../src/hub/RunnerConnectionPool.ts"; import * as RunnerEventDelivery from "../src/hub/RunnerEventDelivery.ts"; import * as ThreadMachineStates from "../src/hub/ThreadMachineStates.ts"; +import * as HubProviderSnapshots from "../src/hub/HubProviderSnapshots.ts"; import { serveRunner } from "../src/hub/testUtils/runnerServer.ts"; import { makeOrchestrationCommandDispatcher } from "../src/orchestration/CommandDispatcher.ts"; import { normalizeDispatchCommand } from "../src/orchestration/Normalizer.ts"; @@ -399,7 +400,7 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") ), ), Layer.provideMerge(ThreadMachineStates.readerLayer), - Layer.provideMerge(ThreadMachineStates.layer), + Layer.provideMerge(Layer.mergeAll(ThreadMachineStates.layer, HubProviderSnapshots.layer)), Layer.provideMerge(HubLayers.makeHubStateStoresLayer(persistence)), ); const remoteAdapterRegistry = Layer.effect( diff --git a/apps/server/src/hub/HubLayers.ts b/apps/server/src/hub/HubLayers.ts index 29489f3e63ba..cdab10d50d5c 100644 --- a/apps/server/src/hub/HubLayers.ts +++ b/apps/server/src/hub/HubLayers.ts @@ -44,6 +44,7 @@ import { hubWorkspaceFileSystemLayer, hubWorkspacePathsLayer, } from "./HubWorkspace.ts"; +import * as HubProviderSnapshots from "./HubProviderSnapshots.ts"; import * as MachineDirectory from "./MachineDirectory.ts"; import { makeRemoteProviderDriver } from "./RemoteProviderDriver.ts"; import * as RemoteSessionRegistry from "./RemoteSessionRegistry.ts"; @@ -68,7 +69,7 @@ export const makeHubStateStoresLayer = ( /** * Hub-only services underneath the whole runtime: stores, machine states, - * machine directory (observed by the machine states), connection pool, + * provider snapshots, machine directory (observed by the machine states), connection pool, * session registry, event delivery and git status cache. `persistence` is * the server's persistence layer (SQLite, or the hub's Postgres client in hub * mode with a database). @@ -87,7 +88,7 @@ export const makeHubInfrastructureLayer = (parts: { ThreadMachineStates.observedMachineDirectoryLayer.pipe(Layer.provide(MachineDirectory.layer)), ), Layer.provideMerge(ThreadMachineStates.readerLayer), - Layer.provideMerge(ThreadMachineStates.layer), + Layer.provideMerge(Layer.mergeAll(ThreadMachineStates.layer, HubProviderSnapshots.layer)), Layer.provideMerge(makeHubStateStoresLayer(parts.persistence)), ); diff --git a/apps/server/src/hub/HubProviderSnapshots.test.ts b/apps/server/src/hub/HubProviderSnapshots.test.ts new file mode 100644 index 000000000000..774f2ee88e8a --- /dev/null +++ b/apps/server/src/hub/HubProviderSnapshots.test.ts @@ -0,0 +1,164 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { it } from "@effect/vitest"; +import { + ProviderDriverKind, + ProviderInstanceId, + type ServerProvider, + ThreadId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import { describe, expect } from "vite-plus/test"; + +import { ServerConfig } from "../config.ts"; +import { HubThreadMachineStateSqliteLive } from "../persistence/Layers/HubThreadMachineState.ts"; +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; +import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; +import { ClaudeDriver } from "../provider/Drivers/ClaudeDriver.ts"; +import { CodexDriver } from "../provider/Drivers/CodexDriver.ts"; +import * as HubProviderSnapshots from "./HubProviderSnapshots.ts"; +import { MachineDirectory, makeFakeMachineDirectory } from "./MachineDirectory.ts"; +import { makeRemoteProviderDriver } from "./RemoteProviderDriver.ts"; +import * as RemoteSessionRegistry from "./RemoteSessionRegistry.ts"; +import { make as makePool, RunnerConnectionPool } from "./RunnerConnectionPool.ts"; +import * as RunnerEventDelivery from "./RunnerEventDelivery.ts"; +import { fakeRunner, fakeRunnerHello, serveRunner } from "./testUtils/runnerServer.ts"; + +const claude = ProviderDriverKind.make("claudeAgent"); +const codex = ProviderDriverKind.make("codex"); +const claudeInstance = ProviderInstanceId.make("claudeAgent"); +const threadId = ThreadId.make("thread-provider-snapshots"); + +const identity = (driverKind: ProviderDriverKind, enabled = true) => ({ + driverKind, + instanceId: ProviderInstanceId.make(driverKind), + displayName: undefined, + accentColor: undefined, + enabled, + continuationGroupKey: `${driverKind}:group`, +}); + +const StoresLive = Layer.mergeAll( + HubThreadMachineStateSqliteLive, + ProviderSessionRuntime.layer, +).pipe(Layer.provideMerge(SqlitePersistenceMemory), Layer.provideMerge(NodeServices.layer)); + +const reported = (auth: ServerProvider["auth"]["status"]): ServerProvider => ({ + instanceId: claudeInstance, + driver: claude, + displayName: "Claude", + enabled: true, + installed: true, + version: "2.1.220", + status: "ready", + auth: { status: auth }, + checkedAt: "2026-09-26T00:00:00.000Z", + models: [{ slug: "claude-sonnet-5", name: "Sonnet 5", isCustom: false, capabilities: null }], + slashCommands: [], + skills: [], + versionAdvisory: { + status: "behind_latest", + currentVersion: "2.1.220", + latestVersion: "2.2.0", + updateCommand: null, + canUpdate: true, + checkedAt: "2026-09-26T00:00:00.000Z", + message: null, + }, +}); + +describe("hub provider snapshots", () => { + it.effect("stands in with selectable pending snapshots before any runner reports", () => + Effect.gen(function* () { + const pendingClaude = yield* HubProviderSnapshots.pendingRemoteSnapshot( + identity(claude), + ClaudeDriver.defaultConfig(), + ); + expect(pendingClaude).toMatchObject({ + instanceId: "claudeAgent", + driver: "claudeAgent", + enabled: true, + installed: true, + status: "ready", + auth: { status: "unknown" }, + availability: "available", + }); + expect(pendingClaude.models.length).toBeGreaterThan(0); + + const pendingCodex = yield* HubProviderSnapshots.pendingRemoteSnapshot( + identity(codex), + CodexDriver.defaultConfig(), + ); + expect(pendingCodex.status).toBe("ready"); + expect(pendingCodex.models.some((model) => !model.isCustom && model.isDefault)).toBe(true); + + const disabled = yield* HubProviderSnapshots.pendingRemoteSnapshot( + identity(codex, false), + CodexDriver.defaultConfig(), + ); + expect(disabled).toMatchObject({ enabled: false, status: "disabled" }); + }), + ); + + it.live("persists what a runner reports and serves it after a restart", () => + Effect.gen(function* () { + const runner = yield* serveRunner( + fakeRunner({ + "runner.hello": (input) => + Effect.succeed( + fakeRunnerHello({ threadId: input.threadId, instances: [claudeInstance] }), + ), + "runner.provider.getCapabilities": () => + Effect.succeed({ + snapshot: reported("authenticated"), + sessionModelSwitch: "in-session", + }), + }), + ); + const fake = yield* makeFakeMachineDirectory({ + onWake: () => ({ state: "running", runnerUrl: runner.url }), + }); + const snapshots = yield* HubProviderSnapshots.make; + const pool = yield* makePool({ + wakePollInterval: "5 millis", + idleCheckInterval: "1 hour", + }).pipe(Effect.provideService(MachineDirectory, fake.directory)); + const registry = yield* RemoteSessionRegistry.make; + const delivery = yield* RunnerEventDelivery.make.pipe( + Effect.provideService(RunnerConnectionPool, pool), + Effect.provideService(RemoteSessionRegistry.RemoteSessionRegistry, registry), + ); + const instance = yield* makeRemoteProviderDriver(ClaudeDriver) + .create({ + instanceId: claudeInstance, + displayName: "Claude (work)", + environment: [], + enabled: true, + config: ClaudeDriver.defaultConfig(), + }) + .pipe( + Effect.provideService(RunnerConnectionPool, pool), + Effect.provideService(RunnerEventDelivery.RunnerEventDelivery, delivery), + Effect.provideService(RemoteSessionRegistry.RemoteSessionRegistry, registry), + Effect.provideService(HubProviderSnapshots.HubProviderSnapshots, snapshots), + Effect.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-snapshots-" })), + ); + expect((yield* instance.snapshot.getSnapshot).auth.status).toBe("unknown"); + + // Connecting to a runner reports the instance. + yield* pool.use(threadId, { wake: true, operation: "test" }, () => Effect.void); + yield* Effect.repeat(instance.snapshot.getSnapshot, { + until: (snapshot) => snapshot.auth.status === "authenticated", + }).pipe(Effect.timeout("5 seconds")); + const live = yield* instance.snapshot.getSnapshot; + expect(live).toMatchObject({ displayName: "Claude (work)", version: "2.1.220" }); + expect(live.versionAdvisory).toBeUndefined(); + + // A restarted hub starts from the persisted report. + const restarted = yield* HubProviderSnapshots.make; + const persisted = yield* restarted.get(claudeInstance); + expect(Option.getOrThrow(persisted).auth.status).toBe("authenticated"); + }).pipe(Effect.scoped, Effect.provide(StoresLive)), + ); +}); diff --git a/apps/server/src/hub/HubProviderSnapshots.ts b/apps/server/src/hub/HubProviderSnapshots.ts new file mode 100644 index 000000000000..3895c4012b79 --- /dev/null +++ b/apps/server/src/hub/HubProviderSnapshots.ts @@ -0,0 +1,231 @@ +/** + * HubProviderSnapshots - provider status the hub can show without a machine. + * + * A hub cannot probe provider CLIs; every runner reports the snapshots of the + * instances it hosts when the hub connects (and on refresh). The latest report + * per provider instance is kept here and persisted (`ProviderSnapshotStore`, + * hub Postgres or SQLite in development), so a restarted hub, or one whose + * machines are all asleep, still shows real status, auth and models. + * + * Before any runner has reported an instance, `pendingRemoteSnapshot` stands + * in: the driver's own initial snapshot (built-in and custom models), marked + * installed and ready with auth `unknown`, so the composer can select the + * provider and the first turn can start its machine. + * + * The hub's settings stay authoritative for identity and enablement + * (`overlayHubIdentity`), and provider maintenance fields are dropped: CLIs on + * thread machines come from the machine image, not from the hub. + * + * @module hub/HubProviderSnapshots + */ +import { + type ClaudeSettings, + type CodexSettings, + type CursorSettings, + type GrokSettings, + type MuseSettings, + type OpenCodeSettings, + type PrimeSettings, + type ProviderDriverKind, + type ProviderInstanceId, + type ServerProvider, + type ServerProviderModel, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; +import * as Stream from "effect/Stream"; + +import { + ProviderSnapshotStore, + type ProviderSnapshotRow, +} from "../persistence/Services/HubThreadMachineState.ts"; +import { makePendingClaudeProvider } from "../provider/Layers/ClaudeProvider.ts"; +import { makePendingCodexProvider } from "../provider/Layers/CodexProvider.ts"; +import { buildInitialCursorProviderSnapshot } from "../provider/Layers/CursorProvider.ts"; +import { buildInitialGrokProviderSnapshot } from "../provider/Layers/GrokProvider.ts"; +import { buildInitialMuseProviderSnapshot } from "../provider/Layers/MuseProvider.ts"; +import { makePendingOpenCodeProvider } from "../provider/Layers/OpenCodeProvider.ts"; +import { buildInitialPrimeProviderSnapshot } from "../provider/Layers/PrimeProvider.ts"; +import { BUNDLED_MODEL_MANIFEST } from "../provider/ModelManifest.ts"; +import type { ServerProviderDraft } from "../provider/providerSnapshot.ts"; +import { buildUnavailableProviderSnapshot } from "../provider/unavailableProviderSnapshot.ts"; + +export interface HubProviderSnapshotsShape { + /** The latest snapshot a runner reported for the instance, if any. */ + readonly get: (instanceId: ProviderInstanceId) => Effect.Effect>; + /** Records (and persists) a runner-reported snapshot and publishes it. */ + readonly put: (snapshot: ServerProvider) => Effect.Effect; + readonly changes: Stream.Stream; +} + +export class HubProviderSnapshots extends Context.Service< + HubProviderSnapshots, + HubProviderSnapshotsShape +>()("t3/hub/HubProviderSnapshots") {} + +export const make = Effect.gen(function* () { + const store = yield* ProviderSnapshotStore; + const snapshots = new Map(); + for (const row of yield* store + .list() + .pipe( + Effect.catch((error) => + Effect.logWarning("provider snapshots could not be loaded", { detail: error.message }).pipe( + Effect.as([] as ReadonlyArray), + ), + ), + )) { + snapshots.set(row.instanceId, row.snapshot); + } + const changes = yield* PubSub.unbounded(); + + return HubProviderSnapshots.of({ + get: (instanceId) => Effect.sync(() => Option.fromNullishOr(snapshots.get(instanceId))), + put: (snapshot) => + Effect.gen(function* () { + snapshots.set(snapshot.instanceId, snapshot); + yield* store + .put({ + instanceId: snapshot.instanceId, + snapshot, + updatedAt: DateTime.formatIso(yield* DateTime.now), + }) + .pipe( + Effect.catch((error) => + Effect.logWarning("provider snapshot was not persisted", { + instanceId: snapshot.instanceId, + detail: error.message, + }), + ), + ); + yield* PubSub.publish(changes, snapshot); + }), + changes: Stream.fromPubSub(changes), + }); +}); + +export const layer = Layer.effect(HubProviderSnapshots, make); + +const PENDING_MESSAGE = "Checked when a thread machine starts."; + +/** Manifest models for drivers whose initial snapshot lists none (models come from a probe). */ +const withManifestModels = ( + draft: ServerProviderDraft, + driverKind: ProviderDriverKind, +): ServerProviderDraft => { + if (draft.models.some((model) => !model.isCustom)) return draft; + const current = BUNDLED_MODEL_MANIFEST.currentModels[driverKind] ?? []; + const known = new Set(draft.models.map((model) => model.slug)); + const models: Array = current + .filter((slug) => !known.has(slug)) + .map((slug, index) => ({ + slug, + name: slug, + isCustom: false, + ...(index === 0 ? { isDefault: true } : {}), + capabilities: null, + })); + return { ...draft, models: [...models, ...draft.models] }; +}; + +const pendingDraft = ( + driverKind: ProviderDriverKind, + config: unknown, + enabled: boolean, +): Effect.Effect => { + const settings = { ...(config as object), enabled }; + switch (driverKind) { + case "claudeAgent": + return makePendingClaudeProvider(settings as ClaudeSettings); + case "codex": + return makePendingCodexProvider(settings as CodexSettings); + case "opencode": + return makePendingOpenCodeProvider(settings as OpenCodeSettings); + case "cursor": + return buildInitialCursorProviderSnapshot(settings as CursorSettings); + case "grok": + return buildInitialGrokProviderSnapshot(settings as GrokSettings); + case "primeAgent": + return buildInitialPrimeProviderSnapshot(settings as PrimeSettings); + case "muse": + return buildInitialMuseProviderSnapshot(settings as MuseSettings); + default: + return Effect.succeed(null); + } +}; + +export interface RemoteInstanceIdentity { + readonly driverKind: ProviderDriverKind; + readonly instanceId: ProviderInstanceId; + readonly displayName: string | undefined; + readonly accentColor: string | undefined; + readonly enabled: boolean; + readonly continuationGroupKey: string; +} + +/** + * Applies the hub's identity and enablement to a snapshot, and drops update + * advisories: a hub cannot update the CLIs on thread machines. + */ +export const overlayHubIdentity = ( + snapshot: ServerProvider, + identity: RemoteInstanceIdentity, +): ServerProvider => { + const { versionAdvisory: _versionAdvisory, updateState: _updateState, ...rest } = snapshot; + return { + ...rest, + instanceId: identity.instanceId, + driver: identity.driverKind, + ...(identity.displayName ? { displayName: identity.displayName } : {}), + ...(identity.accentColor ? { accentColor: identity.accentColor } : {}), + continuation: snapshot.continuation ?? { groupKey: identity.continuationGroupKey }, + enabled: identity.enabled, + status: identity.enabled + ? snapshot.status === "disabled" + ? "warning" + : snapshot.status + : "disabled", + }; +}; + +/** + * The snapshot shown before any runner reported the instance: the driver's + * initial snapshot, installed and ready with auth `unknown`. + */ +export const pendingRemoteSnapshot = ( + identity: RemoteInstanceIdentity, + config: unknown, +): Effect.Effect => + Effect.gen(function* () { + const draft = yield* pendingDraft(identity.driverKind, config, identity.enabled); + if (draft === null) { + return yield* buildUnavailableProviderSnapshot({ + driverKind: identity.driverKind, + instanceId: identity.instanceId, + displayName: identity.displayName, + accentColor: identity.accentColor, + reason: "This build has no driver for this provider.", + }); + } + const { versionAdvisory: _versionAdvisory, ...pending } = withManifestModels( + draft, + identity.driverKind, + ); + return overlayHubIdentity( + { + ...pending, + instanceId: identity.instanceId, + driver: identity.driverKind, + installed: true, + status: "ready", + auth: { status: "unknown" }, + message: PENDING_MESSAGE, + availability: "available", + }, + identity, + ); + }); diff --git a/apps/server/src/hub/RemoteProviderDriver.test.ts b/apps/server/src/hub/RemoteProviderDriver.test.ts index dadcdeefc0f1..55e6be6e1999 100644 --- a/apps/server/src/hub/RemoteProviderDriver.test.ts +++ b/apps/server/src/hub/RemoteProviderDriver.test.ts @@ -30,6 +30,7 @@ import { } from "../persistence/Postgres/hubTestDatabase.ts"; import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; import { CodexDriver } from "../provider/Drivers/CodexDriver.ts"; +import * as HubProviderSnapshots from "./HubProviderSnapshots.ts"; import { MachineDirectory, makeFakeMachineDirectory } from "./MachineDirectory.ts"; import { attachmentFilesForRunner, @@ -126,10 +127,11 @@ describe("remote provider driver helpers", () => { ); }); -const StoresLive = Layer.mergeAll( - HubThreadMachineStateSqliteLive, - ProviderSessionRuntime.layer, -).pipe(Layer.provideMerge(SqlitePersistenceMemory), Layer.provideMerge(NodeServices.layer)); +const StoresLive = HubProviderSnapshots.layer.pipe( + Layer.provideMerge(Layer.mergeAll(HubThreadMachineStateSqliteLive, ProviderSessionRuntime.layer)), + Layer.provideMerge(SqlitePersistenceMemory), + Layer.provideMerge(NodeServices.layer), +); describe("remote provider adapter session answers", () => { it.live("answers hasSession and listSessions from the hub and never wakes a machine", () => diff --git a/apps/server/src/hub/RemoteProviderDriver.ts b/apps/server/src/hub/RemoteProviderDriver.ts index 5228f62bc7ae..da0d30a3ecc6 100644 --- a/apps/server/src/hub/RemoteProviderDriver.ts +++ b/apps/server/src/hub/RemoteProviderDriver.ts @@ -74,9 +74,14 @@ import type { ProviderThreadSnapshot, } from "../provider/Services/ProviderAdapter.ts"; import type { ServerProviderShape } from "../provider/Services/ServerProvider.ts"; -import { buildUnavailableProviderSnapshot } from "../provider/unavailableProviderSnapshot.ts"; import type { TextGenerationShape } from "../textGeneration/TextGeneration.ts"; import { fromRunnerRemoteError, ProviderAdapterErrorSchema } from "../runner/remoteErrors.ts"; +import { + HubProviderSnapshots, + overlayHubIdentity, + pendingRemoteSnapshot, + type RemoteInstanceIdentity, +} from "./HubProviderSnapshots.ts"; import { RemoteSessionRegistry } from "./RemoteSessionRegistry.ts"; import { RunnerConnectionPool, type RunnerConnection } from "./RunnerConnectionPool.ts"; import { RunnerEventDelivery } from "./RunnerEventDelivery.ts"; @@ -85,6 +90,7 @@ export type RemoteProviderDriverEnv = | RunnerConnectionPool | RunnerEventDelivery | RemoteSessionRegistry + | HubProviderSnapshots | ServerConfig | FileSystem.FileSystem | Path.Path; @@ -166,7 +172,7 @@ export function makeRemoteProviderDriver( metadata: base.metadata, configSchema: base.configSchema, defaultConfig: base.defaultConfig, - create: ({ instanceId, displayName, accentColor, enabled }) => + create: ({ instanceId, displayName, accentColor, enabled, config: driverConfig }) => Effect.gen(function* () { const config = yield* ServerConfig; const platform = yield* Effect.context(); @@ -185,10 +191,21 @@ export function makeRemoteProviderDriver( const capabilities: { current: Omit; } = { current: { sessionModelSwitch: "in-session" } }; + const continuationIdentity = defaultProviderContinuationIdentity({ + driverKind, + instanceId, + }); const snapshot = yield* makeRemoteSnapshot({ context, - displayName, - accentColor, + identity: { + driverKind, + instanceId, + displayName, + accentColor, + enabled, + continuationGroupKey: continuationIdentity.continuationKey, + }, + config: driverConfig, onCapabilities: (sessionModelSwitch) => Effect.sync(() => { capabilities.current = { ...capabilities.current, sessionModelSwitch }; @@ -197,7 +214,7 @@ export function makeRemoteProviderDriver( return { instanceId, driverKind, - continuationIdentity: defaultProviderContinuationIdentity({ driverKind, instanceId }), + continuationIdentity, displayName, accentColor, enabled, @@ -408,33 +425,42 @@ function makeRemoteAdapter( } /** - * Provider status and models as the most recent runner reported them. A hub - * cannot probe provider CLIs; until a thread machine reports this instance - * the snapshot says so. + * Provider status and models as the most recent runner reported them, + * persisted by `HubProviderSnapshots` so they survive hub restarts. Before + * any runner reported this instance the snapshot is the driver's pending + * one, installed and ready with auth `unknown` (see `pendingRemoteSnapshot`). + * Every runner connection reports its hosted instances; `refresh` asks a + * connected runner to probe again. */ const makeRemoteSnapshot = (input: { readonly context: InstanceContext; - readonly displayName: string | undefined; - readonly accentColor: string | undefined; + readonly identity: RemoteInstanceIdentity; + readonly config: unknown; readonly onCapabilities: ( sessionModelSwitch: ProviderAdapterCapabilities["sessionModelSwitch"], ) => Effect.Effect; }) => Effect.gen(function* () { - const { context } = input; - const unavailable = (reason: string) => - buildUnavailableProviderSnapshot({ - driverKind: context.driverKind, - instanceId: context.instanceId, - displayName: input.displayName, - accentColor: input.accentColor, - reason, - }); + const { context, identity } = input; + const snapshots = yield* HubProviderSnapshots; + const persisted = yield* snapshots.get(context.instanceId); const current = yield* Ref.make( - yield* unavailable("No thread machine has reported this provider yet."), + Option.isSome(persisted) + ? overlayHubIdentity(persisted.value, identity) + : yield* pendingRemoteSnapshot(identity, input.config), ); const changes = yield* PubSub.unbounded(); + // Reports from any runner (or the sign-in flow) arrive through the store. + yield* snapshots.changes.pipe( + Stream.filter((snapshot) => snapshot.instanceId === context.instanceId), + Stream.runForEach((snapshot) => { + const next = overlayHubIdentity(snapshot, identity); + return Ref.set(current, next).pipe(Effect.andThen(PubSub.publish(changes, next))); + }), + Effect.forkScoped, + ); + const fetchFrom = (connection: RunnerConnection, refresh: boolean) => connection.hello.instances.includes(context.instanceId) ? connection.client["runner.provider.getCapabilities"]({ @@ -442,10 +468,9 @@ const makeRemoteSnapshot = (input: { ...(refresh ? { refresh } : {}), }).pipe( Effect.tap((result) => input.onCapabilities(result.sessionModelSwitch)), - Effect.map((result) => result.snapshot), - Effect.tap((snapshot) => Ref.set(current, snapshot)), - Effect.tap((snapshot) => PubSub.publish(changes, snapshot)), - Effect.asVoid, + Effect.flatMap((result) => + snapshots.put({ ...result.snapshot, instanceId: context.instanceId }), + ), Effect.catchCause((cause) => Effect.logWarning("runner did not report provider capabilities", { instanceId: context.instanceId, diff --git a/apps/server/src/persistence/Layers/HubThreadMachineState.ts b/apps/server/src/persistence/Layers/HubThreadMachineState.ts index 82502f409555..4540bace6be4 100644 --- a/apps/server/src/persistence/Layers/HubThreadMachineState.ts +++ b/apps/server/src/persistence/Layers/HubThreadMachineState.ts @@ -8,7 +8,12 @@ * * @module HubThreadMachineStateSqlite */ -import { VcsStatusLocalResult, VcsStatusRemoteResult, type ThreadId } from "@t3tools/contracts"; +import { + ServerProvider, + VcsStatusLocalResult, + VcsStatusRemoteResult, + type ThreadId, +} from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -21,6 +26,8 @@ import { CheckpointTurnDiff, CheckpointTurnDiffKey, CheckpointTurnDiffStore, + ProviderSnapshotRow, + ProviderSnapshotStore, RunnerCursor, RunnerCursorStore, ThreadMachineStatusRow, @@ -59,6 +66,11 @@ const HUB_SQLITE_TABLES = [ boot_id TEXT, updated_at TEXT NOT NULL )`, + `CREATE TABLE IF NOT EXISTS hub_provider_snapshots ( + instance_id TEXT PRIMARY KEY, + snapshot_json TEXT NOT NULL, + updated_at TEXT NOT NULL + )`, ] as const; const ensureTables = Effect.gen(function* () { @@ -261,10 +273,46 @@ export const makeThreadMachineStatusStore = Effect.gen(function* () { }); }); +const ProviderSnapshotDbRow = ProviderSnapshotRow.mapFields((fields) => ({ + ...fields, + snapshot: Schema.fromJsonString(ServerProvider), +})); + +export const makeProviderSnapshotStore = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* ensureTables; + + const selectAll = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProviderSnapshotDbRow, + execute: () => sql` + SELECT instance_id AS "instanceId", snapshot_json AS snapshot, updated_at AS "updatedAt" + FROM hub_provider_snapshots + `, + }); + const upsert = SqlSchema.void({ + Request: ProviderSnapshotDbRow, + execute: (row) => sql` + INSERT INTO hub_provider_snapshots (instance_id, snapshot_json, updated_at) + VALUES (${row.instanceId}, ${row.snapshot}, ${row.updatedAt}) + ON CONFLICT (instance_id) DO UPDATE SET + snapshot_json = excluded.snapshot_json, + updated_at = excluded.updated_at + `, + }); + + return ProviderSnapshotStore.of({ + list: () => + selectAll(undefined).pipe(Effect.mapError(sqlOrDecode("ProviderSnapshotStore.list"))), + put: (row) => upsert(row).pipe(Effect.mapError(sqlOrDecode("ProviderSnapshotStore.put"))), + }); +}); + /** Every repository over the configured SQLite client. */ export const HubThreadMachineStateSqliteLive = Layer.mergeAll( Layer.effect(RunnerCursorStore, makeRunnerCursorStore), Layer.effect(CheckpointTurnDiffStore, makeCheckpointTurnDiffStore), Layer.effect(ThreadVcsStatusStore, makeThreadVcsStatusStore), Layer.effect(ThreadMachineStatusStore, makeThreadMachineStatusStore), + Layer.effect(ProviderSnapshotStore, makeProviderSnapshotStore), ); diff --git a/apps/server/src/persistence/Postgres/HubThreadMachineState.ts b/apps/server/src/persistence/Postgres/HubThreadMachineState.ts index 2a321ed13f1e..9d0ecf66ddd2 100644 --- a/apps/server/src/persistence/Postgres/HubThreadMachineState.ts +++ b/apps/server/src/persistence/Postgres/HubThreadMachineState.ts @@ -5,7 +5,12 @@ * * @module HubThreadMachineStatePostgres */ -import { VcsStatusLocalResult, VcsStatusRemoteResult, type ThreadId } from "@t3tools/contracts"; +import { + ServerProvider, + VcsStatusLocalResult, + VcsStatusRemoteResult, + type ThreadId, +} from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -18,6 +23,8 @@ import { CheckpointTurnDiff, CheckpointTurnDiffKey, CheckpointTurnDiffStore, + ProviderSnapshotRow, + ProviderSnapshotStore, RunnerCursor, RunnerCursorStore, ThreadMachineStatusRow, @@ -232,10 +239,46 @@ export const makeThreadMachineStatusStore = Effect.gen(function* () { }); }); +const ProviderSnapshotDbRow = ProviderSnapshotRow.mapFields((fields) => ({ + ...fields, + snapshot: Schema.fromJsonString(ServerProvider), +})); + +export const makeProviderSnapshotStore = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const selectAll = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProviderSnapshotDbRow, + execute: () => sql` + SELECT instance_id AS "instanceId", snapshot_json AS snapshot, updated_at AS "updatedAt" + FROM hub_provider_snapshots WHERE user_id = ${userId} + `, + }); + const upsert = SqlSchema.void({ + Request: ProviderSnapshotDbRow, + execute: (row) => sql` + INSERT INTO hub_provider_snapshots (user_id, instance_id, snapshot_json, updated_at) + VALUES (${userId}, ${row.instanceId}, ${row.snapshot}, ${row.updatedAt}) + ON CONFLICT (user_id, instance_id) DO UPDATE SET + snapshot_json = excluded.snapshot_json, + updated_at = excluded.updated_at + `, + }); + + return ProviderSnapshotStore.of({ + list: () => + selectAll(undefined).pipe(Effect.mapError(sqlOrDecode("ProviderSnapshotStore.list"))), + put: (row) => upsert(row).pipe(Effect.mapError(sqlOrDecode("ProviderSnapshotStore.put"))), + }); +}); + /** Every repository; requires the hub Postgres `SqlClient` and `HubTenant`. */ export const HubThreadMachineStatePostgresLive = Layer.mergeAll( Layer.effect(RunnerCursorStore, makeRunnerCursorStore), Layer.effect(CheckpointTurnDiffStore, makeCheckpointTurnDiffStore), Layer.effect(ThreadVcsStatusStore, makeThreadVcsStatusStore), Layer.effect(ThreadMachineStatusStore, makeThreadMachineStatusStore), + Layer.effect(ProviderSnapshotStore, makeProviderSnapshotStore), ); diff --git a/apps/server/src/persistence/Postgres/migrations/051_HubThreadMachineServices.ts b/apps/server/src/persistence/Postgres/migrations/051_HubThreadMachineServices.ts index 379f691c046a..18f15f8a6968 100644 --- a/apps/server/src/persistence/Postgres/migrations/051_HubThreadMachineServices.ts +++ b/apps/server/src/persistence/Postgres/migrations/051_HubThreadMachineServices.ts @@ -3,6 +3,8 @@ * * - `hub_thread_machine_status`: the last machine state the directory reported * per thread, so thread shells show it without asking the directory again. + * - `hub_provider_snapshots`: the last provider snapshot (status, auth, models) + * a runner reported per provider instance. * * Every table leads its key with `user_id` and carries the forced tenant policy * from the baseline (`hubTenantPolicyStatements`). The statements are @@ -17,7 +19,10 @@ import { hubTenantPolicyStatements } from "./001_HubBaseline.ts"; const c = `COLLATE "C"`; -export const HUB_THREAD_MACHINE_SERVICE_TABLES = ["hub_thread_machine_status"] as const; +export const HUB_THREAD_MACHINE_SERVICE_TABLES = [ + "hub_thread_machine_status", + "hub_provider_snapshots", +] as const; const statements: ReadonlyArray = [ `CREATE TABLE IF NOT EXISTS hub_thread_machine_status ( @@ -29,6 +34,13 @@ const statements: ReadonlyArray = [ updated_at text ${c} NOT NULL, PRIMARY KEY (user_id, thread_id) )`, + `CREATE TABLE IF NOT EXISTS hub_provider_snapshots ( + user_id text ${c} NOT NULL, + instance_id text ${c} NOT NULL, + snapshot_json text NOT NULL, + updated_at text ${c} NOT NULL, + PRIMARY KEY (user_id, instance_id) + )`, ...HUB_THREAD_MACHINE_SERVICE_TABLES.flatMap(hubTenantPolicyStatements), ]; diff --git a/apps/server/src/persistence/Services/HubThreadMachineState.ts b/apps/server/src/persistence/Services/HubThreadMachineState.ts index 55d7d8c90ad8..0afac55e3d9f 100644 --- a/apps/server/src/persistence/Services/HubThreadMachineState.ts +++ b/apps/server/src/persistence/Services/HubThreadMachineState.ts @@ -12,6 +12,8 @@ * - `ThreadVcsStatusStore`: the last git status a runner reported per thread. * - `ThreadMachineStatusStore`: the last machine state the directory reported * per thread, shown on thread shells without asking the directory again. + * - `ProviderSnapshotStore`: the last provider snapshot a runner reported per + * provider instance, so a hub shows providers before any machine runs. * * Postgres implementations live in `persistence/Postgres/HubThreadMachineState.ts` * (hub migrations 050 and 051); the SQLite implementations back hub mode without a @@ -23,6 +25,8 @@ import { IsoDateTime, NonNegativeInt, + ProviderInstanceId, + ServerProvider, ThreadId, ThreadMachineState, TrimmedNonEmptyString, @@ -140,3 +144,20 @@ export class ThreadMachineStatusStore extends Context.Service< ThreadMachineStatusStore, ThreadMachineStatusStoreShape >()("t3/persistence/Services/HubThreadMachineState/ThreadMachineStatusStore") {} + +export const ProviderSnapshotRow = Schema.Struct({ + instanceId: ProviderInstanceId, + snapshot: ServerProvider, + updatedAt: IsoDateTime, +}); +export type ProviderSnapshotRow = typeof ProviderSnapshotRow.Type; + +export interface ProviderSnapshotStoreShape { + readonly list: () => Effect.Effect, ProjectionRepositoryError>; + readonly put: (row: ProviderSnapshotRow) => Effect.Effect; +} + +export class ProviderSnapshotStore extends Context.Service< + ProviderSnapshotStore, + ProviderSnapshotStoreShape +>()("t3/persistence/Services/HubThreadMachineState/ProviderSnapshotStore") {} From 5df744810da04418a6577a45846ad3937e873972 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 01:15:06 +0000 Subject: [PATCH 39/58] feat(server): push the hub's provider instance settings to thread runners Runner protocol 2 adds runner.provider.configure. The hub sends its effective settings for an instance, sensitive environment values decrypted from its secret store, when a thread's runner connects (for the thread's instance), before every session start and before title or branch-name generation. The runner layers them over its own settings in memory only (RunnerProviderSettings), so custom instances and API keys configured on the hub work on thread machines without anything being written to the machine; identical settings are a no-op. Runners that speak protocol 1 are left as they are. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../HubRunnerLoopbackHarness.integration.ts | 17 ++- .../src/hub/HubProviderSnapshots.test.ts | 119 +++++++++++++++ apps/server/src/hub/HubRunnerLifecycle.ts | 5 +- .../src/hub/RemoteProviderDriver.test.ts | 9 +- apps/server/src/hub/RemoteProviderDriver.ts | 106 ++++++++++++-- apps/server/src/hub/RunnerConnectionPool.ts | 63 +++++--- apps/server/src/runner/RunnerHandlers.test.ts | 43 +++++- apps/server/src/runner/RunnerHandlers.ts | 9 ++ .../src/runner/RunnerProviderSettings.test.ts | 91 ++++++++++++ .../src/runner/RunnerProviderSettings.ts | 136 ++++++++++++++++++ apps/server/src/runner/RunnerServer.ts | 6 +- packages/contracts/src/runner.ts | 30 +++- 12 files changed, 596 insertions(+), 38 deletions(-) create mode 100644 apps/server/src/runner/RunnerProviderSettings.test.ts create mode 100644 apps/server/src/runner/RunnerProviderSettings.ts diff --git a/apps/server/integration/HubRunnerLoopbackHarness.integration.ts b/apps/server/integration/HubRunnerLoopbackHarness.integration.ts index c55057324ddc..93e38faf906d 100644 --- a/apps/server/integration/HubRunnerLoopbackHarness.integration.ts +++ b/apps/server/integration/HubRunnerLoopbackHarness.integration.ts @@ -32,6 +32,7 @@ import { ThreadId, type OrchestrationThread, type ProviderApprovalDecision, + type ProviderInstanceConfig, type ServerProvider, } from "@t3tools/contracts"; import { @@ -61,6 +62,7 @@ import * as RemoteSessionRegistry from "../src/hub/RemoteSessionRegistry.ts"; import { make as makePool, RunnerConnectionPool } from "../src/hub/RunnerConnectionPool.ts"; import * as RunnerEventDelivery from "../src/hub/RunnerEventDelivery.ts"; import * as ThreadMachineStates from "../src/hub/ThreadMachineStates.ts"; +import { RunnerProviderSettings } from "../src/runner/RunnerProviderSettings.ts"; import * as HubProviderSnapshots from "../src/hub/HubProviderSnapshots.ts"; import { serveRunner } from "../src/hub/testUtils/runnerServer.ts"; import { makeOrchestrationCommandDispatcher } from "../src/orchestration/CommandDispatcher.ts"; @@ -221,6 +223,8 @@ export interface LoopbackRunner { readonly url: string; readonly adapterHarness: TestProviderAdapterHarness; readonly outbox: RunnerOutboxShape; + /** Every `runner.provider.configure` push, as the settings the hub sent. */ + readonly configured: Array>>; } export interface HubRunnerLoopbackHarness { @@ -293,8 +297,19 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") const outbox = yield* makeOutbox({ databasePath: NodePath.join(runnerStateDir, "runner", "outbox.sqlite"), }).pipe(Scope.provide(scope)); + const configured: Array>> = []; const runnerLayer = Layer.mergeAll(RunnerEventPumpLive, RunnerRpcHandlersLive).pipe( Layer.provideMerge(Layer.succeed(RunnerOutbox, outbox)), + Layer.provideMerge( + Layer.succeed(RunnerProviderSettings, { + apply: (instances) => + Effect.sync(() => { + configured.push(instances); + return [LOOPBACK_INSTANCE_ID]; + }), + effectiveSettings: Effect.die("unused in the loopback harness"), + }), + ), Layer.provideMerge(instanceRegistryLayer(adapterHarness.adapter)), Layer.provideMerge( Layer.mergeAll( @@ -335,7 +350,7 @@ export const makeHubRunnerLoopbackHarness = (threadIdValue = "thread-loopback") ); const served = yield* serveRunner(runnerLayer).pipe(Scope.provide(scope)); runnerScope = scope; - runner = { url: served.url, adapterHarness, outbox }; + runner = { url: served.url, adapterHarness, outbox, configured }; return runner; }); diff --git a/apps/server/src/hub/HubProviderSnapshots.test.ts b/apps/server/src/hub/HubProviderSnapshots.test.ts index 774f2ee88e8a..740d3f847529 100644 --- a/apps/server/src/hub/HubProviderSnapshots.test.ts +++ b/apps/server/src/hub/HubProviderSnapshots.test.ts @@ -15,6 +15,7 @@ import { ServerConfig } from "../config.ts"; import { HubThreadMachineStateSqliteLive } from "../persistence/Layers/HubThreadMachineState.ts"; import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; import { ClaudeDriver } from "../provider/Drivers/ClaudeDriver.ts"; import { CodexDriver } from "../provider/Drivers/CodexDriver.ts"; import * as HubProviderSnapshots from "./HubProviderSnapshots.ts"; @@ -28,6 +29,7 @@ import { fakeRunner, fakeRunnerHello, serveRunner } from "./testUtils/runnerServ const claude = ProviderDriverKind.make("claudeAgent"); const codex = ProviderDriverKind.make("codex"); const claudeInstance = ProviderInstanceId.make("claudeAgent"); +const claudeWork = ProviderInstanceId.make("claude_work"); const threadId = ThreadId.make("thread-provider-snapshots"); const identity = (driverKind: ProviderDriverKind, enabled = true) => ({ @@ -39,9 +41,23 @@ const identity = (driverKind: ProviderDriverKind, enabled = true) => ({ continuationGroupKey: `${driverKind}:group`, }); +const workInstance = ProviderInstanceId.make("claude_work"); + const StoresLive = Layer.mergeAll( HubThreadMachineStateSqliteLive, ProviderSessionRuntime.layer, + ServerSettingsService.layerTest({ + providerInstances: { + [workInstance]: { + driver: claude, + displayName: "Claude (work)", + environment: [ + { name: "ANTHROPIC_BASE_URL", value: "https://proxy.example", sensitive: false }, + { name: "ANTHROPIC_API_KEY", value: "sk-hub-secret", sensitive: true }, + ], + }, + }, + }), ).pipe(Layer.provideMerge(SqlitePersistenceMemory), Layer.provideMerge(NodeServices.layer)); const reported = (auth: ServerProvider["auth"]["status"]): ServerProvider => ({ @@ -161,4 +177,107 @@ describe("hub provider snapshots", () => { expect(Option.getOrThrow(persisted).auth.status).toBe("authenticated"); }).pipe(Effect.scoped, Effect.provide(StoresLive)), ); + + it.live( + "pushes the hub's instance settings, secrets included, on connect and session start", + () => + Effect.gen(function* () { + const calls: Array = []; + const pushed: Array = []; + const runner = yield* serveRunner( + fakeRunner({ + "runner.hello": (input) => + Effect.succeed(fakeRunnerHello({ threadId: input.threadId, instances: [] })), + "runner.provider.configure": ({ instances }) => + Effect.sync(() => { + calls.push("configure"); + pushed.push(instances); + return { + instances: Object.keys(instances).map((id) => ProviderInstanceId.make(id)), + }; + }), + "runner.provider.getCapabilities": () => + Effect.sync(() => { + calls.push("getCapabilities"); + return { + snapshot: { ...reported("authenticated"), instanceId: claudeWork }, + sessionModelSwitch: "in-session" as const, + }; + }), + "runner.provider.startSession": ({ input }) => + Effect.sync(() => { + calls.push("startSession"); + return { + provider: claude, + status: "ready" as const, + runtimeMode: input.runtimeMode, + threadId: input.threadId, + createdAt: "t", + updatedAt: "t", + }; + }), + }), + ); + const fake = yield* makeFakeMachineDirectory({ + onWake: () => ({ state: "running", runnerUrl: runner.url }), + }); + const pool = yield* makePool({ + wakePollInterval: "5 millis", + idleCheckInterval: "1 hour", + }).pipe(Effect.provideService(MachineDirectory, fake.directory)); + yield* pool.setContextResolver(() => + Effect.succeed({ + projectId: null, + repository: null, + branch: null, + providerInstanceId: claudeWork, + }), + ); + const registry = yield* RemoteSessionRegistry.make; + const delivery = yield* RunnerEventDelivery.make.pipe( + Effect.provideService(RunnerConnectionPool, pool), + Effect.provideService(RemoteSessionRegistry.RemoteSessionRegistry, registry), + ); + const instance = yield* makeRemoteProviderDriver(ClaudeDriver) + .create({ + instanceId: claudeWork, + displayName: "Claude (work)", + environment: [], + enabled: true, + config: ClaudeDriver.defaultConfig(), + }) + .pipe( + Effect.provideService(RunnerConnectionPool, pool), + Effect.provideService(RunnerEventDelivery.RunnerEventDelivery, delivery), + Effect.provideService(RemoteSessionRegistry.RemoteSessionRegistry, registry), + Effect.provide(HubProviderSnapshots.layer), + Effect.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-push-" })), + ); + + // Connecting pushes the thread's instance, then reads its status. + yield* pool.use(threadId, { wake: true, operation: "test" }, () => Effect.void); + yield* Effect.repeat(instance.snapshot.getSnapshot, { + until: (snapshot) => snapshot.auth.status === "authenticated", + }).pipe(Effect.timeout("5 seconds")); + expect(calls).toEqual(["configure", "getCapabilities"]); + + yield* instance.adapter.startSession({ + threadId, + provider: claude, + providerInstanceId: claudeWork, + runtimeMode: "full-access", + }); + expect(calls).toEqual(["configure", "getCapabilities", "configure", "startSession"]); + expect(pushed[1]).toEqual({ + claude_work: { + driver: "claudeAgent", + displayName: "Claude (work)", + environment: [ + { name: "ANTHROPIC_BASE_URL", value: "https://proxy.example", sensitive: false }, + { name: "ANTHROPIC_API_KEY", value: "sk-hub-secret", sensitive: true }, + ], + }, + }); + }).pipe(Effect.scoped, Effect.provide(StoresLive)), + ); }); diff --git a/apps/server/src/hub/HubRunnerLifecycle.ts b/apps/server/src/hub/HubRunnerLifecycle.ts index 2d5b63f91335..911bde527c15 100644 --- a/apps/server/src/hub/HubRunnerLifecycle.ts +++ b/apps/server/src/hub/HubRunnerLifecycle.ts @@ -88,7 +88,9 @@ export const layer = Layer.effectDiscard( .getThreadShellById(threadId) .pipe(Effect.orElseSucceed(() => Option.none())), ); - if (!thread) return { projectId: null, repository: null, branch: null }; + if (!thread) { + return { projectId: null, repository: null, branch: null, providerInstanceId: null }; + } const project = Option.getOrUndefined( yield* projections .getProjectShellById(thread.projectId) @@ -99,6 +101,7 @@ export const layer = Layer.effectDiscard( projectId: thread.projectId, repository: identity ? { url: identity.locator.remoteUrl, ref: null } : null, branch: thread.branch, + providerInstanceId: thread.modelSelection.instanceId, }; }), ); diff --git a/apps/server/src/hub/RemoteProviderDriver.test.ts b/apps/server/src/hub/RemoteProviderDriver.test.ts index 55e6be6e1999..9454505e2edb 100644 --- a/apps/server/src/hub/RemoteProviderDriver.test.ts +++ b/apps/server/src/hub/RemoteProviderDriver.test.ts @@ -29,6 +29,7 @@ import { makeHubTestSchema, } from "../persistence/Postgres/hubTestDatabase.ts"; import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; import { CodexDriver } from "../provider/Drivers/CodexDriver.ts"; import * as HubProviderSnapshots from "./HubProviderSnapshots.ts"; import { MachineDirectory, makeFakeMachineDirectory } from "./MachineDirectory.ts"; @@ -128,7 +129,13 @@ describe("remote provider driver helpers", () => { }); const StoresLive = HubProviderSnapshots.layer.pipe( - Layer.provideMerge(Layer.mergeAll(HubThreadMachineStateSqliteLive, ProviderSessionRuntime.layer)), + Layer.provideMerge( + Layer.mergeAll( + HubThreadMachineStateSqliteLive, + ProviderSessionRuntime.layer, + ServerSettingsService.layerTest(), + ), + ), Layer.provideMerge(SqlitePersistenceMemory), Layer.provideMerge(NodeServices.layer), ); diff --git a/apps/server/src/hub/RemoteProviderDriver.ts b/apps/server/src/hub/RemoteProviderDriver.ts index da0d30a3ecc6..87ba7574dc83 100644 --- a/apps/server/src/hub/RemoteProviderDriver.ts +++ b/apps/server/src/hub/RemoteProviderDriver.ts @@ -25,6 +25,12 @@ * recreated) is restarted from the recorded resume cursor before a turn is * sent, after boot reconciliation has settled the old one. * + * Provider settings: the hub's effective settings for the instance, sensitive + * environment values decrypted from the hub's secret store, are pushed to the + * runner (`runner.provider.configure`, protocol 2) when a thread's runner + * connects (for the thread's instance), before every session start and before + * title or branch-name generation. The runner keeps them in memory only. + * * @module hub/RemoteProviderDriver */ import * as NodeFS from "node:fs"; @@ -32,6 +38,7 @@ import * as NodeFS from "node:fs"; import { type ChatAttachment, type ProviderDriverKind, + type ProviderInstanceConfig, type ProviderInstanceId, type ProviderSession, type ServerProvider, @@ -40,6 +47,7 @@ import { } from "@t3tools/contracts"; import { parseThreadCheckoutPath, + RUNNER_PROTOCOL_PROVIDER_SETTINGS, type RunnerAttachmentFile, type RunnerMcpSession, ThreadMachineUnavailableError, @@ -57,6 +65,8 @@ import * as Stream from "effect/Stream"; import { resolveAttachmentPath } from "../attachmentStore.ts"; import { ServerConfig } from "../config.ts"; import * as McpProviderSession from "../mcp/McpProviderSession.ts"; +import { deriveProviderInstanceConfigMap } from "../provider/Layers/ProviderInstanceRegistryHydration.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; import { hydrateHubAttachment } from "../persistence/Postgres/HubAttachments.ts"; import { type ProviderAdapterError, @@ -91,6 +101,7 @@ export type RemoteProviderDriverEnv = | RunnerEventDelivery | RemoteSessionRegistry | HubProviderSnapshots + | ServerSettingsService | ServerConfig | FileSystem.FileSystem | Path.Path; @@ -161,8 +172,39 @@ interface InstanceContext { attachments: ReadonlyArray | undefined, ) => Effect.Effect, RunnerAttachmentReadError>; readonly publicUrl: string | undefined; + /** + * Pushes the hub's effective settings for this instance to a runner that + * speaks protocol 2; returns the instances the runner hosts afterwards, or + * none when nothing was pushed. + */ + readonly pushSettings: ( + connection: RunnerConnection, + ) => Effect.Effect>>; } +/** + * The hub's effective settings for one instance (explicit `providerInstances` + * or the legacy `providers.` mirror), with sensitive values included. + */ +export const effectiveInstanceSettings = ( + serverSettings: ServerSettingsService["Service"], + instanceId: ProviderInstanceId, +) => + serverSettings.getSettings.pipe( + Effect.map((settings): ProviderInstanceConfig | undefined => { + const entry = deriveProviderInstanceConfigMap(settings)[instanceId]; + if (entry === undefined) return undefined; + return entry.environment === undefined + ? entry + : { + ...entry, + environment: entry.environment.map( + ({ valueRedacted: _redacted, ...variable }) => variable, + ), + }; + }), + ); + export function makeRemoteProviderDriver( base: AnyProviderDriver, ): AnyProviderDriver { @@ -175,7 +217,27 @@ export function makeRemoteProviderDriver( create: ({ instanceId, displayName, accentColor, enabled, config: driverConfig }) => Effect.gen(function* () { const config = yield* ServerConfig; + const serverSettings = yield* ServerSettingsService; const platform = yield* Effect.context(); + const pushSettings: InstanceContext["pushSettings"] = (connection) => + connection.hello.protocolVersion < RUNNER_PROTOCOL_PROVIDER_SETTINGS + ? Effect.succeed(Option.none()) + : effectiveInstanceSettings(serverSettings, instanceId).pipe( + Effect.flatMap((settings) => + settings === undefined + ? Effect.succeed(Option.none>()) + : connection.client["runner.provider.configure"]({ + instances: { [instanceId]: settings }, + }).pipe(Effect.map((result) => Option.some(result.instances))), + ), + Effect.catchCause((cause) => + Effect.logWarning("provider settings were not pushed to the runner", { + threadId: connection.threadId, + instanceId, + detail: Cause.pretty(cause).slice(0, 300), + }).pipe(Effect.as(Option.none>())), + ), + ); const context: InstanceContext = { driverKind, instanceId, @@ -187,6 +249,7 @@ export function makeRemoteProviderDriver( Effect.provideContext(platform), ), publicUrl: config.hub?.publicUrl, + pushSettings, }; const capabilities: { current: Omit; @@ -291,11 +354,16 @@ function makeRemoteAdapter( connection: RunnerConnection, input: Parameters["startSession"]>[0], ) => - connection.client["runner.provider.startSession"]({ - instanceId, - input, - mcp: mcpSessionForRunner(input.threadId, context.publicUrl), - }).pipe(Effect.tap((session) => recordSession(session, connection))); + context.pushSettings(connection).pipe( + Effect.andThen( + connection.client["runner.provider.startSession"]({ + instanceId, + input, + mcp: mcpSessionForRunner(input.threadId, context.publicUrl), + }), + ), + Effect.tap((session) => recordSession(session, connection)), + ); const hostedSession = (threadId: ThreadId) => registry @@ -461,8 +529,8 @@ const makeRemoteSnapshot = (input: { Effect.forkScoped, ); - const fetchFrom = (connection: RunnerConnection, refresh: boolean) => - connection.hello.instances.includes(context.instanceId) + const fetchFrom = (connection: RunnerConnection, refresh: boolean, hosted?: boolean) => + (hosted ?? connection.hello.instances.includes(context.instanceId)) ? connection.client["runner.provider.getCapabilities"]({ instanceId: context.instanceId, ...(refresh ? { refresh } : {}), @@ -480,7 +548,25 @@ const makeRemoteSnapshot = (input: { ) : Effect.void; - yield* context.pool.onConnection((connection) => fetchFrom(connection, false)); + /** The instance a thread uses: its live session's, else its model selection's. */ + const threadInstance = (threadId: ThreadId) => + Effect.gen(function* () { + const record = yield* context.registry.get(threadId); + if (Option.isSome(record)) return record.value.instanceId; + return (yield* context.pool.contextOf(threadId))?.providerInstanceId ?? null; + }); + + // Scoped to the instance: a rebuilt instance (settings changed) replaces it. + yield* context.pool.onConnectionScoped((connection) => + Effect.gen(function* () { + let hosted = connection.hello.instances; + if ((yield* threadInstance(connection.threadId)) === context.instanceId) { + const pushed = yield* context.pushSettings(connection); + if (Option.isSome(pushed)) hosted = pushed.value; + } + if (hosted.includes(context.instanceId)) yield* fetchFrom(connection, false, true); + }), + ); const refresh = Effect.gen(function* () { const [connection] = yield* context.pool.connections; @@ -527,7 +613,9 @@ function makeRemoteTextGeneration(context: InstanceContext): TextGenerationShape }), ) : pool - .use(threadId, { wake: true, operation: `text.${operation}` }, f) + .use(threadId, { wake: true, operation: `text.${operation}` }, (connection) => + context.pushSettings(connection).pipe(Effect.andThen(f(connection))), + ) .pipe(Effect.mapError(toTextGenerationError(operation))); }; const onRunnerOnly = (operation: string) => diff --git a/apps/server/src/hub/RunnerConnectionPool.ts b/apps/server/src/hub/RunnerConnectionPool.ts index ae5cf12f4a05..3b6647ad77ee 100644 --- a/apps/server/src/hub/RunnerConnectionPool.ts +++ b/apps/server/src/hub/RunnerConnectionPool.ts @@ -22,7 +22,7 @@ * * @module hub/RunnerConnectionPool */ -import { type ProjectId, type ThreadId } from "@t3tools/contracts"; +import { type ProjectId, type ProviderInstanceId, type ThreadId } from "@t3tools/contracts"; import { RUNNER_MIN_PROTOCOL_VERSION, RUNNER_PROTOCOL_VERSION, @@ -71,6 +71,8 @@ export interface ThreadMachineContext { readonly projectId: ProjectId | null; readonly repository: ThreadMachineRepository | null; readonly branch: string | null; + /** The provider instance the thread's model selection names, when known. */ + readonly providerInstanceId?: ProviderInstanceId | null; } export type ThreadMachineContextResolver = ( @@ -121,6 +123,12 @@ export interface RunnerConnectionPoolShape { ) => Effect.Effect; /** Runs `handler` in the scope of every current and future connection. */ readonly onConnection: (handler: ConnectionHandler) => Effect.Effect; + /** Like `onConnection`, until the caller's scope closes. */ + readonly onConnectionScoped: ( + handler: ConnectionHandler, + ) => Effect.Effect; + /** The thread's machine context from the registered resolver, if any. */ + readonly contextOf: (threadId: ThreadId) => Effect.Effect; readonly lifecycle: Stream.Stream; /** Busy marks keep a connection open (and reconnecting) while set. */ readonly setBusy: (threadId: ThreadId, reason: string, busy: boolean) => Effect.Effect; @@ -651,6 +659,26 @@ export const make = (options: RunnerConnectionPoolOptions = {}) => }), ); + const registerHandler = (handler: ConnectionHandler) => + Effect.gen(function* () { + handlers.push(handler); + for (const slot of slots.values()) { + if (slot.connection && slot.scope && !slot.closing) { + yield* handler(slot.connection).pipe( + Scope.provide(slot.scope), + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.void + : Effect.logWarning("runner connection handler failed", { + cause: Cause.pretty(cause), + }), + ), + Effect.forkIn(slot.scope), + ); + } + } + }); + return RunnerConnectionPool.of({ sweepIdle, checkoutRoot, @@ -671,25 +699,20 @@ export const make = (options: RunnerConnectionPoolOptions = {}) => ), machineStatus: (threadId) => directory.status(threadId).pipe(Effect.mapError(fromDirectoryError(threadId, "status"))), - onConnection: (handler) => - Effect.gen(function* () { - handlers.push(handler); - for (const slot of slots.values()) { - if (slot.connection && slot.scope && !slot.closing) { - yield* handler(slot.connection).pipe( - Scope.provide(slot.scope), - Effect.catchCause((cause) => - Cause.hasInterruptsOnly(cause) - ? Effect.void - : Effect.logWarning("runner connection handler failed", { - cause: Cause.pretty(cause), - }), - ), - Effect.forkIn(slot.scope), - ); - } - } - }), + contextOf: (threadId) => + contextResolver === null + ? Effect.succeed(null) + : contextResolver(threadId).pipe( + Effect.catchCause(() => Effect.succeed(null)), + ), + onConnectionScoped: (handler) => + Effect.acquireRelease(registerHandler(handler), () => + Effect.sync(() => { + const index = handlers.indexOf(handler); + if (index >= 0) handlers.splice(index, 1); + }), + ), + onConnection: (handler) => registerHandler(handler), lifecycle: Stream.fromPubSub(lifecycle), setBusy: (threadId, reason, busy) => Clock.currentTimeMillis.pipe( diff --git a/apps/server/src/runner/RunnerHandlers.test.ts b/apps/server/src/runner/RunnerHandlers.test.ts index d2d09c73c984..dceb1634c489 100644 --- a/apps/server/src/runner/RunnerHandlers.test.ts +++ b/apps/server/src/runner/RunnerHandlers.test.ts @@ -42,6 +42,7 @@ import * as WorkspaceFileSystem from "../workspace/WorkspaceFileSystem.ts"; import * as RunnerCheckout from "./RunnerCheckout.ts"; import { RunnerRpcHandlersLive } from "./RunnerHandlers.ts"; import { makeWithOptions as makeOutbox, RunnerOutbox } from "./RunnerOutbox.ts"; +import { RunnerProviderSettings } from "./RunnerProviderSettings.ts"; const threadId = ThreadId.make("thread-handlers"); const provider = ProviderDriverKind.make("claudeAgent"); @@ -59,6 +60,7 @@ const setup = Effect.gen(function* () { const checkout = NodePath.join(root, "t", threadId); NodeFS.mkdirSync(checkout, { recursive: true }); const turns: Array = []; + const configured: Array> = []; const mcpAtStart: Array = []; const events = yield* PubSub.unbounded(); const adapter = { @@ -143,6 +145,17 @@ const setup = Effect.gen(function* () { Layer.mock(VcsProvisioningService.VcsProvisioningService)({}), Layer.mock(ReviewService.ReviewService)({}), Layer.mock(TerminalManager.TerminalManager)({}), + Layer.succeed(RunnerProviderSettings, { + apply: (instances) => + Effect.sync(() => { + configured.push(Object.keys(instances)); + return [ + instanceId, + ...Object.keys(instances).map((id) => ProviderInstanceId.make(id)), + ]; + }), + effectiveSettings: Effect.die("unused"), + }), ServerSettingsService.layerTest(), ), ), @@ -159,7 +172,15 @@ const setup = Effect.gen(function* () { const runnerConfig = yield* ServerConfig.pipe( Effect.provide(config.pipe(Layer.provide(NodeServices.layer))), ); - return { root, checkout, turns, mcpAtStart, pool, attachmentsDir: runnerConfig.attachmentsDir }; + return { + root, + checkout, + turns, + mcpAtStart, + configured, + pool, + attachmentsDir: runnerConfig.attachmentsDir, + }; }); describe("runner handlers", () => { @@ -253,4 +274,24 @@ describe("runner handlers", () => { }), ), ); + + it.live("applies the hub's provider settings and reports the hosted instances", () => + Effect.scoped( + Effect.gen(function* () { + const { pool, configured } = yield* setup; + const result = yield* pool.use(threadId, { wake: false, operation: "test" }, (connection) => + connection.client["runner.provider.configure"]({ + instances: { + [ProviderInstanceId.make("claude_work")]: { + driver: provider, + environment: [{ name: "ANTHROPIC_API_KEY", value: "sk-test", sensitive: true }], + }, + }, + }), + ); + expect(result.instances).toEqual(["claudeAgent", "claude_work"]); + expect(configured).toEqual([["claude_work"]]); + }), + ), + ); }); diff --git a/apps/server/src/runner/RunnerHandlers.ts b/apps/server/src/runner/RunnerHandlers.ts index 1a1d626b94ff..3e79d81bd01e 100644 --- a/apps/server/src/runner/RunnerHandlers.ts +++ b/apps/server/src/runner/RunnerHandlers.ts @@ -69,6 +69,7 @@ import * as WorkspaceFileSystem from "../workspace/WorkspaceFileSystem.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; import { RunnerCheckout } from "./RunnerCheckout.ts"; import { RunnerOutbox } from "./RunnerOutbox.ts"; +import { RunnerProviderSettings } from "./RunnerProviderSettings.ts"; import { ProviderAdapterErrorSchema, toRunnerRemoteError } from "./remoteErrors.ts"; const DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL = Duration.minutes(5); @@ -126,6 +127,7 @@ export const RunnerRpcHandlersLive = RunnerRpcGroup.toLayer( const review = yield* ReviewService.ReviewService; const terminals = yield* TerminalManager.TerminalManager; const serverSettings = yield* ServerSettingsService; + const providerSettings = yield* RunnerProviderSettings; const automaticGitFetchInterval = serverSettings.getSettings.pipe( Effect.map( @@ -338,6 +340,13 @@ export const RunnerRpcHandlersLive = RunnerRpcGroup.toLayer( }), // ── Provider sessions ──────────────────────────────────────────── + // Only instance ids are logged; the settings carry secrets. + "runner.provider.configure": ({ instances }) => + logged( + "configure", + { instances: Object.keys(instances) }, + providerSettings.apply(instances).pipe(Effect.map((hosted) => ({ instances: hosted }))), + ), "runner.provider.startSession": ({ instanceId, input, mcp }) => guardThread("startSession", input.threadId).pipe( Effect.andThen( diff --git a/apps/server/src/runner/RunnerProviderSettings.test.ts b/apps/server/src/runner/RunnerProviderSettings.test.ts new file mode 100644 index 000000000000..fae07ac28499 --- /dev/null +++ b/apps/server/src/runner/RunnerProviderSettings.test.ts @@ -0,0 +1,91 @@ +import { it } from "@effect/vitest"; +import { + type ProviderInstanceEnvironment, + ProviderDriverKind, + ProviderInstanceId, + type ServerProvider, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import { describe, expect } from "vite-plus/test"; + +import type { AnyProviderDriver, ProviderInstance } from "../provider/ProviderDriver.ts"; +import type { ProviderAdapterShape } from "../provider/Services/ProviderAdapter.ts"; +import { ProviderInstanceRegistry } from "../provider/Services/ProviderInstanceRegistry.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; +import type { TextGenerationShape } from "../textGeneration/TextGeneration.ts"; +import { + makeRunnerProviderRegistryLayer, + RunnerProviderSettings, +} from "./RunnerProviderSettings.ts"; + +const driverKind = ProviderDriverKind.make("claudeAgent"); + +/** A driver that records every instance it builds and the environment it got. */ +const recordingDriver = (created: Array<{ id: string; env: ProviderInstanceEnvironment }>) => + ({ + driverKind, + metadata: { displayName: "Claude", supportsMultipleInstances: true }, + configSchema: Schema.Unknown as unknown as Schema.Codec, + defaultConfig: () => ({}), + create: ({ instanceId, displayName, enabled, environment }) => + Effect.sync(() => { + created.push({ id: instanceId, env: environment }); + return { + instanceId, + driverKind, + continuationIdentity: { driverKind, continuationKey: instanceId }, + displayName, + enabled, + snapshot: { + maintenanceCapabilities: { provider: driverKind, packageName: null, update: null }, + getSnapshot: Effect.succeed({} as ServerProvider), + refresh: Effect.succeed({} as ServerProvider), + streamChanges: Stream.empty, + }, + adapter: {} as ProviderAdapterShape, + textGeneration: {} as TextGenerationShape, + } satisfies ProviderInstance; + }), + }) satisfies AnyProviderDriver; + +describe("runner provider settings", () => { + it.effect("layers the hub's instance settings over the local ones, in memory only", () => + Effect.gen(function* () { + const created: Array<{ id: string; env: ProviderInstanceEnvironment }> = []; + const layer = makeRunnerProviderRegistryLayer([recordingDriver(created)]).pipe( + Layer.provideMerge(ServerSettingsService.layerTest()), + ); + yield* Effect.gen(function* () { + const settings = yield* RunnerProviderSettings; + const registry = yield* ProviderInstanceRegistry; + const serverSettings = yield* ServerSettingsService; + expect(created.map((entry) => entry.id)).toEqual(["claudeAgent"]); + + const work = { + driver: driverKind, + environment: [{ name: "ANTHROPIC_API_KEY", value: "sk-hub", sensitive: true }], + }; + const hosted = yield* settings.apply({ claude_work: work }); + expect(hosted).toEqual(["claudeAgent", "claude_work"]); + expect(created.at(-1)).toEqual({ id: "claude_work", env: work.environment }); + + // The same settings again change nothing; new settings rebuild the instance. + yield* settings.apply({ claude_work: work }); + expect(created.length).toBe(2); + yield* settings.apply({ + claude_work: { ...work, environment: [{ ...work.environment[0]!, value: "sk-new" }] }, + }); + expect(created.length).toBe(3); + expect(yield* registry.getInstance(ProviderInstanceId.make("claude_work"))).toBeDefined(); + + // The runner's own settings never receive the pushed secret. + expect((yield* serverSettings.getSettings).providerInstances).toEqual({}); + const effective = yield* settings.effectiveSettings; + expect(Object.keys(effective.providerInstances)).toEqual(["claude_work"]); + }).pipe(Effect.provide(layer)); + }), + ); +}); diff --git a/apps/server/src/runner/RunnerProviderSettings.ts b/apps/server/src/runner/RunnerProviderSettings.ts new file mode 100644 index 000000000000..e49b20baa30e --- /dev/null +++ b/apps/server/src/runner/RunnerProviderSettings.ts @@ -0,0 +1,136 @@ +/** + * RunnerProviderSettings - provider instance settings on a runner. + * + * A runner's provider registry starts from its own settings file (defaults on + * a fresh thread machine). The hub pushes its effective settings for the + * instances a thread uses (`runner.provider.configure`, on connect and before + * each session start), sensitive environment values included. They are + * layered over the local settings in memory only: nothing is written to the + * runner's settings or secrets, and a restarted runner starts from its local + * settings again until the hub pushes. A runner serves exactly one thread, so + * pushed settings apply to that thread's sessions only. + * + * Applying settings that equal the live ones is a no-op; a changed instance is + * rebuilt by the registry, as a settings change is on any server. + * + * @module runner/RunnerProviderSettings + */ +import type { + ProviderInstanceConfig, + ProviderInstanceConfigMap, + ProviderInstanceId, + ServerSettings, + ServerSettingsError, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; + +import { type BuiltInDriversEnv } from "../provider/builtInDrivers.ts"; +import { deriveProviderInstanceConfigMap } from "../provider/Layers/ProviderInstanceRegistryHydration.ts"; +import { ProviderInstanceRegistryMutableLayer } from "../provider/Layers/ProviderInstanceRegistryLive.ts"; +import type { AnyProviderDriver } from "../provider/ProviderDriver.ts"; +import { ProviderInstanceRegistry } from "../provider/Services/ProviderInstanceRegistry.ts"; +import { ProviderInstanceRegistryMutator } from "../provider/Services/ProviderInstanceRegistryMutator.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; + +export interface RunnerProviderSettingsShape { + /** Layers the hub's instance settings over the local ones; returns the hosted instances. */ + readonly apply: ( + instances: Readonly>, + ) => Effect.Effect>; + /** Local settings with the hub's instance settings layered over them. */ + readonly effectiveSettings: Effect.Effect; +} + +export class RunnerProviderSettings extends Context.Service< + RunnerProviderSettings, + RunnerProviderSettingsShape +>()("t3/runner/RunnerProviderSettings") {} + +/** + * The runner's provider registry: the local settings file with hub-pushed + * instance settings layered over it, reconciled whenever either changes. + */ +export const makeRunnerProviderRegistryLayer = ( + drivers: ReadonlyArray>, +): Layer.Layer< + ProviderInstanceRegistry | RunnerProviderSettings, + never, + R | ServerSettingsService +> => { + const configDrivers = drivers as unknown as ReadonlyArray>; + return Layer.unwrap( + Effect.gen(function* () { + const serverSettings = yield* ServerSettingsService; + const initial = yield* serverSettings.getSettings.pipe(Effect.orElseSucceed(() => undefined)); + const initialMap = + initial === undefined + ? ({} as ProviderInstanceConfigMap) + : deriveProviderInstanceConfigMap(initial, configDrivers); + + const settingsLayer = Layer.effect( + RunnerProviderSettings, + Effect.gen(function* () { + const mutator = yield* ProviderInstanceRegistryMutator; + const registry = yield* ProviderInstanceRegistry; + const pushed = yield* Ref.make>({}); + const lock = yield* Semaphore.make(1); + let local: ServerSettings | undefined = initial; + + const reconcile = lock.withPermit( + Effect.gen(function* () { + const base = + local === undefined ? {} : deriveProviderInstanceConfigMap(local, configDrivers); + yield* mutator.reconcile({ + ...base, + ...(yield* Ref.get(pushed)), + } as ProviderInstanceConfigMap); + }), + ); + + yield* serverSettings.streamChanges.pipe( + Stream.runForEach((next) => + Effect.sync(() => { + local = next; + }).pipe(Effect.andThen(reconcile)), + ), + Effect.forkScoped, + ); + + return RunnerProviderSettings.of({ + apply: (instances) => + Ref.update(pushed, (current) => ({ ...current, ...instances })).pipe( + Effect.andThen(reconcile), + Effect.andThen(registry.listInstances), + Effect.map((live) => live.map((instance) => instance.instanceId)), + ), + effectiveSettings: Effect.gen(function* () { + const settings = yield* serverSettings.getSettings; + return { + ...settings, + providerInstances: { + ...settings.providerInstances, + ...(yield* Ref.get(pushed)), + } as ServerSettings["providerInstances"], + }; + }), + }); + }), + ); + + return settingsLayer.pipe( + Layer.provideMerge( + ProviderInstanceRegistryMutableLayer({ drivers, configMap: initialMap }), + ), + ); + }), + ) as Layer.Layer< + ProviderInstanceRegistry | RunnerProviderSettings, + never, + R | ServerSettingsService + >; +}; diff --git a/apps/server/src/runner/RunnerServer.ts b/apps/server/src/runner/RunnerServer.ts index 3214f54310d3..2a071ec19c8a 100644 --- a/apps/server/src/runner/RunnerServer.ts +++ b/apps/server/src/runner/RunnerServer.ts @@ -36,7 +36,6 @@ import type { ProviderAdapterError } from "../provider/Errors.ts"; import * as ModelManifest from "../provider/ModelManifest.ts"; import * as OpenCodeRuntime from "../provider/opencodeRuntime.ts"; import * as ProviderEventLoggers from "../provider/Layers/ProviderEventLoggers.ts"; -import { makeProviderInstanceRegistryHydration } from "../provider/Layers/ProviderInstanceRegistryHydration.ts"; import { ProviderRegistryLive } from "../provider/Layers/ProviderRegistry.ts"; import type { ProviderAdapterShape } from "../provider/Services/ProviderAdapter.ts"; import { ProviderInstanceRegistry } from "../provider/Services/ProviderInstanceRegistry.ts"; @@ -62,6 +61,7 @@ import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; import * as RunnerCheckout from "./RunnerCheckout.ts"; import { RunnerRpcHandlersLive, resolveRunnerBinding } from "./RunnerHandlers.ts"; import { RunnerOutbox, layer as RunnerOutboxLayer } from "./RunnerOutbox.ts"; +import { makeRunnerProviderRegistryLayer } from "./RunnerProviderSettings.ts"; import * as RunnerProjectionSnapshotQuery from "./RunnerProjectionSnapshotQuery.ts"; /** Time given to session-exit events to reach the outbox on graceful shutdown. */ @@ -70,6 +70,8 @@ const SHUTDOWN_DRAIN = "300 millis"; /** * `T3CODE_RUNNER_DRIVERS=claudeAgent,codex` limits the hosted drivers (for * development and tests); by default a runner hosts every built-in driver. + * Instance settings come from the runner's settings file with the hub's + * pushed settings layered over them in memory (`RunnerProviderSettings`). */ const RunnerProviderInstanceRegistryLive = Layer.unwrap( Effect.gen(function* () { @@ -81,7 +83,7 @@ const RunnerProviderInstanceRegistryLive = Layer.unwrap( .map((value) => value.trim()) .filter((value) => value.length > 0), ); - return makeProviderInstanceRegistryHydration( + return makeRunnerProviderRegistryLayer( wanted.size === 0 ? drivers : drivers.filter((driver) => wanted.has(driver.driverKind)), ); }), diff --git a/packages/contracts/src/runner.ts b/packages/contracts/src/runner.ts index a34f42e19949..b5a9db25f0fb 100644 --- a/packages/contracts/src/runner.ts +++ b/packages/contracts/src/runner.ts @@ -89,7 +89,7 @@ import { ProviderSessionStartInput, ProviderTurnStartResult, } from "./provider.ts"; -import { ProviderInstanceId } from "./providerInstance.ts"; +import { ProviderInstanceConfig, ProviderInstanceId } from "./providerInstance.ts"; import { ThreadMachineState } from "./threadMachine.ts"; import { ProviderRuntimeEvent } from "./providerRuntime.ts"; import { @@ -116,8 +116,17 @@ import { } from "./terminal.ts"; import { VcsError } from "./vcs.ts"; -/** Current runner protocol version. Bump when an RPC changes incompatibly. */ -export const RUNNER_PROTOCOL_VERSION = 1; +/** + * Current runner protocol version. Bump when RPCs are added or change; a hub + * checks `hello.protocolVersion` before using newer calls. + * + * - 1: the initial protocol. + * - 2: `runner.provider.configure` (hub-pushed provider instance settings) and + * `runner.auth.*` (provider sign-in on the sign-in machine). + */ +export const RUNNER_PROTOCOL_VERSION = 2; +/** First version with `runner.provider.configure` and `runner.auth.*`. */ +export const RUNNER_PROTOCOL_PROVIDER_SETTINGS = 2; /** Oldest protocol version this build still speaks. */ export const RUNNER_MIN_PROTOCOL_VERSION = 1; export const RUNNER_WS_PATH = "/runner/ws"; @@ -553,6 +562,20 @@ export const RunnerRollbackThreadRpc = Rpc.make("runner.provider.rollbackThread" error: RunnerRemoteError, }); +/** + * The hub's effective settings for provider instances, sensitive environment + * values included. The runner applies them to its provider registry in memory + * (it serves one thread, so they apply to that thread's sessions only) and + * never writes them to disk; the reply lists the instances it now hosts. + */ +export const RunnerConfigureProvidersRpc = Rpc.make("runner.provider.configure", { + payload: Schema.Struct({ + instances: Schema.Record(ProviderInstanceId, ProviderInstanceConfig), + }), + success: Schema.Struct({ instances: Schema.Array(ProviderInstanceId) }), + error: RunnerRemoteError, +}); + /** Provider status and models as the runner's real driver sees them. */ export const RunnerGetCapabilitiesRpc = Rpc.make("runner.provider.getCapabilities", { payload: Schema.Struct({ ...InstanceScoped, refresh: Schema.optional(Schema.Boolean) }), @@ -941,6 +964,7 @@ export const RunnerRpcGroup = RpcGroup.make( RunnerReadThreadRpc, RunnerRollbackThreadRpc, RunnerGetCapabilitiesRpc, + RunnerConfigureProvidersRpc, RunnerGenerateThreadTitleRpc, RunnerGenerateBranchNameRpc, RunnerSubscribeEventsRpc, From 45d4df64ecf70f250f7c14dfa1f4e4bf63e278af Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 01:24:22 +0000 Subject: [PATCH 40/58] feat(web): recognize hub environments and the hub bootstrap A hub serves T3 without a machine of its own and runs every thread on its own machine. The web client decides hub mode per environment, at render time, from capabilities.threadMachines; before a managed primary's config arrives it trusts the managed bootstrap (serverMode "hub") or the cached environment descriptor, so local-checkout controls never flash. The managed bootstrap gains serverMode and the per-thread preview and browser URL templates. On a hub, primary commands skip the durable wake queue (there is no workspace to resume), each thread gets its own machine browser, and a thread's preview ports resolve through its own template. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/web/src/hubMode.test.ts | 140 ++++++++++++++++++++++++++++++ apps/web/src/hubMode.ts | 138 +++++++++++++++++++++++++++++ apps/web/src/managedDevPc.test.ts | 71 +++++++++++++++ apps/web/src/managedDevPc.ts | 52 ++++++++++- 4 files changed, 398 insertions(+), 3 deletions(-) create mode 100644 apps/web/src/hubMode.test.ts create mode 100644 apps/web/src/hubMode.ts diff --git a/apps/web/src/hubMode.test.ts b/apps/web/src/hubMode.test.ts new file mode 100644 index 000000000000..cd053e7ee98e --- /dev/null +++ b/apps/web/src/hubMode.test.ts @@ -0,0 +1,140 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +const hubConfig = { + environment: { capabilities: { repositoryIdentity: true, threadMachines: true } }, +}; +const standaloneConfig = { environment: { capabilities: { repositoryIdentity: true } } }; + +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.resetModules(); +}); + +async function loadHubMode(input: { + readonly managed: boolean; + readonly bootstrap?: Record; + readonly cachedDescriptor?: unknown; +}) { + vi.stubEnv("VITE_DEVPC_MANAGED", input.managed ? "1" : "0"); + vi.resetModules(); + const storage = new Map(); + if (input.cachedDescriptor) { + storage.set( + "t3-managed-primary-environment-descriptor-v1", + JSON.stringify(input.cachedDescriptor), + ); + } + vi.stubGlobal("window", { + __DEVPC_MANAGED_BOOTSTRAP__: input.bootstrap, + localStorage: { + getItem: (key: string) => storage.get(key) ?? null, + setItem: (key: string, value: string) => storage.set(key, value), + }, + }); + return import("./hubMode"); +} + +describe("resolveIsHubEnvironment", () => { + it("follows the loaded server config for every environment", async () => { + const { resolveIsHubEnvironment } = await loadHubMode({ managed: false }); + const managedPrimaryIsHub = () => true; + expect( + resolveIsHubEnvironment({ config: hubConfig, isPrimary: false, managedPrimaryIsHub }), + ).toBe(true); + // A loaded standalone config wins over a stale managed hint. + expect( + resolveIsHubEnvironment({ config: standaloneConfig, isPrimary: true, managedPrimaryIsHub }), + ).toBe(false); + }); + + it("falls back to the managed bootstrap only for a primary without a config", async () => { + const { resolveIsHubEnvironment } = await loadHubMode({ managed: false }); + const managedPrimaryIsHub = () => true; + expect(resolveIsHubEnvironment({ config: null, isPrimary: true, managedPrimaryIsHub })).toBe( + true, + ); + expect(resolveIsHubEnvironment({ config: null, isPrimary: false, managedPrimaryIsHub })).toBe( + false, + ); + }); +}); + +describe("readManagedPrimaryIsHub", () => { + it("reads a hub bootstrap so the first render never shows local-checkout controls", async () => { + const { readManagedPrimaryIsHub } = await loadHubMode({ + managed: true, + bootstrap: { managed: true, serverMode: "hub", state: "ready", ready: true }, + }); + expect(readManagedPrimaryIsHub()).toBe(true); + }); + + it("reads the cached environment descriptor when the bootstrap says nothing", async () => { + const { readManagedPrimaryIsHub } = await loadHubMode({ + managed: true, + bootstrap: { managed: true, state: "ready", ready: true }, + cachedDescriptor: { + environmentId: "environment-hub", + label: "Aldo", + platform: { os: "linux", arch: "x64" }, + serverVersion: "0.0.0-test", + capabilities: { repositoryIdentity: true, threadMachines: true }, + }, + }); + expect(readManagedPrimaryIsHub()).toBe(true); + }); + + it("is never a hub outside a managed build", async () => { + const { readManagedPrimaryIsHub } = await loadHubMode({ + managed: false, + bootstrap: { managed: true, serverMode: "hub" }, + }); + expect(readManagedPrimaryIsHub()).toBe(false); + }); +}); + +describe("hubNewThreadOptions", () => { + it("turns a carried-over checkout into a fresh machine checkout from the same branch", async () => { + const { hubNewThreadOptions } = await loadHubMode({ managed: false }); + expect( + hubNewThreadOptions({ + branch: "feature/login", + worktreePath: "/workspace/t/thread-1", + envMode: "local", + startFromOrigin: true, + }), + ).toEqual({ + branch: "feature/login", + worktreePath: null, + envMode: "worktree", + startFromOrigin: false, + }); + }); + + it("leaves absent options absent so existing drafts keep their context", async () => { + const { hubNewThreadOptions } = await loadHubMode({ managed: false }); + expect(hubNewThreadOptions(undefined)).toBeUndefined(); + // The caller's full option shape, with only `replace` given. + const replaceOnly: Parameters[0] & { replace?: boolean } = { + replace: true, + }; + expect(hubNewThreadOptions(replaceOnly)).toEqual({ replace: true }); + }); +}); + +describe("hub project helpers", () => { + it("uses the virtual root the hub assigns", async () => { + const { hubProjectWorkspaceRoot } = await loadHubMode({ managed: false }); + expect(hubProjectWorkspaceRoot("project-1")).toBe("/workspace/p/project-1"); + }); + + it("names a project by its repository, or as blank", async () => { + const { hubProjectLocationLabel } = await loadHubMode({ managed: false }); + expect( + hubProjectLocationLabel({ + repositoryIdentity: { locator: { remoteUrl: "https://github.com/acme/app" } }, + }), + ).toBe("https://github.com/acme/app"); + expect(hubProjectLocationLabel({ repositoryIdentity: null })).toBe("Blank project"); + }); +}); diff --git a/apps/web/src/hubMode.ts b/apps/web/src/hubMode.ts new file mode 100644 index 000000000000..8d0ab7805fb9 --- /dev/null +++ b/apps/web/src/hubMode.ts @@ -0,0 +1,138 @@ +/** + * Hub mode: the environment serves T3 without a machine of its own, and every + * thread runs on its own on-demand machine (docs/internals/thread-machines.md). + * + * Always decided per environment and at render time: a client can be + * connected to a hub and to ordinary servers at once, and which environment + * is primary changes at runtime. Before a managed primary's server config + * arrives, the managed bootstrap (or the cached environment descriptor) + * answers, so a hub never flashes the local-checkout UI while connecting. + */ +import { useAtomValue } from "@effect/atom-react"; +import type { EnvironmentId, ExecutionEnvironmentCapabilities } from "@t3tools/contracts"; +import { Atom } from "effect/unstable/reactivity"; + +import { isManagedDevPc, isManagedHubBootstrap } from "./managedDevPc"; +import { readManagedPrimaryEnvironmentDescriptor } from "./managedPrimaryEnvironment"; +import { appAtomRegistry } from "./rpc/atomRegistry"; +import { primaryEnvironmentIdAtom } from "./state/primaryEnvironment"; +import { environmentServerConfigsAtom } from "./state/server"; + +interface HubCapabilitySource { + readonly environment: { readonly capabilities: ExecutionEnvironmentCapabilities }; +} + +/** Whether a loaded server config advertises thread machines. */ +export function isHubServerConfig(config: HubCapabilitySource | null | undefined): boolean { + return config?.environment.capabilities.threadMachines === true; +} + +/** What a managed deployment says about its primary before the server config arrives. */ +export function readManagedPrimaryIsHub(): boolean { + if (!isManagedDevPc) return false; + return ( + isManagedHubBootstrap() || + readManagedPrimaryEnvironmentDescriptor()?.capabilities.threadMachines === true + ); +} + +/** + * The loaded config always wins; only a managed primary without one yet falls + * back to the bootstrap, so standalone and remote environments never guess. + */ +export function resolveIsHubEnvironment(input: { + readonly config: HubCapabilitySource | null | undefined; + readonly isPrimary: boolean; + readonly managedPrimaryIsHub: () => boolean; +}): boolean { + if (input.config) return isHubServerConfig(input.config); + return input.isPrimary && input.managedPrimaryIsHub(); +} + +const environmentIsHubAtom = Atom.family((environmentId: string) => + Atom.make((get) => + resolveIsHubEnvironment({ + config: get(environmentServerConfigsAtom).get(environmentId as EnvironmentId), + isPrimary: get(primaryEnvironmentIdAtom) === environmentId, + managedPrimaryIsHub: readManagedPrimaryIsHub, + }), + ).pipe(Atom.withLabel(`web-environment-is-hub:${environmentId}`)), +); + +const NOT_HUB_ATOM = Atom.make(false).pipe(Atom.withLabel("web-environment-is-hub:none")); + +const primaryIsHubAtom = Atom.make((get) => { + const primaryEnvironmentId = get(primaryEnvironmentIdAtom); + return primaryEnvironmentId === null + ? readManagedPrimaryIsHub() + : get(environmentIsHubAtom(primaryEnvironmentId)); +}).pipe(Atom.withLabel("web-primary-is-hub")); + +export function useIsHubEnvironment(environmentId: EnvironmentId | null | undefined): boolean { + return useAtomValue(environmentId ? environmentIsHubAtom(environmentId) : NOT_HUB_ATOM); +} + +/** Primary-scoped surfaces: settings, the workspace status, add-project defaults. */ +export function usePrimaryIsHub(): boolean { + return useAtomValue(primaryIsHubAtom); +} + +/** Non-React callers (command handlers) read the same answer from the app registry. */ +export function readIsHubEnvironment(environmentId: EnvironmentId | null | undefined): boolean { + return environmentId ? appAtomRegistry.get(environmentIsHubAtom(environmentId)) : false; +} + +/** + * Hub projects have no folder: the server gives them this virtual root and + * nothing reads it. See `projectVirtualRoot` in the runner contracts. + */ +export function hubProjectWorkspaceRoot(projectId: string): string { + return `/workspace/p/${encodeURIComponent(projectId)}`; +} + +/** + * Base ref a hub thread starts from when the user picked none: the runner + * resolves it to the repository's default branch. + */ +export const HUB_DEFAULT_BASE_REF = "HEAD"; + +/** + * New-thread options as a hub understands them: every thread gets its own + * checkout on its own machine, so a carried-over branch becomes the base of + * a fresh checkout and a worktree path never carries over. + */ +export function hubNewThreadOptions< + T extends { + readonly worktreePath?: string | null; + readonly envMode?: "local" | "worktree"; + readonly startFromOrigin?: boolean; + }, +>(options: T | undefined): T | undefined { + if (!options) return options; + return { + ...options, + ...(options.worktreePath !== undefined ? { worktreePath: null } : {}), + ...(options.envMode !== undefined ? { envMode: "worktree" as const } : {}), + ...(options.startFromOrigin !== undefined ? { startFromOrigin: false } : {}), + }; +} + +/** Where a hub project comes from, in place of its virtual root. */ +export function hubProjectLocationLabel(project: { + readonly repositoryIdentity?: + | { readonly locator: { readonly remoteUrl: string } } + | null + | undefined; +}): string { + return project.repositoryIdentity?.locator.remoteUrl ?? "Blank project"; +} + +/** Short form for lists: `owner/repo` when recorded. */ +export function hubProjectDisplayLabel(project: { + readonly repositoryIdentity?: + | { readonly displayName?: string; readonly locator: { readonly remoteUrl: string } } + | null + | undefined; +}): string { + return project.repositoryIdentity?.displayName ?? hubProjectLocationLabel(project); +} diff --git a/apps/web/src/managedDevPc.test.ts b/apps/web/src/managedDevPc.test.ts index 0a4db35dc7d8..e0d26dc7e2f7 100644 --- a/apps/web/src/managedDevPc.test.ts +++ b/apps/web/src/managedDevPc.test.ts @@ -1229,3 +1229,74 @@ describe("shared workspace browser", () => { expect(managedWorkspaceBrowserUrl()).toBeNull(); }); }); + +describe("hub bootstrap", () => { + const stubHubBootstrap = (fetch = vi.fn()) => { + vi.stubGlobal("fetch", fetch); + vi.stubGlobal("window", { + location: { origin: "https://app.example.test" }, + localStorage: memoryLocalStorage(), + __DEVPC_MANAGED_BOOTSTRAP__: { + managed: true, + state: "ready", + ready: true, + serverMode: "hub", + previewUrlTemplate: "", + threadPreviewUrlTemplate: "/_devpc/threads/{threadId}/preview/{port}", + threadBrowserUrlTemplate: "/_devpc/threads/{threadId}/browser", + }, + }); + return fetch; + }; + + it("gives each thread its own machine browser", async () => { + vi.stubEnv("VITE_DEVPC_MANAGED", "1"); + vi.resetModules(); + stubHubBootstrap(); + + const { + isManagedHubBootstrap, + isManagedWorkspaceBrowserAvailable, + managedWorkspaceBrowserUrl, + } = await import("./managedDevPc"); + + expect(isManagedHubBootstrap()).toBe(true); + expect(isManagedWorkspaceBrowserAvailable()).toBe(true); + expect(managedWorkspaceBrowserUrl("thread/1")).toBe("/_devpc/threads/thread%2F1/browser"); + // Without a thread there is no machine to show. + expect(managedWorkspaceBrowserUrl()).toBeNull(); + }); + + it("routes a thread's preview port through the per-thread template", async () => { + vi.stubEnv("VITE_DEVPC_MANAGED", "1"); + vi.resetModules(); + stubHubBootstrap(); + + const { resolveManagedPreviewUrl } = await import("./managedDevPc"); + + expect(resolveManagedPreviewUrl(5173, "/", "thread-1")).toBe( + "https://app.example.test/_devpc/threads/thread-1/preview/5173", + ); + expect(resolveManagedPreviewUrl(3000, "/docs?page=2", "thread-1")).toBe( + "https://app.example.test/_devpc/threads/thread-1/preview/3000?path=%2Fdocs%3Fpage%3D2", + ); + expect(resolveManagedPreviewUrl(3000, "/", null)).toBeNull(); + }); + + it("dispatches commands directly instead of queueing a workspace wake", async () => { + vi.stubEnv("VITE_DEVPC_MANAGED", "1"); + vi.resetModules(); + const fetch = stubHubBootstrap(); + + const { prepareManagedCommandDispatch } = await import("./managedDevPc"); + const command = { + type: "thread.turn.interrupt", + commandId: CommandId.make("command-1"), + threadId: ThreadId.make("thread-1"), + createdAt: "2026-09-26T10:00:00.000Z", + } as ClientOrchestrationCommand; + + await expect(prepareManagedCommandDispatch({ command, primary: true })).resolves.toBeNull(); + expect(fetch).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/managedDevPc.ts b/apps/web/src/managedDevPc.ts index 628570614c98..0c178bfe6123 100644 --- a/apps/web/src/managedDevPc.ts +++ b/apps/web/src/managedDevPc.ts @@ -83,10 +83,25 @@ export interface ManagedDevPcBootstrap { readonly detail?: string; /** The in-flight wake is installing a newer workspace runtime first. */ readonly runtimeUpdating?: boolean; + /** + * Hub mode: T3 is served without a machine and every thread runs on its own + * machine (docs/internals/thread-machines.md). There is no workspace to + * resume, so the workspace lifecycle surfaces stay out of the way. + */ + readonly serverMode?: "hub"; + /** Hub mode: a thread's preview, `{threadId}` and `{port}` substituted. */ + readonly threadPreviewUrlTemplate?: string; + /** Hub mode: a thread's machine browser, `{threadId}` substituted. */ + readonly threadBrowserUrlTemplate?: string; } export const isManagedDevPc = import.meta.env.VITE_DEVPC_MANAGED === "1"; +/** The managed deployment serves a hub: threads run on their own machines. */ +export function isManagedHubBootstrap(): boolean { + return isManagedDevPc && window.__DEVPC_MANAGED_BOOTSTRAP__?.serverMode === "hub"; +} + const BOOTSTRAP_PATH = "/_devpc/bootstrap"; const START_PATH = "/_devpc/workspace/start"; const WEBSOCKET_TICKET_PATH = "/_devpc/ws-ticket"; @@ -182,7 +197,8 @@ export async function prepareManagedCommandDispatch(input: { readonly command: ClientOrchestrationCommand; readonly primary: boolean; }): Promise { - if (!input.primary) return null; + // A hub is always reachable and wakes each thread's machine itself. + if (!input.primary || isManagedHubBootstrap()) return null; if (managedCommandRequiresLiveTransport(input.command)) { await requestManagedResume(`dispatch-${input.command.commandId}`); await waitForManagedCommandTransportReady(); @@ -1072,15 +1088,45 @@ export const MANAGED_WORKSPACE_BROWSER_PORT = 6080; * client with the viewer options already set, because the port's own root serves * a directory listing rather than a screen. */ -export function managedWorkspaceBrowserUrl(): string | null { +export function managedWorkspaceBrowserUrl(threadId?: string | null): string | null { if (!isManagedDevPc) return null; + // Hub mode: each thread's machine has its own browser. + const threadTemplate = window.__DEVPC_MANAGED_BOOTSTRAP__?.threadBrowserUrlTemplate; + if (threadTemplate) { + return threadId ? threadTemplate.replace("{threadId}", encodeURIComponent(threadId)) : null; + } const granted = window.__DEVPC_MANAGED_BOOTSTRAP__?.previewUrls?.[String(MANAGED_WORKSPACE_BROWSER_PORT)]; return granted ?? null; } -export function resolveManagedPreviewUrl(port: number, path: string): string | null { +/** Whether this deployment offers a workspace (or per-thread machine) browser at all. */ +export function isManagedWorkspaceBrowserAvailable(): boolean { + if (!isManagedDevPc) return false; + return Boolean( + window.__DEVPC_MANAGED_BOOTSTRAP__?.threadBrowserUrlTemplate || + window.__DEVPC_MANAGED_BOOTSTRAP__?.previewUrls?.[String(MANAGED_WORKSPACE_BROWSER_PORT)], + ); +} + +export function resolveManagedPreviewUrl( + port: number, + path: string, + threadId?: string | null, +): string | null { if (!isManagedDevPc) return null; + const threadTemplate = window.__DEVPC_MANAGED_BOOTSTRAP__?.threadPreviewUrlTemplate; + if (threadTemplate) { + if (!threadId) return null; + const resolved = new URL( + threadTemplate + .replace("{threadId}", encodeURIComponent(threadId)) + .replace("{port}", String(port)), + window.location.origin, + ); + if (path !== "/" && path !== "") resolved.searchParams.set("path", path); + return resolved.toString(); + } const template = window.__DEVPC_MANAGED_BOOTSTRAP__?.previewUrlTemplate; const granted = window.__DEVPC_MANAGED_BOOTSTRAP__?.previewUrls?.[String(port)]; if (granted) { From a43db7c1c57553e7909a9070d5638a7cc0ea35d3 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 01:24:35 +0000 Subject: [PATCH 41/58] feat(web): show each hub thread's machine state The hub projects a thread's machine state onto its shell without waking it. One view model (threadMachine.ts) turns it into what the client shows: a status pill in the chat header (starting, running, asleep, failed), the sidebar status (a starting machine reads as working, a failed one as failed), and the building blocks the composer uses for its asleep and failed notices. thread-machine.* activities stay out of the work log unless they report an error. A hub has no folder to open, so the header drops Open In there. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../web/src/components/ManagedDevPcStatus.tsx | 2 +- apps/web/src/components/Sidebar.logic.test.ts | 58 ++++++++ apps/web/src/components/Sidebar.logic.ts | 12 +- .../src/components/ThreadMachineStatus.tsx | 137 ++++++++++++++++++ apps/web/src/components/chat/ChatHeader.tsx | 24 ++- apps/web/src/session-logic.test.ts | 28 ++++ apps/web/src/session-logic.ts | 2 + apps/web/src/threadMachine.test.ts | 98 +++++++++++++ apps/web/src/threadMachine.ts | 92 ++++++++++++ 9 files changed, 444 insertions(+), 9 deletions(-) create mode 100644 apps/web/src/components/ThreadMachineStatus.tsx create mode 100644 apps/web/src/threadMachine.test.ts create mode 100644 apps/web/src/threadMachine.ts diff --git a/apps/web/src/components/ManagedDevPcStatus.tsx b/apps/web/src/components/ManagedDevPcStatus.tsx index a1ed571a2662..15c4b3c1b73f 100644 --- a/apps/web/src/components/ManagedDevPcStatus.tsx +++ b/apps/web/src/components/ManagedDevPcStatus.tsx @@ -411,7 +411,7 @@ export function autoPauseDescription( return "Pauses after 15 minutes of inactivity."; } -function StatusDot({ status }: { status: ManagedDevPcDisplayStatus }) { +export function StatusDot({ status }: { status: ManagedDevPcDisplayStatus }) { const transitional = ["starting", "restarting", "pausing", "restoring"].includes(status); const inactive = ["paused", "stopped"].includes(status); const color = diff --git a/apps/web/src/components/Sidebar.logic.test.ts b/apps/web/src/components/Sidebar.logic.test.ts index dbf8fcf78532..00b5a0252857 100644 --- a/apps/web/src/components/Sidebar.logic.test.ts +++ b/apps/web/src/components/Sidebar.logic.test.ts @@ -724,6 +724,52 @@ describe("resolveSidebarThreadStatus", () => { it("defaults to ready with no session", () => { expect(resolveSidebarThreadStatus({ ...idle, session: null })).toBe("ready"); }); + + describe("hub thread machines", () => { + const machine = (state: "starting" | "preparing" | "paused" | "running" | "failed") => ({ + state, + detail: state === "failed" ? "Quota exceeded" : null, + updatedAt: "2026-03-09T10:00:00.000Z", + }); + + it("reads a preparing or starting machine as working", () => { + expect( + resolveSidebarThreadStatus({ ...idle, session: null, machine: machine("starting") }), + ).toBe("working"); + expect( + resolveSidebarThreadStatus({ ...idle, session: null, machine: machine("preparing") }), + ).toBe("working"); + }); + + it("reads a failed machine as failed", () => { + expect( + resolveSidebarThreadStatus({ ...idle, session: null, machine: machine("failed") }), + ).toBe("failed"); + }); + + it("says nothing extra for a sleeping or running machine", () => { + expect( + resolveSidebarThreadStatus({ ...idle, session: null, machine: machine("paused") }), + ).toBe("ready"); + expect( + resolveSidebarThreadStatus({ ...idle, session: null, machine: machine("running") }), + ).toBe("ready"); + }); + + it("keeps approvals, input and live turns ahead of the machine", () => { + expect( + resolveSidebarThreadStatus({ + ...idle, + hasPendingApprovals: true, + session: null, + machine: machine("failed"), + }), + ).toBe("approval"); + expect(resolveSidebarThreadStatus({ ...idle, session, machine: machine("failed") })).toBe( + "working", + ); + }); + }); }); describe("searchSidebarThreadsByTitle", () => { @@ -1147,6 +1193,18 @@ describe("resolveThreadStatusPill", () => { }, }; + it("shows a hub machine that is starting as connecting", () => { + expect( + resolveThreadStatusPill({ + thread: { + ...baseThread, + session: null, + machine: { state: "starting", detail: null, updatedAt: "2026-03-09T10:00:00.000Z" }, + }, + }), + ).toMatchObject({ label: "Connecting", pulse: true }); + }); + it("shows pending approval before all other statuses", () => { expect( resolveThreadStatusPill({ diff --git a/apps/web/src/components/Sidebar.logic.ts b/apps/web/src/components/Sidebar.logic.ts index af2896355359..773a0d57bc6a 100644 --- a/apps/web/src/components/Sidebar.logic.ts +++ b/apps/web/src/components/Sidebar.logic.ts @@ -10,6 +10,7 @@ import { type ThreadSortInput, } from "../lib/threadSort"; import type { SidebarThreadSummary, Thread } from "../types"; +import { threadMachineSidebarStatus } from "../threadMachine"; import type { ThreadRouteTarget } from "../threadRoutes"; import { cn } from "../lib/utils"; import { isLatestTurnSettled } from "../session-logic"; @@ -160,6 +161,7 @@ type ThreadStatusInput = Pick< | "latestTurn" | "session" | "backgroundLiveness" + | "machine" > & { lastVisitedAt?: string | undefined; }; @@ -469,7 +471,7 @@ export type SidebarThreadStatus = type SidebarThreadStatusInput = Pick< SidebarThreadSummary, - "hasPendingApprovals" | "hasPendingUserInput" | "session" | "backgroundLiveness" + "hasPendingApprovals" | "hasPendingUserInput" | "session" | "backgroundLiveness" | "machine" >; export function resolveSidebarThreadStatus(thread: SidebarThreadStatusInput): SidebarThreadStatus { @@ -492,6 +494,9 @@ export function resolveSidebarThreadStatus(thread: SidebarThreadStatusInput): Si if (thread.session?.status === "error") { return "failed"; } + // Hub threads: a machine starting reads as work in motion, a failed one as failed. + const machineStatus = threadMachineSidebarStatus(thread.machine); + if (machineStatus !== null) return machineStatus; // Background work outlives the turn: fleets read as working; monitoring // only when watch loops are the sole live work. if (thread.backgroundLiveness === "working") { @@ -714,7 +719,10 @@ export function resolveThreadStatusPill(input: { }; } - if (thread.session?.status === "starting") { + if ( + thread.session?.status === "starting" || + threadMachineSidebarStatus(thread.machine) === "working" + ) { return { label: "Connecting", colorClass: "text-sky-600 dark:text-sky-300/80", diff --git a/apps/web/src/components/ThreadMachineStatus.tsx b/apps/web/src/components/ThreadMachineStatus.tsx new file mode 100644 index 000000000000..7d6b1ea8cc58 --- /dev/null +++ b/apps/web/src/components/ThreadMachineStatus.tsx @@ -0,0 +1,137 @@ +import type { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { CloudIcon, MoonIcon, TriangleAlertIcon } from "lucide-react"; +import { useMemo } from "react"; + +import type { ManagedDevPcDisplayStatus } from "../managedDevPc"; +import { useThreadShell } from "../state/entities"; +import { + deriveThreadMachineView, + type ThreadMachinePhase, + type ThreadMachineView, +} from "../threadMachine"; +import type { ComposerBannerStackItem } from "./chat/ComposerBannerStack"; +import { StatusDot } from "./ManagedDevPcStatus"; +import { Button } from "./ui/button"; +import { Tooltip, TooltipPopup, TooltipTrigger } from "./ui/tooltip"; + +const DOT_STATUS: Record = { + transitional: "starting", + running: "running", + asleep: "paused", + failed: "attention", +}; + +/** The thread machine's state in the chat header. Renders nothing off a hub. */ +export function ThreadMachineStatusPill(props: { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; +}) { + const threadRef = useMemo( + () => scopeThreadRef(props.environmentId, props.threadId), + [props.environmentId, props.threadId], + ); + const machine = useThreadShell(threadRef)?.machine; + const view = deriveThreadMachineView(machine); + if (!view) return null; + return ( + + + } + > + + {view.label} + + + {view.description} + + + ); +} + +/** + * The composer notice for a machine the user has to know about: asleep (the + * next send wakes it) or failed (with a retry when one is possible). + */ +export function buildThreadMachineBannerItem(input: { + readonly threadKey: string; + readonly view: ThreadMachineView | null; + readonly onRetry: (() => void) | null; + readonly retrying: boolean; + readonly onDismiss: () => void; +}): ComposerBannerStackItem | null { + const { view } = input; + if (!view) return null; + if (view.phase === "asleep") { + return { + id: `thread-machine:${input.threadKey}:${view.state}`, + variant: "info", + priority: "notice", + icon: , + title: view.description, + dismissLabel: "Dismiss machine status", + onDismiss: input.onDismiss, + }; + } + if (view.phase === "failed") { + return { + id: `thread-machine:${input.threadKey}:failed:${view.updatedAt}`, + variant: "error", + icon: , + title: "This thread's machine failed", + description: view.detail ?? "Send a message to try again.", + ...(input.onRetry + ? { + actions: ( + + ), + } + : {}), + }; + } + return null; +} + +/** Stands in for machine-backed panels (files) while the machine sleeps. */ +export function ThreadMachineAsleepPanel({ view }: { readonly view: ThreadMachineView }) { + return ( +
+ +

+ {view.phase === "failed" ? "Machine unavailable" : "Machine asleep"} +

+

+ {view.phase === "failed" + ? (view.detail ?? "This thread's machine could not start.") + : "Files live on this thread's machine. Send a message to wake it."} +

+
+ ); +} + +/** Where a new hub thread runs, in place of the local/worktree picker. */ +export function HubRunContextLabel() { + return ( + + + + New machine + + + ); +} diff --git a/apps/web/src/components/chat/ChatHeader.tsx b/apps/web/src/components/chat/ChatHeader.tsx index eaea871f434a..6e5b97d9e568 100644 --- a/apps/web/src/components/chat/ChatHeader.tsx +++ b/apps/web/src/components/chat/ChatHeader.tsx @@ -40,6 +40,8 @@ import { useThreadActionMenu } from "~/hooks/useThreadActionMenu"; import { threadEnvironment } from "../../state/threads"; import { useAtomCommand } from "../../state/use-atom-command"; import { ProjectFavicon } from "../ProjectFavicon"; +import { ThreadMachineStatusPill } from "../ThreadMachineStatus"; +import { useIsHubEnvironment } from "~/hubMode"; import { WorkspaceBreadcrumb, WorkspaceBreadcrumbItem, @@ -145,12 +147,16 @@ export const ChatHeader = memo(function ChatHeader({ activeProjectScripts ? activeProjectCwd : null, ); const remoteOpenState = useRemoteOpenState(activeThreadEnvironmentId); - const showOpenInPicker = shouldShowOpenInPicker({ - activeProjectName, - activeThreadEnvironmentId, - primaryEnvironmentId, - remoteOpenMode: remoteOpenState.mode, - }); + // A hub has no folder to open; each thread's checkout lives on its machine. + const isHub = useIsHubEnvironment(activeThreadEnvironmentId); + const showOpenInPicker = + !isHub && + shouldShowOpenInPicker({ + activeProjectName, + activeThreadEnvironmentId, + primaryEnvironmentId, + remoteOpenMode: remoteOpenState.mode, + }); const activeThreadRef = useMemo( () => scopeThreadRef(activeThreadEnvironmentId, activeThreadId), [activeThreadEnvironmentId, activeThreadId], @@ -440,6 +446,12 @@ export const ChatHeader = memo(function ChatHeader({ rightPanelOpen ? "pr-0" : "pr-16", )} > + {isServerThread ? ( + + ) : null} {activeProjectName && ( { }); describe("deriveWorkLogEntries", () => { + it("hides thread-machine lifecycle rows unless they report an error", () => { + const activities = [ + makeActivity({ + id: "starting", + kind: "thread-machine.starting", + summary: "Starting machine", + tone: "info", + sequence: 0, + }), + makeActivity({ + id: "state", + kind: "thread-machine.state", + summary: "Machine running", + tone: "info", + sequence: 1, + }), + makeActivity({ + id: "failed", + kind: "thread-machine.failed", + summary: "Thread machine is unavailable", + tone: "error", + sequence: 2, + }), + makeActivity({ id: "tool", kind: "tool.completed", summary: "Ran tests", sequence: 3 }), + ]; + expect(deriveWorkLogEntries(activities).map((entry) => entry.id)).toEqual(["failed", "tool"]); + }); + it("keeps the latest task progress without emitting plan-update log entries", () => { const activities = [ makeActivity({ id: "before", kind: "tool.completed", summary: "Read files", sequence: 0 }), diff --git a/apps/web/src/session-logic.ts b/apps/web/src/session-logic.ts index 8240cd783469..0bd498f98fdb 100644 --- a/apps/web/src/session-logic.ts +++ b/apps/web/src/session-logic.ts @@ -3,6 +3,7 @@ import * as Arr from "effect/Array"; import * as Schema from "effect/Schema"; import { isBackgroundTaskActivity } from "@t3tools/client-runtime/state/subagentRuntime"; import { isWorktreeSetupActivity } from "@t3tools/client-runtime/work-log/presentation"; +import { isThreadMachineActivity } from "./threadMachine"; import { ApprovalRequestId, isToolLifecycleItemType, @@ -836,6 +837,7 @@ export function deriveWorkLogEntries( const entries: DerivedWorkLogEntry[] = []; for (const activity of ordered) { if (activity.tone !== "error" && isWorktreeSetupActivity(activity.kind)) continue; + if (activity.tone !== "error" && isThreadMachineActivity(activity.kind)) continue; if (activity.kind === "tool.started") continue; // Agent task.started rows are CTA seeds: they carry the true spawn turn, // which is the batch key (completions of background subagents arrive diff --git a/apps/web/src/threadMachine.test.ts b/apps/web/src/threadMachine.test.ts new file mode 100644 index 000000000000..64b97974aab4 --- /dev/null +++ b/apps/web/src/threadMachine.test.ts @@ -0,0 +1,98 @@ +import type { ThreadMachineStatus } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { + deriveThreadMachineView, + isThreadMachineActivity, + threadMachineSidebarStatus, +} from "./threadMachine"; + +const UPDATED_AT = "2026-09-26T10:00:00.000Z"; + +function machine( + state: ThreadMachineStatus["state"], + detail: string | null = null, +): ThreadMachineStatus { + return { state, detail, updatedAt: UPDATED_AT }; +} + +describe("deriveThreadMachineView", () => { + it("has nothing to show without a machine", () => { + expect(deriveThreadMachineView(null)).toBeNull(); + expect(deriveThreadMachineView(undefined)).toBeNull(); + }); + + it("treats preparing and starting as transitional and surfaces platform progress", () => { + expect(deriveThreadMachineView(machine("preparing"))).toMatchObject({ + phase: "transitional", + label: "Preparing", + detail: null, + }); + expect(deriveThreadMachineView(machine("starting", " Cloning repository "))).toMatchObject({ + phase: "transitional", + label: "Starting", + description: "Cloning repository", + detail: "Cloning repository", + }); + }); + + it("reads paused and saved machines as asleep until the next send", () => { + for (const state of ["paused", "saved"] as const) { + expect(deriveThreadMachineView(machine(state))).toMatchObject({ + phase: "asleep", + label: "Asleep", + description: "Machine asleep — wakes when you send", + }); + } + }); + + it("reads a missing machine as one that starts on send", () => { + expect(deriveThreadMachineView(machine("none"))).toMatchObject({ + phase: "asleep", + description: "No machine yet — one starts when you send", + }); + }); + + it("keeps the platform's error for a failed machine", () => { + expect(deriveThreadMachineView(machine("failed", "Out of capacity"))).toMatchObject({ + phase: "failed", + label: "Failed", + detail: "Out of capacity", + description: "Out of capacity", + updatedAt: UPDATED_AT, + }); + expect(deriveThreadMachineView(machine("failed", " "))).toMatchObject({ + detail: null, + description: "This thread's machine could not start", + }); + }); + + it("reads a running machine as running", () => { + expect(deriveThreadMachineView(machine("running"))).toMatchObject({ + phase: "running", + label: "Running", + }); + }); +}); + +describe("isThreadMachineActivity", () => { + it("matches every thread-machine lifecycle kind and nothing else", () => { + expect(isThreadMachineActivity("thread-machine.state")).toBe(true); + expect(isThreadMachineActivity("thread-machine.starting")).toBe(true); + expect(isThreadMachineActivity("thread-machine.checkout.preparing")).toBe(true); + expect(isThreadMachineActivity("thread-machine.failed")).toBe(true); + expect(isThreadMachineActivity("tool.completed")).toBe(false); + expect(isThreadMachineActivity("setup-script.started")).toBe(false); + }); +}); + +describe("threadMachineSidebarStatus", () => { + it("maps motion to working, failure to failed, and rest to nothing", () => { + expect(threadMachineSidebarStatus(machine("starting"))).toBe("working"); + expect(threadMachineSidebarStatus(machine("preparing"))).toBe("working"); + expect(threadMachineSidebarStatus(machine("failed"))).toBe("failed"); + expect(threadMachineSidebarStatus(machine("paused"))).toBeNull(); + expect(threadMachineSidebarStatus(machine("running"))).toBeNull(); + expect(threadMachineSidebarStatus(null)).toBeNull(); + }); +}); diff --git a/apps/web/src/threadMachine.ts b/apps/web/src/threadMachine.ts new file mode 100644 index 000000000000..d9a000febef1 --- /dev/null +++ b/apps/web/src/threadMachine.ts @@ -0,0 +1,92 @@ +/** + * Presentation of a hub thread's machine (docs/internals/thread-machines.md). + * + * The hub projects the machine's latest known state onto the thread shell + * without waking it; everything a client shows about the machine derives from + * that one field, so the header pill, the timeline, the composer banner and + * the sidebar cannot disagree. + */ +import type { ThreadMachineStatus } from "@t3tools/contracts"; + +export type ThreadMachinePhase = "transitional" | "running" | "asleep" | "failed"; + +export interface ThreadMachineView { + readonly state: ThreadMachineStatus["state"]; + readonly phase: ThreadMachinePhase; + /** Short label for the header pill. */ + readonly label: string; + /** One sentence for tooltips and banners. */ + readonly description: string; + /** What the platform reported (progress or error), when it said anything. */ + readonly detail: string | null; + readonly updatedAt: string; +} + +function trimmedDetail(detail: string | null): string | null { + const trimmed = detail?.trim() ?? ""; + return trimmed.length > 0 ? trimmed : null; +} + +export function deriveThreadMachineView( + machine: ThreadMachineStatus | null | undefined, +): ThreadMachineView | null { + if (!machine) return null; + const detail = trimmedDetail(machine.detail); + const base = { state: machine.state, detail, updatedAt: machine.updatedAt } as const; + switch (machine.state) { + case "preparing": + return { + ...base, + phase: "transitional", + label: "Preparing", + description: detail ?? "Preparing this thread's machine", + }; + case "starting": + return { + ...base, + phase: "transitional", + label: "Starting", + description: detail ?? "Starting this thread's machine", + }; + case "running": + return { ...base, phase: "running", label: "Running", description: "Machine running" }; + case "paused": + case "saved": + return { + ...base, + phase: "asleep", + label: "Asleep", + description: "Machine asleep — wakes when you send", + }; + case "none": + return { + ...base, + phase: "asleep", + label: "No machine", + description: "No machine yet — one starts when you send", + }; + case "failed": + return { + ...base, + phase: "failed", + label: "Failed", + description: detail ?? "This thread's machine could not start", + }; + } +} + +/** Machine lifecycle activities the hub records on a thread. */ +export function isThreadMachineActivity(kind: string): boolean { + return kind.startsWith("thread-machine."); +} + +/** + * The sidebar reads a machine in motion as working and a failed one as + * failed; a sleeping or running machine says nothing on its own. + */ +export function threadMachineSidebarStatus( + machine: ThreadMachineStatus | null | undefined, +): "working" | "failed" | null { + const phase = deriveThreadMachineView(machine)?.phase; + return phase === "transitional" ? "working" : phase === "failed" ? "failed" : null; +} From eaa6b7066573d2fa4941d127a06ca7a61faf8644 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 01:25:06 +0000 Subject: [PATCH 42/58] feat(web): run hub threads on their own machines On a hub every thread gets a fresh checkout on its own machine, so there is no local/worktree choice to make. useHubThreadContext tells ChatView where hub threads differ, and ChatView only swaps values at those points: - A draft shows "New machine" and a base branch listed from the repository; with none picked it starts from the default branch (the "HEAD" base-ref sentinel). Drafts never ask the hub for git status, files, terminals or scripts, since their machine does not exist yet. New-thread defaults and "New thread on " become a fresh checkout from that branch; start-from-origin and PR checkout go away. - The timeline's waking row covers machine start with the platform's progress detail. The composer notes an asleep machine (wakes on send) and a failed one, with Retry re-driving the failed turn. Explicit wake waits for the threadMachines.wake RPC and is hidden until then. - Checkout-bound surfaces adapt: files show "Machine asleep" while the machine sleeps, the diff panel opens on the latest captured turn, opening a terminal says it is waking the machine, previews and the machine browser use per-thread URLs, and provider sign-in opens the session's workspace browser with a "Starting a machine" stage. - Copy path, t3.json reads, the new-thread workspace settings, the managed workspace status and /settings/workspace are hidden on a hub. Project settings show the repository instead of the virtual root, and usage tolerates a summary without a workspace machine. Standalone and persistent-workspace behavior is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/browser/browserTargetResolver.test.ts | 19 +- apps/web/src/browser/browserTargetResolver.ts | 33 +++- apps/web/src/components/BranchToolbar.tsx | 12 +- .../BranchToolbarBranchSelector.tsx | 25 ++- apps/web/src/components/ChatView.tsx | 108 +++++++---- apps/web/src/components/DiffPanel.tsx | 2 + .../src/components/ManagedDevPcAccount.tsx | 5 +- apps/web/src/components/RightPanelTabs.tsx | 6 +- apps/web/src/components/Sidebar.tsx | 2 + .../src/components/WorkspaceBrowserPanel.tsx | 6 +- .../components/hub/hubAuthConnector.test.ts | 50 +++++ .../src/components/hub/hubAuthConnector.ts | 27 +++ .../components/hub/threadMachineActions.ts | 20 ++ .../src/components/hub/useHubDiffDefaults.ts | 31 ++++ .../hub/useHubThreadContext.test.ts | 94 ++++++++++ .../src/components/hub/useHubThreadContext.ts | 172 ++++++++++++++++++ .../preview/PreviewAutomationHosts.tsx | 10 +- .../src/components/preview/PreviewView.tsx | 6 +- .../components/preview/openDiscoveredPort.ts | 6 +- .../settings/AuthConnectorDialog.tsx | 13 +- .../settings/ProjectSettingsPanel.tsx | 23 ++- .../components/settings/SettingsPanels.tsx | 7 +- .../settings/SettingsSidebarNav.tsx | 11 +- .../settings/settingsSearch.test.ts | 19 ++ .../src/components/settings/settingsSearch.ts | 9 +- .../useAvailableSettingsSearchItems.ts | 4 + .../components/threadActionMenu.logic.test.ts | 10 + .../src/components/threadActionMenu.logic.ts | 6 +- apps/web/src/hooks/useHandleNewThread.ts | 6 +- apps/web/src/hooks/useT3ProjectFileScripts.ts | 5 +- apps/web/src/hooks/useThreadActionMenu.ts | 2 + apps/web/src/routes/settings.workspace.tsx | 9 +- apps/web/src/state/aldoWorkspaceUsage.test.ts | 5 + apps/web/src/state/aldoWorkspaceUsage.ts | 5 +- 34 files changed, 680 insertions(+), 88 deletions(-) create mode 100644 apps/web/src/components/hub/hubAuthConnector.test.ts create mode 100644 apps/web/src/components/hub/hubAuthConnector.ts create mode 100644 apps/web/src/components/hub/threadMachineActions.ts create mode 100644 apps/web/src/components/hub/useHubDiffDefaults.ts create mode 100644 apps/web/src/components/hub/useHubThreadContext.test.ts create mode 100644 apps/web/src/components/hub/useHubThreadContext.ts diff --git a/apps/web/src/browser/browserTargetResolver.test.ts b/apps/web/src/browser/browserTargetResolver.test.ts index 910956f82a16..363e69d94785 100644 --- a/apps/web/src/browser/browserTargetResolver.test.ts +++ b/apps/web/src/browser/browserTargetResolver.test.ts @@ -31,7 +31,24 @@ describe("browser target resolver", () => { resolutionKind: "direct", environmentId: "environment-1", }); - expect(resolveManagedPreviewUrl).toHaveBeenCalledWith(5173, "/dashboard"); + expect(resolveManagedPreviewUrl).toHaveBeenCalledWith(5173, "/dashboard", undefined); + }); + + it("names the thread whose machine serves the port on a hub", async () => { + readPreparedConnection.mockReturnValue({ httpBaseUrl: "https://code.example.test" }); + resolveManagedPreviewUrl.mockReturnValue( + "https://code.example.test/_devpc/threads/thread-1/preview/5173", + ); + const { resolveBrowserNavigationTarget } = await import("./browserTargetResolver"); + + expect( + resolveBrowserNavigationTarget( + EnvironmentId.make("environment-1"), + { kind: "url", url: "http://localhost:5173/" }, + "thread-1", + ).resolvedUrl, + ).toBe("https://code.example.test/_devpc/threads/thread-1/preview/5173"); + expect(resolveManagedPreviewUrl).toHaveBeenCalledWith(5173, "/", "thread-1"); }); it("maps environment ports onto a private network host", async () => { diff --git a/apps/web/src/browser/browserTargetResolver.ts b/apps/web/src/browser/browserTargetResolver.ts index c2e8520cd57e..24f50d91bdaf 100644 --- a/apps/web/src/browser/browserTargetResolver.ts +++ b/apps/web/src/browser/browserTargetResolver.ts @@ -170,10 +170,11 @@ const resolveEnvironmentPortTarget = ( environmentUrl: URL, requestedUrl?: string, sourceUrl?: URL, + threadId?: string, ): PreviewUrlResolution => { const protocol = target.protocol ?? "http"; const path = target.path?.startsWith("/") ? target.path : `/${target.path ?? ""}`; - const managedPreviewUrl = resolveManagedPreviewUrl(target.port, path); + const managedPreviewUrl = resolveManagedPreviewUrl(target.port, path, threadId); if (managedPreviewUrl) { return { requestedUrl: requestedUrl ?? `${protocol}://localhost:${target.port}${path}`, @@ -212,9 +213,14 @@ const resolveEnvironmentPortTarget = ( }; }; +/** + * `threadId` names the thread whose machine serves the port on a hub, where + * every thread has its own machine; elsewhere the environment serves it. + */ export function resolveBrowserNavigationTarget( environmentId: EnvironmentId, target: BrowserNavigationTarget, + threadId?: string, ): PreviewUrlResolution { if (target.kind === "url") { let parsed: URL | null = null; @@ -238,6 +244,7 @@ export function resolveBrowserNavigationTarget( environmentUrl, target.url, parsed, + threadId, ); } } @@ -248,16 +255,28 @@ export function resolveBrowserNavigationTarget( environmentId, }; } - return resolveEnvironmentPortTarget(environmentId, target, readEnvironmentUrl(environmentId)); + return resolveEnvironmentPortTarget( + environmentId, + target, + readEnvironmentUrl(environmentId), + undefined, + undefined, + threadId, + ); } -export function resolveDiscoveredServerUrl(environmentId: EnvironmentId, rawUrl: string): string { +export function resolveDiscoveredServerUrl( + environmentId: EnvironmentId, + rawUrl: string, + threadId?: string, +): string { try { const normalizedUrl = normalizePreviewUrl(rawUrl); - return resolveBrowserNavigationTarget(environmentId, { - kind: "url", - url: normalizedUrl, - }).resolvedUrl; + return resolveBrowserNavigationTarget( + environmentId, + { kind: "url", url: normalizedUrl }, + threadId, + ).resolvedUrl; } catch { return rawUrl; } diff --git a/apps/web/src/components/BranchToolbar.tsx b/apps/web/src/components/BranchToolbar.tsx index b0b1440587ea..62b8ad4d7be3 100644 --- a/apps/web/src/components/BranchToolbar.tsx +++ b/apps/web/src/components/BranchToolbar.tsx @@ -41,6 +41,8 @@ import { } from "./ui/menu"; import { Separator } from "./ui/separator"; import { ComposerSurface } from "./chat/ComposerSurface"; +import { HubRunContextLabel } from "./ThreadMachineStatus"; +import { useIsHubEnvironment } from "../hubMode"; interface BranchToolbarProps { environmentId: EnvironmentId; @@ -417,7 +419,10 @@ export const BranchToolbar = memo(function BranchToolbar({ hasServerThread: serverThread !== null, draftThreadEnvMode: draftThread?.envMode, }); - const envModeLocked = envLocked || (serverThread !== null && activeWorktreePath !== null); + // Every hub thread runs in a fresh checkout on its own machine: no choice to make. + const isHub = useIsHubEnvironment(environmentId); + const envModeLocked = + envLocked || (serverThread !== null && activeWorktreePath !== null) || isHub; // "Previous worktree" hops a draft into the most recently active worktree // of this project — the "keep going where I just was" follow-up flow. Only @@ -473,7 +478,7 @@ export const BranchToolbar = memo(function BranchToolbar({ ref={setStripElement} data-compact={labelsOverflow ? "" : undefined} > - {isMobile && showGitControls ? ( + {isMobile && showGitControls && !isHub ? ( )} - {showGitControls ? ( + {isHub && serverThread === null ? : null} + {showGitControls && !isHub ? ( - {isSelectingWorktreeBase ? ( + {isSelectingWorktreeBase && !isHub ? ( { startNewThreadForProject(activeProjectRef, handleNewThread); }, [activeProjectRef, handleNewThread]); @@ -2924,12 +2933,13 @@ function ChatViewContent(props: ChatViewProps) { return byUserMessageId; }, [inferredCheckpointTurnCountByTurnId, timelineEntries, turnDiffSummaryByAssistantMessageId]); - const gitCwd = activeProject - ? projectScriptCwd({ - project: { cwd: activeProject.workspaceRoot }, - worktreePath: activeThread?.worktreePath ?? null, - }) - : null; + const gitCwd = + activeProject && !hubThread.draftWithoutMachine + ? projectScriptCwd({ + project: { cwd: activeProject.workspaceRoot }, + worktreePath: activeThread?.worktreePath ?? null, + }) + : null; const gitStatusCwd = activeThread?.worktreePath ?? gitCwd; const gitStatusQuery = useEnvironmentQuery( gitStatusCwd === null @@ -3066,10 +3076,11 @@ function ChatViewContent(props: ChatViewProps) { const activeTerminalLaunchContext = terminalUiLaunchContext?.threadId === activeThreadId ? terminalUiLaunchContext : null; // Default true while loading to avoid toolbar flicker. - const isGitRepo = gitStatusQuery.data?.isRepo ?? true; + const isGitRepo = hubThread.isGitRepoOverride ?? gitStatusQuery.data?.isRepo ?? true; const showComposerContextStrip = shouldShowComposerContextStrip({ hasActiveProject: activeProject !== null, - isGitRepo, + // A hub draft always says where it will run, repository or not. + isGitRepo: isGitRepo || hubThread.draftWithoutMachine, showEnvironmentIndicator: showComposerEnvironmentIndicator, }); const initialDiffPanelGitScope = @@ -3273,13 +3284,14 @@ function ChatViewContent(props: ChatViewProps) { if (!activeThreadRef) return; const nextOpen = !terminalUiState.terminalOpen; if (nextOpen && terminalUiState.terminalIds.length === 0) { - if (!activeThreadId || !activeProject) { + if (!activeThreadId || !activeProject || !hubThread.workspaceAvailable) { return; } const cwdForOpen = gitCwd ?? activeProject.workspaceRoot; if (!cwdForOpen) { return; } + hubThread.announceMachineWake(); const terminalId = nextTerminalId(allocatableActiveTerminalIds); storeEnsureTerminal(activeThreadRef, terminalId, { open: true }); void openTerminal({ @@ -3306,6 +3318,8 @@ function ChatViewContent(props: ChatViewProps) { allocatableActiveTerminalIds, environmentId, gitCwd, + hubThread.announceMachineWake, + hubThread.workspaceAvailable, openTerminal, setTerminalOpen, storeEnsureTerminal, @@ -3737,7 +3751,7 @@ function ChatViewContent(props: ChatViewProps) { * reached from a browser, so a preview surface can never paint there. */ const createWorkspaceBrowserSurface = useCallback(() => { - if (!activeThreadRef || !managedWorkspaceBrowserUrl()) return; + if (!activeThreadRef || !managedWorkspaceBrowserUrl(activeThreadRef.threadId)) return; useRightPanelStore.getState().open(activeThreadRef, "workspaceBrowser"); }, [activeThreadRef]); const addDiffSurface = useCallback(() => { @@ -3826,6 +3840,7 @@ function ChatViewContent(props: ChatViewProps) { }, [activeThreadRef]); const addTerminalSurface = useCallback(() => { if (!activeThreadRef || !activeThreadId || !activeProject) return; + hubThread.announceMachineWake(); const cwd = gitCwd ?? activeProject.workspaceRoot; const terminalId = nextTerminalId(allocatableActiveTerminalIds); useRightPanelStore.getState().openTerminal(activeThreadRef, terminalId); @@ -3850,6 +3865,7 @@ function ChatViewContent(props: ChatViewProps) { activeThreadWorktreePath, allocatableActiveTerminalIds, gitCwd, + hubThread.announceMachineWake, openTerminal, ]); const splitPanelTerminal = useCallback( @@ -4640,19 +4656,23 @@ function ChatViewContent(props: ChatViewProps) { activeThread.worktreePath === null && !envLocked, ); - const envMode: DraftThreadEnvMode = canOverrideServerThreadEnvMode - ? (pendingServerThreadEnvMode ?? draftThread?.envMode ?? derivedEnvMode) - : derivedEnvMode; + const envMode: DraftThreadEnvMode = + hubThread.forcedEnvMode ?? + (canOverrideServerThreadEnvMode + ? (pendingServerThreadEnvMode ?? draftThread?.envMode ?? derivedEnvMode) + : derivedEnvMode); const activeThreadBranch = canOverrideServerThreadEnvMode && pendingServerThreadBranch !== undefined ? pendingServerThreadBranch : (activeThread?.branch ?? null); - const startFromOrigin = isLocalDraftThread - ? (draftThread?.startFromOrigin ?? false) - : canOverrideServerThreadEnvMode - ? (pendingServerThreadStartFromOriginByThreadId[activeThread?.id ?? ""] ?? - primaryServerSettings.newWorktreesStartFromOrigin) - : false; + const startFromOrigin = hubThread.hub + ? false + : isLocalDraftThread + ? (draftThread?.startFromOrigin ?? false) + : canOverrideServerThreadEnvMode + ? (pendingServerThreadStartFromOriginByThreadId[activeThread?.id ?? ""] ?? + primaryServerSettings.newWorktreesStartFromOrigin) + : false; const sendEnvMode = resolveSendEnvMode({ requestedEnvMode: envMode, isGitRepo, @@ -5291,9 +5311,11 @@ function ChatViewContent(props: ChatViewProps) { resumeCompactionBannerItem === null ? [] : [resumeCompactionBannerItem]; const wokeThreadItems = wokeThreadBannerItem === null ? [] : [wokeThreadBannerItem]; const parkedThreadItems = parkedThreadBannerItem === null ? [] : [parkedThreadBannerItem]; + const threadMachineItems = hubThread.machineBannerItem ? [hubThread.machineBannerItem] : []; if (!localCheckoutBranchMismatch || !showBranchMismatchBanner || !activeBranchMismatchKey) { return [ ...systemComposerBannerItems, + ...threadMachineItems, ...backgroundLivenessItems, ...resumeCompactionItems, ...wokeThreadItems, @@ -5302,6 +5324,7 @@ function ChatViewContent(props: ChatViewProps) { } return [ ...systemComposerBannerItems, + ...threadMachineItems, ...backgroundLivenessItems, ...resumeCompactionItems, ...wokeThreadItems, @@ -5350,6 +5373,7 @@ function ChatViewContent(props: ChatViewProps) { activeBranchMismatchKey, backgroundLivenessBannerItem, handleRestoreThreadBranch, + hubThread.machineBannerItem, isRestoringThreadBranch, localCheckoutBranchMismatch, parkedThreadBannerItem, @@ -6011,14 +6035,14 @@ function ChatViewContent(props: ChatViewProps) { const isFirstMessage = !isServerThread || activeThread.messages.length === 0; const baseBranchForWorktree = isFirstMessage && sendEnvMode === "worktree" && !activeThread.worktreePath - ? activeThreadBranch + ? (activeThreadBranch ?? hubThread.defaultBaseRef) : null; // In worktree mode, require an explicit base branch so we don't silently // fall back to local execution when branch selection is missing. const shouldCreateWorktree = isFirstMessage && sendEnvMode === "worktree" && !activeThread.worktreePath; - if (shouldCreateWorktree && !activeThreadBranch) { + if (shouldCreateWorktree && !baseBranchForWorktree) { setThreadError(threadIdForSend, "Select a base branch before sending in New worktree mode."); return; } @@ -7170,7 +7194,7 @@ function ChatViewContent(props: ChatViewProps) { const panelToggleControls = ( ) : activeRightPanelSurface?.kind === "workspaceBrowser" ? ( - + ) : activeRightPanelSurface?.kind === "terminal" ? ( + ) : (activeRightPanelSurface?.kind === "files" || activeRightPanelSurface?.kind === "file") && + hubThread.machineUnavailable && + hubThread.machineView ? ( + ) : (activeRightPanelSurface?.kind === "files" || activeRightPanelSurface?.kind === "file") && activeProject && activeWorkspaceRoot ? ( @@ -7361,7 +7389,9 @@ function ChatViewContent(props: ChatViewProps) { activeProjectCwd={activeProject?.workspaceRoot ?? null} activeProjectFaviconPath={activeProject?.faviconPath ?? null} openInCwd={gitCwd} - activeProjectScripts={activeProject?.scripts} + activeProjectScripts={ + hubThread.draftWithoutMachine ? undefined : activeProject?.scripts + } preferredScriptId={ activeProject ? (lastInvokedScriptByProjectId[activeProject.id] ?? null) : null } @@ -7440,10 +7470,14 @@ function ChatViewContent(props: ChatViewProps) { onOpenAgents={addAgentsSurface} key={activeThread.id} isWorking={isWorking} - isWakingWorkspace={isWakingManagedWorkspace} - workingStepLabel={workingStepLabel} + isWakingWorkspace={ + isWakingManagedWorkspace || + hubThread.machineTransitional || + (hubThread.hub && isPreparingWorktree) + } + workingStepLabel={hubThread.machineDetail ?? workingStepLabel} activeTurnInProgress={isWorking || !latestTurnSettled} - isPreparingWorktree={isPreparingWorktree} + isPreparingWorktree={isPreparingWorktree && !hubThread.hub} activeTurnStartedAt={activeWorkStartedAt} activeToolWait={activeToolWait} listRef={legendListRef} @@ -7654,7 +7688,7 @@ function ChatViewContent(props: ChatViewProps) { onEnvModeChange={onEnvModeChange} startFromOrigin={startFromOrigin} onStartFromOriginChange={onStartFromOriginChange} - {...(canOverrideServerThreadEnvMode + {...(canOverrideServerThreadEnvMode || hubThread.forcedEnvMode ? { effectiveEnvModeOverride: envMode } : {})} {...(canOverrideServerThreadEnvMode @@ -7666,7 +7700,7 @@ function ChatViewContent(props: ChatViewProps) { : {})} envLocked={envLocked} onComposerFocusRequest={scheduleComposerFocus} - {...(canCheckoutPullRequestIntoThread + {...(canCheckoutPullRequestIntoThread && !hubThread.hub ? { onCheckoutPullRequestRequest: openPullRequestDialog } : {})} {...(hasMultipleEnvironments ? { onEnvironmentChange } : {})} @@ -7784,16 +7818,18 @@ function ChatViewContent(props: ChatViewProps) { onCloseAllSurfaces={closeAllRightPanelSurfaces} onCopyFilePath={copyRightPanelFilePath} onAddBrowser={createBrowserSurface} - onAddWorkspaceBrowser={createWorkspaceBrowserSurface} + onAddWorkspaceBrowser={ + hubThread.workspaceAvailable ? createWorkspaceBrowserSurface : undefined + } onAddTerminal={addTerminalSurface} onAddDiff={addDiffSurface} onAddFiles={addFilesSurface} onAddPullRequest={addPullRequestSurface} onAddAgents={addAgentsSurface} browserAvailable={isPreviewSupportedInRuntime()} - terminalAvailable={activeProject !== null} + terminalAvailable={activeProject !== null && hubThread.workspaceAvailable} diffAvailable={isServerThread && isGitRepo} - filesAvailable={activeProject !== null} + filesAvailable={activeProject !== null && hubThread.workspaceAvailable} pullRequestAvailable={pullRequestSurfaceAvailable} agentsAvailable pullRequestStatuses={pullRequestTabStatuses} @@ -7825,16 +7861,18 @@ function ChatViewContent(props: ChatViewProps) { onCloseAllSurfaces={closeAllRightPanelSurfaces} onCopyFilePath={copyRightPanelFilePath} onAddBrowser={createBrowserSurface} - onAddWorkspaceBrowser={createWorkspaceBrowserSurface} + onAddWorkspaceBrowser={ + hubThread.workspaceAvailable ? createWorkspaceBrowserSurface : undefined + } onAddTerminal={addTerminalSurface} onAddDiff={addDiffSurface} onAddFiles={addFilesSurface} onAddPullRequest={addPullRequestSurface} onAddAgents={addAgentsSurface} browserAvailable={isPreviewSupportedInRuntime()} - terminalAvailable={activeProject !== null} + terminalAvailable={activeProject !== null && hubThread.workspaceAvailable} diffAvailable={isServerThread && isGitRepo} - filesAvailable={activeProject !== null} + filesAvailable={activeProject !== null && hubThread.workspaceAvailable} pullRequestAvailable={pullRequestSurfaceAvailable} agentsAvailable pullRequestStatuses={pullRequestTabStatuses} diff --git a/apps/web/src/components/DiffPanel.tsx b/apps/web/src/components/DiffPanel.tsx index 66886c3d474d..461455b29cad 100644 --- a/apps/web/src/components/DiffPanel.tsx +++ b/apps/web/src/components/DiffPanel.tsx @@ -76,6 +76,7 @@ import { reviewEnvironment } from "../state/review"; import { vcsEnvironment } from "../state/vcs"; import { buildBaseRefChoices, filterBaseRefChoices } from "../lib/baseRefChoices"; import { createGitDiffFileContentsLoader } from "../lib/diffFileContents"; +import { useDiffPanelTurnDefaultWhileMachineAsleep } from "./hub/useHubDiffDefaults"; type DiffThemeType = "light" | "dark"; const AUTOMATIC_BASE_REF = "__automatic_base_ref__"; @@ -200,6 +201,7 @@ export default function DiffPanel({ selectedTurn && (selectedTurn.checkpointTurnCount ?? inferredCheckpointTurnCountByTurnId[selectedTurn.turnId]); const latestTurn = orderedTurnDiffSummaries[0]; + useDiffPanelTurnDefaultWhileMachineAsleep(routeThreadRef, latestTurn?.turnId ?? null); const selectedScopeLabel = selectedTurnId === null ? selectedGitScope === "unstaged" diff --git a/apps/web/src/components/ManagedDevPcAccount.tsx b/apps/web/src/components/ManagedDevPcAccount.tsx index 38350e0f5297..0adaecce7c00 100644 --- a/apps/web/src/components/ManagedDevPcAccount.tsx +++ b/apps/web/src/components/ManagedDevPcAccount.tsx @@ -3,6 +3,7 @@ import { UserRoundIcon } from "lucide-react"; import { useNavigate } from "@tanstack/react-router"; import { isManagedDevPc } from "../managedDevPc"; +import { usePrimaryIsHub } from "../hubMode"; import { Dialog, DialogDescription, @@ -189,12 +190,14 @@ export function ManagedDevPcFooterAccount({ }) { const navigate = useNavigate(); const { isMobile, setOpenMobile } = useSidebar(); + // A hub has no workspace machine: each thread shows its own machine instead. + const primaryIsHub = usePrimaryIsHub(); if (!isManagedDevPc || isLandingDemo()) return null; return (
- {showWorkspaceStatus ? ( + {showWorkspaceStatus && !primaryIsHub ? ( { if (isMobile) setOpenMobile(false); diff --git a/apps/web/src/components/RightPanelTabs.tsx b/apps/web/src/components/RightPanelTabs.tsx index c5612d8e5c92..afa7c93d0207 100644 --- a/apps/web/src/components/RightPanelTabs.tsx +++ b/apps/web/src/components/RightPanelTabs.tsx @@ -29,7 +29,7 @@ import type { DesktopPreviewOverlay } from "~/previewStateStore"; import type { RightPanelSurface } from "~/rightPanelStore"; import { cn } from "~/lib/utils"; import { readLocalApi } from "~/localApi"; -import { managedWorkspaceBrowserUrl } from "~/managedDevPc"; +import { isManagedWorkspaceBrowserAvailable } from "~/managedDevPc"; import { Button } from "~/components/ui/button"; import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip"; import { Kbd } from "~/components/ui/kbd"; @@ -638,7 +638,7 @@ export function RightPanelTabs(props: RightPanelTabsProps) { disabledReason: string | null; onClick: () => void; }> = [ - ...(props.onAddWorkspaceBrowser && managedWorkspaceBrowserUrl() + ...(props.onAddWorkspaceBrowser && isManagedWorkspaceBrowserAvailable() ? [ { label: "Workspace browser", @@ -984,7 +984,7 @@ export function RightPanelTabs(props: RightPanelTabsProps) { ; } - const url = managedWorkspaceBrowserUrl(); + const url = managedWorkspaceBrowserUrl(threadId); // Remounting the frame is the only way to force a reconnect from here: the // noVNC session lives inside the iframe's own document. const [reloadKey, setReloadKey] = useState(0); diff --git a/apps/web/src/components/hub/hubAuthConnector.test.ts b/apps/web/src/components/hub/hubAuthConnector.test.ts new file mode 100644 index 000000000000..c1871962c61a --- /dev/null +++ b/apps/web/src/components/hub/hubAuthConnector.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.resetModules(); +}); + +async function load(bootstrap: Record) { + vi.stubEnv("VITE_DEVPC_MANAGED", "1"); + vi.resetModules(); + vi.stubGlobal("window", { + location: { origin: "https://app.example.test" }, + __DEVPC_MANAGED_BOOTSTRAP__: { managed: true, state: "ready", ready: true, ...bootstrap }, + }); + return import("./hubAuthConnector"); +} + +describe("resolveAuthWorkspaceBrowserUrl", () => { + it("prefers the browser the sign-in session names", async () => { + const { resolveAuthWorkspaceBrowserUrl } = await load({ + serverMode: "hub", + threadBrowserUrlTemplate: "/_devpc/threads/{threadId}/browser", + }); + expect( + resolveAuthWorkspaceBrowserUrl({ id: "session", workspaceBrowserUrl: "/_aldo/sign-in" }), + ).toBe("/_aldo/sign-in"); + }); + + it("opens the hub's sign-in machine browser when the session names none", async () => { + const { resolveAuthWorkspaceBrowserUrl } = await load({ + serverMode: "hub", + threadBrowserUrlTemplate: "/_devpc/threads/{threadId}/browser", + }); + expect(resolveAuthWorkspaceBrowserUrl({ id: "session" })).toBe( + "/_devpc/threads/aldo-provider-sign-in/browser", + ); + expect(resolveAuthWorkspaceBrowserUrl(null)).toBe( + "/_devpc/threads/aldo-provider-sign-in/browser", + ); + }); + + it("keeps the shared workspace browser on a persistent workspace", async () => { + const { resolveAuthWorkspaceBrowserUrl } = await load({ + previewUrlTemplate: "https://{port}.preview.example.test/", + previewUrls: { "6080": "https://6080.preview.example.test/vnc.html" }, + }); + expect(resolveAuthWorkspaceBrowserUrl(null)).toBe("https://6080.preview.example.test/vnc.html"); + }); +}); diff --git a/apps/web/src/components/hub/hubAuthConnector.ts b/apps/web/src/components/hub/hubAuthConnector.ts new file mode 100644 index 000000000000..c2fa7eb08ef1 --- /dev/null +++ b/apps/web/src/components/hub/hubAuthConnector.ts @@ -0,0 +1,27 @@ +import { managedWorkspaceBrowserUrl } from "../../managedDevPc"; + +/** + * The machine a hub starts for provider sign-in has its own browser, reached + * like a thread's under this reserved id. + */ +export const HUB_SIGN_IN_BROWSER_THREAD_ID = "aldo-provider-sign-in"; + +/** + * Where a sign-in that needs the workspace browser should open it: the URL + * the session names (a hub's sign-in machine), else the deployment's browser. + * + * TODO(thread-machines): read `session.workspaceBrowserUrl` directly once the + * `AuthConnectorSession` contract carries it; until then decoding drops it + * and the reserved sign-in id stands in. + */ +export function resolveAuthWorkspaceBrowserUrl(session: object | null): string | null { + if ( + session !== null && + "workspaceBrowserUrl" in session && + typeof session.workspaceBrowserUrl === "string" && + session.workspaceBrowserUrl.length > 0 + ) { + return session.workspaceBrowserUrl; + } + return managedWorkspaceBrowserUrl(HUB_SIGN_IN_BROWSER_THREAD_ID); +} diff --git a/apps/web/src/components/hub/threadMachineActions.ts b/apps/web/src/components/hub/threadMachineActions.ts new file mode 100644 index 000000000000..8b3fd6239407 --- /dev/null +++ b/apps/web/src/components/hub/threadMachineActions.ts @@ -0,0 +1,20 @@ +import type { ScopedThreadRef } from "@t3tools/contracts"; + +/** + * Explicit machine lifecycle requests for a hub thread. + * + * The hub's `threadMachines.wake` / `threadMachines.pause` RPCs are not in + * the client contracts yet, so explicit wake is reported as unsupported and + * every caller hides its affordance. A turn (send, retry) and a terminal + * still wake the machine on their own. + * + * TODO(thread-machines): call the `threadMachines.wake({ threadId })` RPC + * here once it is part of `WsRpcGroup`, and flip the flag. + */ +export const THREAD_MACHINE_WAKE_SUPPORTED = false; + +export async function requestThreadMachineWake( + _threadRef: ScopedThreadRef, +): Promise<"requested" | "unsupported"> { + return "unsupported"; +} diff --git a/apps/web/src/components/hub/useHubDiffDefaults.ts b/apps/web/src/components/hub/useHubDiffDefaults.ts new file mode 100644 index 000000000000..5d1bdba01253 --- /dev/null +++ b/apps/web/src/components/hub/useHubDiffDefaults.ts @@ -0,0 +1,31 @@ +import type { ScopedThreadRef, TurnId } from "@t3tools/contracts"; +import { scopedThreadKey } from "@t3tools/client-runtime/environment"; +import { useEffect } from "react"; + +import { useDiffPanelStore } from "../../diffPanelStore"; +import { useIsHubEnvironment } from "../../hubMode"; +import { useThreadShell } from "../../state/entities"; +import { deriveThreadMachineView } from "../../threadMachine"; + +/** + * A sleeping hub machine cannot answer working-tree or branch diffs, but the + * hub keeps every captured turn diff. Until the user picks a scope, open the + * diff panel on the latest turn instead of a view that would wake nothing and + * show an error. + */ +export function useDiffPanelTurnDefaultWhileMachineAsleep( + threadRef: ScopedThreadRef | null | undefined, + latestTurnId: TurnId | null, +): void { + const hub = useIsHubEnvironment(threadRef?.environmentId); + const machine = useThreadShell(hub && threadRef ? threadRef : null)?.machine; + const phase = deriveThreadMachineView(machine)?.phase; + const machineUnavailable = phase === "asleep" || phase === "failed"; + const hasStoredSelection = useDiffPanelStore((state) => + threadRef ? state.byThreadKey[scopedThreadKey(threadRef)] !== undefined : true, + ); + useEffect(() => { + if (!threadRef || !machineUnavailable || hasStoredSelection || latestTurnId === null) return; + useDiffPanelStore.getState().selectTurn(threadRef, latestTurnId); + }, [hasStoredSelection, latestTurnId, machineUnavailable, threadRef]); +} diff --git a/apps/web/src/components/hub/useHubThreadContext.test.ts b/apps/web/src/components/hub/useHubThreadContext.test.ts new file mode 100644 index 000000000000..70f9fd7ca2b9 --- /dev/null +++ b/apps/web/src/components/hub/useHubThreadContext.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { deriveThreadMachineView } from "../../threadMachine"; +import { resolveHubThreadContext } from "./useHubThreadContext"; + +const identity = { + canonicalKey: "github.com/acme/app", + locator: { source: "git-remote" as const, remoteName: "origin", remoteUrl: "x" }, +}; +const view = (state: "starting" | "paused" | "failed" | "running", detail: string | null = null) => + deriveThreadMachineView({ state, detail, updatedAt: "2026-09-26T10:00:00.000Z" }); + +describe("resolveHubThreadContext", () => { + it("changes nothing off a hub", () => { + expect( + resolveHubThreadContext({ + hub: false, + isServerThread: false, + projectRepositoryIdentity: null, + machineView: view("failed"), + }), + ).toEqual({ + hub: false, + draftWithoutMachine: false, + workspaceAvailable: true, + isGitRepoOverride: null, + forcedEnvMode: null, + defaultBaseRef: null, + machineView: null, + machineTransitional: false, + machineDetail: null, + machineUnavailable: false, + }); + }); + + it("starts every hub draft on a fresh machine from the default branch", () => { + expect( + resolveHubThreadContext({ + hub: true, + isServerThread: false, + projectRepositoryIdentity: identity, + machineView: null, + }), + ).toMatchObject({ + draftWithoutMachine: true, + workspaceAvailable: false, + isGitRepoOverride: true, + forcedEnvMode: "worktree", + defaultBaseRef: "HEAD", + }); + }); + + it("treats a blank hub project's draft as no repository", () => { + expect( + resolveHubThreadContext({ + hub: true, + isServerThread: false, + projectRepositoryIdentity: null, + machineView: null, + }).isGitRepoOverride, + ).toBe(false); + }); + + it("reports machine progress for a started thread", () => { + expect( + resolveHubThreadContext({ + hub: true, + isServerThread: true, + projectRepositoryIdentity: identity, + machineView: view("starting", "Cloning repository"), + }), + ).toMatchObject({ + draftWithoutMachine: false, + workspaceAvailable: true, + isGitRepoOverride: null, + machineTransitional: true, + machineDetail: "Cloning repository", + machineUnavailable: false, + }); + }); + + it("marks machine-backed reads unavailable while the machine sleeps or has failed", () => { + for (const state of ["paused", "failed"] as const) { + expect( + resolveHubThreadContext({ + hub: true, + isServerThread: true, + projectRepositoryIdentity: identity, + machineView: view(state), + }).machineUnavailable, + ).toBe(true); + } + }); +}); diff --git a/apps/web/src/components/hub/useHubThreadContext.ts b/apps/web/src/components/hub/useHubThreadContext.ts new file mode 100644 index 000000000000..98e00790c3b7 --- /dev/null +++ b/apps/web/src/components/hub/useHubThreadContext.ts @@ -0,0 +1,172 @@ +import type { + EnvironmentId, + OrchestrationLatestTurn, + RepositoryIdentity, + ThreadId, +} from "@t3tools/contracts"; +import { scopedThreadKey, scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { + isAtomCommandInterrupted, + squashAtomCommandFailure, +} from "@t3tools/client-runtime/state/runtime"; +import { useCallback, useMemo, useState } from "react"; + +import { HUB_DEFAULT_BASE_REF, useIsHubEnvironment } from "../../hubMode"; +import { useThreadShell } from "../../state/entities"; +import { threadEnvironment } from "../../state/threads"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { deriveThreadMachineView, type ThreadMachineView } from "../../threadMachine"; +import type { ComposerBannerStackItem } from "../chat/ComposerBannerStack"; +import { buildThreadMachineBannerItem } from "../ThreadMachineStatus"; +import { stackedThreadToast, toastManager } from "../ui/toast"; +import { requestThreadMachineWake, THREAD_MACHINE_WAKE_SUPPORTED } from "./threadMachineActions"; + +export interface HubThreadContext { + /** The thread's environment is a hub: every thread runs on its own machine. */ + readonly hub: boolean; + /** A hub draft: its machine does not exist until the first send. */ + readonly draftWithoutMachine: boolean; + /** Machine-backed surfaces (terminals, files, scripts) make sense for this thread. */ + readonly workspaceAvailable: boolean; + /** A hub draft is a repository exactly when its project records one. */ + readonly isGitRepoOverride: boolean | null; + /** Every hub thread gets a fresh checkout on its own machine. */ + readonly forcedEnvMode: "worktree" | null; + /** Base ref sent when the user picked none. */ + readonly defaultBaseRef: string | null; + readonly machineView: ThreadMachineView | null; + readonly machineTransitional: boolean; + /** Progress text while the machine is preparing or starting. */ + readonly machineDetail: string | null; + /** The machine cannot serve reads right now (asleep or failed). */ + readonly machineUnavailable: boolean; + readonly machineBannerItem: ComposerBannerStackItem | null; + /** Say so when an action (opening a terminal) is about to wake the machine. */ + readonly announceMachineWake: () => void; +} + +export interface HubThreadContextInput { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId | null; + readonly isServerThread: boolean; + readonly projectRepositoryIdentity: RepositoryIdentity | null | undefined; + readonly latestTurn: Pick | null; +} + +/** Pure part of the context, shared with tests. */ +export function resolveHubThreadContext(input: { + readonly hub: boolean; + readonly isServerThread: boolean; + readonly projectRepositoryIdentity: RepositoryIdentity | null | undefined; + readonly machineView: ThreadMachineView | null; +}): Omit { + const { hub, machineView } = input; + const draftWithoutMachine = hub && !input.isServerThread; + const machineTransitional = hub && machineView?.phase === "transitional"; + return { + hub, + draftWithoutMachine, + workspaceAvailable: !draftWithoutMachine, + isGitRepoOverride: draftWithoutMachine ? input.projectRepositoryIdentity != null : null, + forcedEnvMode: hub ? "worktree" : null, + defaultBaseRef: hub ? HUB_DEFAULT_BASE_REF : null, + machineView: hub ? machineView : null, + machineTransitional, + machineDetail: machineTransitional ? (machineView?.detail ?? null) : null, + machineUnavailable: hub && (machineView?.phase === "asleep" || machineView?.phase === "failed"), + }; +} + +const dismissedMachineBanners = new Set(); + +/** + * Everything ChatView needs to know about hub mode for the active thread, so + * the view itself only swaps values at the points where hub threads differ. + */ +export function useHubThreadContext(input: HubThreadContextInput): HubThreadContext { + const hub = useIsHubEnvironment(input.environmentId); + const threadRef = useMemo( + () => + hub && input.isServerThread && input.threadId + ? scopeThreadRef(input.environmentId, input.threadId) + : null, + [hub, input.environmentId, input.isServerThread, input.threadId], + ); + const machine = useThreadShell(threadRef)?.machine; + const machineView = useMemo(() => deriveThreadMachineView(machine), [machine]); + const base = resolveHubThreadContext({ + hub, + isServerThread: input.isServerThread, + projectRepositoryIdentity: input.projectRepositoryIdentity, + machineView, + }); + + const retryTurn = useAtomCommand(threadEnvironment.retryTurn, { reportFailure: false }); + const [retrying, setRetrying] = useState(false); + const [, setDismissTick] = useState(0); + const failedTurnId = input.latestTurn?.state === "error" ? input.latestTurn.turnId : null; + const canRetry = + base.machineView?.phase === "failed" && + threadRef !== null && + (failedTurnId !== null || THREAD_MACHINE_WAKE_SUPPORTED); + const onRetry = useCallback(() => { + if (!threadRef) return; + setRetrying(true); + void (async () => { + if (failedTurnId !== null) { + // Re-driving the failed turn wakes (or recreates) the machine. + const result = await retryTurn({ + environmentId: threadRef.environmentId, + input: { threadId: threadRef.threadId, turnId: failedTurnId }, + }); + if (result._tag === "Failure" && !isAtomCommandInterrupted(result)) { + const error = squashAtomCommandFailure(result); + toastManager.add( + stackedThreadToast({ + type: "error", + title: "Could not retry", + description: error instanceof Error ? error.message : "An error occurred.", + }), + ); + } + } else { + await requestThreadMachineWake(threadRef); + } + setRetrying(false); + })(); + }, [failedTurnId, retryTurn, threadRef]); + + const threadKey = threadRef ? scopedThreadKey(threadRef) : null; + const bannerView = base.machineView; + const bannerId = threadKey && bannerView ? `${threadKey}:${bannerView.state}` : null; + // Re-read on every render; dismissing bumps the tick to trigger one. + const bannerDismissed = bannerId !== null && dismissedMachineBanners.has(bannerId); + const machineBannerItem = useMemo(() => { + if (threadKey === null || bannerId === null || bannerDismissed) return null; + return buildThreadMachineBannerItem({ + threadKey, + view: bannerView, + onRetry: canRetry ? onRetry : null, + retrying, + onDismiss: () => { + dismissedMachineBanners.add(bannerId); + setDismissTick((tick) => tick + 1); + }, + }); + }, [bannerDismissed, bannerId, bannerView, canRetry, onRetry, retrying, threadKey]); + + const machineWakes = base.machineView !== null && base.machineView.phase !== "running"; + const announceMachineWake = useCallback(() => { + if (!machineWakes) return; + toastManager.add( + stackedThreadToast({ + type: "info", + title: "Waking this thread's machine", + description: "The terminal connects as soon as it is running.", + timeout: 5_000, + }), + ); + }, [machineWakes]); + + return { ...base, machineBannerItem, announceMachineWake }; +} diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 1faf928b1cf5..d575a181e424 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -374,10 +374,11 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) case "open": { const input = request.input as PreviewAutomationOpenInput; const resolvedInputUrl = input.url - ? resolveBrowserNavigationTarget(environmentId, { - kind: "url", - url: input.url, - }).resolvedUrl + ? resolveBrowserNavigationTarget( + environmentId, + { kind: "url", url: input.url }, + threadRef.threadId, + ).resolvedUrl : undefined; let activeTabId = resolvePreviewAutomationOpenTab( state, @@ -487,6 +488,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) kind: "url", url: input.url!, }, + threadRef.threadId, ); await ready.bridge.navigate(ready.runtimeTabId, resolution.resolvedUrl); await waitForNavigationReadiness( diff --git a/apps/web/src/components/preview/PreviewView.tsx b/apps/web/src/components/preview/PreviewView.tsx index 063314863fec..67d670c97f60 100644 --- a/apps/web/src/components/preview/PreviewView.tsx +++ b/apps/web/src/components/preview/PreviewView.tsx @@ -190,7 +190,11 @@ export function PreviewView({ const handleOpenServerUrl = useCallback( async (next: string) => { try { - const resolved = resolveDiscoveredServerUrl(threadRef.environmentId, next); + const resolved = resolveDiscoveredServerUrl( + threadRef.environmentId, + next, + threadRef.threadId, + ); if (await navigateToResolvedUrl(resolved)) { recordVisitForThread(threadRef, next); } diff --git a/apps/web/src/components/preview/openDiscoveredPort.ts b/apps/web/src/components/preview/openDiscoveredPort.ts index a49acbd86104..69352eff876e 100644 --- a/apps/web/src/components/preview/openDiscoveredPort.ts +++ b/apps/web/src/components/preview/openDiscoveredPort.ts @@ -15,7 +15,11 @@ export async function openDiscoveredPort(input: { readonly port: DiscoveredLocalServer; readonly openPreview: OpenPreviewMutation; }): Promise> { - const resolvedUrl = resolveDiscoveredServerUrl(input.threadRef.environmentId, input.port.url); + const resolvedUrl = resolveDiscoveredServerUrl( + input.threadRef.environmentId, + input.port.url, + input.threadRef.threadId, + ); const result = await openPreviewSession({ openPreview: input.openPreview, threadRef: input.threadRef, diff --git a/apps/web/src/components/settings/AuthConnectorDialog.tsx b/apps/web/src/components/settings/AuthConnectorDialog.tsx index 58a3e2339b29..f24e060f9beb 100644 --- a/apps/web/src/components/settings/AuthConnectorDialog.tsx +++ b/apps/web/src/components/settings/AuthConnectorDialog.tsx @@ -41,7 +41,8 @@ import { Input } from "../ui/input"; import { Textarea } from "../ui/textarea"; import { toastManager } from "../ui/toast"; import { APP_BASE_NAME } from "../../branding"; -import { managedWorkspaceBrowserUrl } from "../../managedDevPc"; +import { useIsHubEnvironment } from "../../hubMode"; +import { resolveAuthWorkspaceBrowserUrl } from "../hub/hubAuthConnector"; export type AuthConnectorMethodOption = { readonly method: AuthConnectorMethod; @@ -165,6 +166,8 @@ export function AuthConnectorDialog(props: { const callbackInputRef = useRef(null); const selectedMethod = methods.find((method) => method.method === session?.method) ?? null; + const hub = useIsHubEnvironment(environmentId); + const workspaceBrowserUrl = resolveAuthWorkspaceBrowserUrl(session); useEffect(() => { if (!open || !sessionEnvironmentId || !session) return; @@ -284,7 +287,7 @@ export function AuthConnectorDialog(props: { authVerificationDestination({ verificationUrl: session.verificationUrl, workspaceBrowser: selectedMethod?.workspaceBrowser === true, - workspaceBrowserUrl: managedWorkspaceBrowserUrl(), + workspaceBrowserUrl, }), ); }; @@ -366,7 +369,7 @@ export function AuthConnectorDialog(props: { const remaining = session ? formatRemaining(session.expiresAt, now) : null; const browserName = selectedMethod?.browserName ?? serviceName; const usesManagedWorkspaceBrowser = - selectedMethod?.workspaceBrowser === true && managedWorkspaceBrowserUrl() !== null; + selectedMethod?.workspaceBrowser === true && workspaceBrowserUrl !== null; const authorizeInstruction = authAuthorizeInstruction({ option: selectedMethod, usesManagedWorkspaceBrowser, @@ -380,7 +383,9 @@ export function AuthConnectorDialog(props: { ? `Return to ${APP_BASE_NAME}` : presentationStage === "verifying" ? "Checking your account" - : "Preparing secure sign-in"; + : hub && selectedMethod?.workspaceBrowser === true + ? "Starting a machine for sign-in…" + : "Preparing secure sign-in"; const stageDescription = presentationStage === "authorize" ? (authorizeInstruction ?? diff --git a/apps/web/src/components/settings/ProjectSettingsPanel.tsx b/apps/web/src/components/settings/ProjectSettingsPanel.tsx index dd9c899430b3..0649dae84743 100644 --- a/apps/web/src/components/settings/ProjectSettingsPanel.tsx +++ b/apps/web/src/components/settings/ProjectSettingsPanel.tsx @@ -114,6 +114,7 @@ import { ProjectFaviconPickerDialog, } from "./ProjectFaviconPickerDialog"; import { projectGroupTitleNeedsUpdate } from "./ProjectSettingsPanel.logic"; +import { hubProjectLocationLabel, readIsHubEnvironment, useIsHubEnvironment } from "../../hubMode"; export const PROJECT_GROUPING_MODE_LABELS: Record = { repository: "Group by repository", @@ -289,6 +290,12 @@ export function ProjectSettingsPanel({ projectKey }: { projectKey: string }) { return ; } +function projectLocation(project: SidebarProjectSnapshot["memberProjects"][number]): string { + return readIsHubEnvironment(project.environmentId) + ? hubProjectLocationLabel(project) + : project.workspaceRoot; +} + function ProjectDetail({ group }: { group: SidebarProjectSnapshot }) { const navigate = useNavigate(); const primaryEnvironmentId = usePrimaryEnvironmentId(); @@ -473,6 +480,11 @@ function ProjectDetail({ group }: { group: SidebarProjectSnapshot }) { const selectedCheckout = group.memberProjects.find((member) => member.physicalProjectKey === selectedCheckoutKey) ?? representative; + // A hub project has no folder; its repository is what identifies it. + const selectedCheckoutIsHub = useIsHubEnvironment(selectedCheckout.environmentId); + const selectedCheckoutLocation = selectedCheckoutIsHub + ? hubProjectLocationLabel(selectedCheckout) + : selectedCheckout.workspaceRoot; const selectedServerConfig = useAtomValue( serverEnvironment.configValueAtom(selectedCheckout.environmentId), ); @@ -693,7 +705,7 @@ function ProjectDetail({ group }: { group: SidebarProjectSnapshot }) { : `Remove project "${targetLabel}"?`, ...(singleMember ? [ - `Path: ${singleMember.workspaceRoot}`, + `Path: ${projectLocation(singleMember)}`, ...(singleMember.environmentLabel ? [`Environment: ${singleMember.environmentLabel}`] : []), @@ -884,6 +896,7 @@ function ProjectDetail({ group }: { group: SidebarProjectSnapshot }) { } />
) : null} @@ -4019,7 +4022,7 @@ export default function Sidebar() { className="inline-flex cursor-pointer items-center gap-1.5 rounded-md border border-sidebar-border px-2.5 py-1 text-[11px] font-medium text-sidebar-muted-foreground transition-colors hover:bg-sidebar-row-hover hover:text-sidebar-foreground" > - Add project + {primaryIsHub ? "Add repository" : "Add project"} ) : scopedProjectGroup ? ( diff --git a/apps/web/src/components/hub/hubAddRepository.test.ts b/apps/web/src/components/hub/hubAddRepository.test.ts new file mode 100644 index 000000000000..59190a0d9b93 --- /dev/null +++ b/apps/web/src/components/hub/hubAddRepository.test.ts @@ -0,0 +1,98 @@ +import { ProjectId } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { + buildHubBlankProjectInput, + buildHubRepositoryIdentity, + buildHubRepositoryProjectInput, + findHubProjectForRemoteUrl, +} from "./hubAddRepository"; + +describe("buildHubRepositoryIdentity", () => { + it("records a GitHub HTTPS remote the way a checkout's git remote would", () => { + expect(buildHubRepositoryIdentity("https://github.com/Acme/App.git")).toEqual({ + canonicalKey: "github.com/acme/app", + locator: { + source: "git-remote", + remoteName: "origin", + remoteUrl: "https://github.com/Acme/App.git", + }, + displayName: "acme/app", + provider: "github", + owner: "acme", + name: "app", + }); + }); + + it("normalizes SSH remotes and nested groups", () => { + expect(buildHubRepositoryIdentity("git@gitlab.com:group/sub/project.git")).toMatchObject({ + canonicalKey: "gitlab.com/group/sub/project", + provider: "gitlab", + owner: "group", + name: "project", + }); + }); +}); + +describe("findHubProjectForRemoteUrl", () => { + const projects = [ + { + id: "one", + repositoryIdentity: { + canonicalKey: "github.com/acme/app", + locator: { remoteUrl: "https://github.com/acme/app" }, + }, + }, + { id: "blank", repositoryIdentity: null }, + ]; + + it("finds a project already recording the same repository under another URL form", () => { + expect(findHubProjectForRemoteUrl("git@github.com:Acme/App.git", projects)?.id).toBe("one"); + expect(findHubProjectForRemoteUrl("https://github.com/acme/app/", projects)?.id).toBe("one"); + }); + + it("does not match blank projects or other repositories", () => { + expect(findHubProjectForRemoteUrl("https://github.com/acme/other", projects)).toBeNull(); + }); +}); + +describe("hub project create inputs", () => { + const projectId = ProjectId.make("project-1"); + + it("creates repository projects on the virtual root with the recorded identity", () => { + const input = buildHubRepositoryProjectInput({ + projectId, + remoteUrl: "https://github.com/acme/app", + defaultModelSelection: null, + }); + expect(input).toMatchObject({ + projectId, + title: "app", + workspaceRoot: "/workspace/p/project-1", + createWorkspaceRootIfMissing: false, + repositoryIdentity: { canonicalKey: "github.com/acme/app" }, + }); + }); + + it("prefers an explicit title", () => { + expect( + buildHubRepositoryProjectInput({ + projectId, + remoteUrl: "https://github.com/acme/app", + title: " Marketing site ", + defaultModelSelection: null, + }).title, + ).toBe("Marketing site"); + }); + + it("creates blank projects without a repository", () => { + expect( + buildHubBlankProjectInput({ projectId, title: " ", defaultModelSelection: null }), + ).toMatchObject({ + title: "Untitled project", + workspaceRoot: "/workspace/p/project-1", + createWorkspaceRootIfMissing: false, + repositoryIdentity: null, + }); + }); +}); diff --git a/apps/web/src/components/hub/hubAddRepository.ts b/apps/web/src/components/hub/hubAddRepository.ts new file mode 100644 index 000000000000..9ca322fc0fcf --- /dev/null +++ b/apps/web/src/components/hub/hubAddRepository.ts @@ -0,0 +1,235 @@ +/** + * Adding a project to a hub: a hub has no folders, so a project is either a + * repository (recorded by its remote, cloned onto each thread's machine) or a + * blank project (each machine starts from an empty repository). + */ +import type { + EnvironmentId, + ModelSelection, + ProjectId, + RepositoryIdentity, + ScopedProjectRef, +} from "@t3tools/contracts"; +import { scopeProjectRef, scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { + isAtomCommandInterrupted, + settlePromise, + squashAtomCommandFailure, +} from "@t3tools/client-runtime/state/runtime"; +import { + detectSourceControlProviderFromGitRemoteUrl, + normalizeGitRemoteUrl, +} from "@t3tools/shared/git"; +import { useNavigate } from "@tanstack/react-router"; +import { useCallback } from "react"; + +import { useNewThreadHandler } from "../../hooks/useHandleNewThread"; +import { useClientSettings } from "../../hooks/useSettings"; +import { hubProjectWorkspaceRoot } from "../../hubMode"; +import { getLatestThreadForProject } from "../../lib/threadSort"; +import { newProjectId } from "../../lib/utils"; +import { readProjects, readThreadShells } from "../../state/entities"; +import { projectEnvironment } from "../../state/projects"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { buildThreadRouteParams } from "../../threadRoutes"; +import { normalizeGitRemoteIdentity } from "../GitHubRepositoryBrowser.logic"; +import { stackedThreadToast, toastManager } from "../ui/toast"; + +/** + * The identity a hub records for a repository project, shaped like the one a + * server reads from `git remote` so grouping and pull requests match. + */ +export function buildHubRepositoryIdentity(remoteUrl: string): RepositoryIdentity { + const trimmedUrl = remoteUrl.trim(); + const canonicalKey = normalizeGitRemoteUrl(trimmedUrl); + const repositoryPath = canonicalKey.split("/").slice(1).join("/"); + const segments = repositoryPath.split("/").filter((segment) => segment.length > 0); + const provider = detectSourceControlProviderFromGitRemoteUrl(trimmedUrl); + const owner = segments[0]; + const name = segments.at(-1); + return { + canonicalKey, + locator: { source: "git-remote", remoteName: "origin", remoteUrl: trimmedUrl }, + ...(repositoryPath ? { displayName: repositoryPath } : {}), + ...(provider ? { provider: provider.kind } : {}), + ...(owner && segments.length > 1 ? { owner } : {}), + ...(name ? { name } : {}), + }; +} + +interface HubRepositoryProjectCandidate { + readonly repositoryIdentity?: + | { + readonly canonicalKey: string; + readonly locator: { readonly remoteUrl: string }; + } + | null + | undefined; +} + +/** The project already recording this remote, compared the way the repository browser does. */ +export function findHubProjectForRemoteUrl( + remoteUrl: string, + projects: ReadonlyArray, +): T | null { + const expectedKey = normalizeGitRemoteIdentity(remoteUrl); + return ( + projects.find((project) => { + const identity = project.repositoryIdentity; + if (!identity) return false; + return ( + identity.canonicalKey.toLowerCase() === expectedKey || + normalizeGitRemoteIdentity(identity.locator.remoteUrl) === expectedKey + ); + }) ?? null + ); +} + +export function hubRepositoryProjectTitle(identity: RepositoryIdentity): string { + return identity.name ?? identity.displayName ?? identity.canonicalKey; +} + +/** `project.create` input for a hub repository project. */ +export function buildHubRepositoryProjectInput(input: { + readonly projectId: ProjectId; + readonly remoteUrl: string; + readonly title?: string | undefined; + readonly defaultModelSelection: ModelSelection | null; +}) { + const repositoryIdentity = buildHubRepositoryIdentity(input.remoteUrl); + return { + projectId: input.projectId, + title: input.title?.trim() || hubRepositoryProjectTitle(repositoryIdentity), + workspaceRoot: hubProjectWorkspaceRoot(input.projectId), + createWorkspaceRootIfMissing: false, + defaultModelSelection: input.defaultModelSelection, + repositoryIdentity, + }; +} + +/** `project.create` input for a hub blank project. */ +export function buildHubBlankProjectInput(input: { + readonly projectId: ProjectId; + readonly title: string; + readonly defaultModelSelection: ModelSelection | null; +}) { + return { + projectId: input.projectId, + title: input.title.trim() || "Untitled project", + workspaceRoot: hubProjectWorkspaceRoot(input.projectId), + createWorkspaceRootIfMissing: false, + defaultModelSelection: input.defaultModelSelection, + repositoryIdentity: null, + }; +} + +function failureToast(title: string, error: unknown) { + toastManager.add( + stackedThreadToast({ + type: "error", + title, + description: error instanceof Error ? error.message : "An error occurred.", + }), + ); +} + +/** + * Adds repositories and blank projects to a hub, then lands in the project: + * its latest thread when it already existed, otherwise a new thread. + * Resolves true once the user is in the project. + */ +export function useHubAddRepository() { + const createProject = useAtomCommand(projectEnvironment.create, { reportFailure: false }); + const handleNewThread = useNewThreadHandler(); + const navigate = useNavigate(); + const sortOrder = useClientSettings((settings) => settings.sidebarThreadSortOrder); + + const openProject = useCallback( + async (projectRef: ScopedProjectRef): Promise => { + const latestThread = getLatestThreadForProject( + readThreadShells().filter((thread) => thread.environmentId === projectRef.environmentId), + projectRef.projectId, + sortOrder, + ); + if (latestThread) { + await navigate({ + to: "/$environmentId/$threadId", + params: buildThreadRouteParams( + scopeThreadRef(latestThread.environmentId, latestThread.id), + ), + }); + return true; + } + const result = await settlePromise(() => handleNewThread(projectRef)); + if (result._tag === "Failure") { + failureToast("Failed to open project", squashAtomCommandFailure(result)); + return false; + } + return true; + }, + [handleNewThread, navigate, sortOrder], + ); + + const createAndOpen = useCallback( + async ( + environmentId: EnvironmentId, + input: ReturnType, + ): Promise => { + const result = await createProject({ environmentId, input }); + if (result._tag === "Failure") { + if (!isAtomCommandInterrupted(result)) { + failureToast("Failed to add project", squashAtomCommandFailure(result)); + } + return false; + } + return openProject(scopeProjectRef(environmentId, input.projectId)); + }, + [createProject, openProject], + ); + + const addRepository = useCallback( + async (input: { + readonly environmentId: EnvironmentId; + readonly remoteUrl: string; + readonly title?: string | undefined; + readonly defaultModelSelection: ModelSelection | null; + }): Promise => { + const existing = findHubProjectForRemoteUrl( + input.remoteUrl, + readProjects().filter((project) => project.environmentId === input.environmentId), + ); + if (existing) { + return openProject(scopeProjectRef(existing.environmentId, existing.id)); + } + return createAndOpen( + input.environmentId, + buildHubRepositoryProjectInput({ + projectId: newProjectId(), + remoteUrl: input.remoteUrl, + title: input.title, + defaultModelSelection: input.defaultModelSelection, + }), + ); + }, + [createAndOpen, openProject], + ); + + const addBlankProject = useCallback( + (input: { + readonly environmentId: EnvironmentId; + readonly title: string; + readonly defaultModelSelection: ModelSelection | null; + }): Promise => + createAndOpen( + input.environmentId, + buildHubBlankProjectInput({ + projectId: newProjectId(), + title: input.title, + defaultModelSelection: input.defaultModelSelection, + }), + ), + [createAndOpen], + ); + + return { addRepository, addBlankProject }; +} diff --git a/apps/web/src/components/hub/hubCommandPalette.test.ts b/apps/web/src/components/hub/hubCommandPalette.test.ts new file mode 100644 index 000000000000..be5b2d9b2c58 --- /dev/null +++ b/apps/web/src/components/hub/hubCommandPalette.test.ts @@ -0,0 +1,39 @@ +import { EnvironmentId } from "@t3tools/contracts"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { buildHubBlankProjectSourceItem, hubAddProjectFlowCopy } from "./hubCommandPalette"; + +describe("hub add-repository palette", () => { + it("offers a blank project in place of a local folder", async () => { + const run = vi.fn(); + const item = buildHubBlankProjectSourceItem(EnvironmentId.make("environment-hub"), run); + expect(item).toMatchObject({ + kind: "action", + value: "action:add-project:environment-hub:blank", + title: "Blank project", + keepOpen: true, + }); + await item.run(); + expect(run).toHaveBeenCalledOnce(); + }); + + it("asks for a name, not a destination, when creating a blank project", () => { + expect(hubAddProjectFlowCopy({ source: "url", blankProject: true })).toEqual({ + placeholder: "Name your project", + buttonLabel: "Create", + emptyStateMessage: "Name the project and press Enter to create it.", + }); + }); + + it("adds repositories instead of cloning them", () => { + expect(hubAddProjectFlowCopy({ source: "url" })).toMatchObject({ + placeholder: "Enter Git repository URL", + buttonLabel: "Add", + }); + // Provider sources keep their own placeholder ("owner/repo" hints). + expect(hubAddProjectFlowCopy({ source: "github" })).toMatchObject({ + placeholder: null, + buttonLabel: "Add", + }); + }); +}); diff --git a/apps/web/src/components/hub/hubCommandPalette.tsx b/apps/web/src/components/hub/hubCommandPalette.tsx new file mode 100644 index 000000000000..ea6a07ea8e42 --- /dev/null +++ b/apps/web/src/components/hub/hubCommandPalette.tsx @@ -0,0 +1,56 @@ +import type { EnvironmentId } from "@t3tools/contracts"; +import { FilePlus2Icon } from "lucide-react"; + +import { ITEM_ICON_CLASS, type CommandPaletteActionItem } from "../CommandPalette.logic"; + +/** A hub has no folders to browse: its "local" source is an empty repository. */ +export function buildHubBlankProjectSourceItem( + environmentId: EnvironmentId, + run: () => void, +): CommandPaletteActionItem { + return { + kind: "action", + value: `action:add-project:${environmentId}:blank`, + searchTerms: ["blank", "empty", "new", "scratch", "project"], + title: "Blank project", + description: "Start from an empty repository", + icon: , + keepOpen: true, + run: async () => run(), + }; +} + +export interface HubAddProjectFlowCopy { + readonly placeholder: string | null; + readonly buttonLabel: string; + readonly emptyStateMessage: string; +} + +/** + * Wording for the hub's add-repository step. Nothing is cloned here — each + * thread's machine clones the repository — so the step adds and opens. + */ +export function hubAddProjectFlowCopy(flow: { + readonly source: string; + readonly blankProject?: boolean; +}): HubAddProjectFlowCopy { + if (flow.blankProject) { + return { + placeholder: "Name your project", + buttonLabel: "Create", + emptyStateMessage: "Name the project and press Enter to create it.", + }; + } + if (flow.source === "url") { + return { + placeholder: "Enter Git repository URL", + buttonLabel: "Add", + emptyStateMessage: "Enter a Git repository URL and press Enter to add it.", + }; + } + return { + placeholder: null, + buttonLabel: "Add", + emptyStateMessage: "Enter a repository path and press Enter to add it.", + }; +} From 3f3c879d0acb7ae89a638a7579d6942811caf8e1 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 01:25:24 +0000 Subject: [PATCH 44/58] chore(web): serve the landing demo as a hub with ?aldoDemoHub=1 Browser-local hub data for visual checks: a repository and a blank project, and threads whose machines are running, starting, asleep and failed, with repository refs and a GitHub repository list. The demo now also answers thread search and reports a platform, which the palette needs once there is more than one project. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/web/src/landingDemo/hubScenario.ts | 253 ++++++++++++++++++++++++ apps/web/src/landingDemo/mode.test.ts | 12 +- apps/web/src/landingDemo/mode.ts | 7 + apps/web/src/landingDemo/runtime.ts | 107 ++++++---- 4 files changed, 344 insertions(+), 35 deletions(-) create mode 100644 apps/web/src/landingDemo/hubScenario.ts diff --git a/apps/web/src/landingDemo/hubScenario.ts b/apps/web/src/landingDemo/hubScenario.ts new file mode 100644 index 000000000000..4765ec457f17 --- /dev/null +++ b/apps/web/src/landingDemo/hubScenario.ts @@ -0,0 +1,253 @@ +/** + * The landing demo served as a hub (`?aldoDemo=1&aldoDemoHub=1`): repository + * and blank projects, and threads whose machines are running, starting, + * asleep, and failed. Browser-local data only; nothing here reaches a server. + */ +import { + EventId, + MessageId, + ProjectId, + ThreadId, + TurnId, + type OrchestrationProjectShell, + type OrchestrationThread, + type SourceControlDiscoveryResult, + type SourceControlRepositorySummary, + type ThreadMachineStatus, + type VcsListRefsResult, +} from "@t3tools/contracts"; +import * as Option from "effect/Option"; + +const MINUTE_MS = 60_000; + +function isoMinutesAgo(minutes: number): string { + return new Date(Date.now() - minutes * MINUTE_MS).toISOString(); +} + +export const hubDemoBlankProjectId = ProjectId.make("aldo-demo-scratchpad"); + +export function hubDemoProjects( + primary: OrchestrationProjectShell, +): ReadonlyArray { + return [ + { + ...primary, + workspaceRoot: `/workspace/p/${primary.id}`, + repositoryIdentity: { + canonicalKey: "github.com/aldo-demo/northstar", + locator: { + source: "git-remote", + remoteName: "origin", + remoteUrl: "https://github.com/aldo-demo/northstar", + }, + displayName: "aldo-demo/northstar", + provider: "github", + owner: "aldo-demo", + name: "northstar", + }, + }, + { + ...primary, + id: hubDemoBlankProjectId, + title: "Scratchpad", + workspaceRoot: `/workspace/p/${hubDemoBlankProjectId}`, + repositoryIdentity: null, + }, + ]; +} + +export interface HubDemoThread { + readonly thread: OrchestrationThread; + readonly machine: ThreadMachineStatus; +} + +function userMessage(threadId: string, text: string, createdAt: string, turnId: TurnId) { + return { + id: MessageId.make(`${threadId}-user`), + role: "user" as const, + text, + attachments: [], + turnId, + streaming: false, + createdAt, + updatedAt: createdAt, + }; +} + +/** Threads beside the demo's own, one per machine state worth seeing. */ +export function hubDemoThreads(base: OrchestrationThread): ReadonlyArray { + const make = (input: { + readonly id: string; + readonly title: string; + readonly branch: string; + readonly minutesAgo: number; + readonly prompt: string; + readonly reply: string | null; + readonly turnState: "running" | "completed" | "error"; + readonly sessionStatus: "running" | "ready" | "error"; + readonly lastError?: string; + readonly activities?: OrchestrationThread["activities"]; + readonly machine: Omit; + }): HubDemoThread => { + const threadId = ThreadId.make(input.id); + const turnId = TurnId.make(`${input.id}-turn`); + const requestedAt = isoMinutesAgo(input.minutesAgo); + const completedAt = input.turnState === "running" ? null : isoMinutesAgo(input.minutesAgo - 1); + const replyId = MessageId.make(`${input.id}-reply`); + return { + machine: { ...input.machine, updatedAt: requestedAt }, + thread: { + ...base, + id: threadId, + title: input.title, + branch: input.branch, + worktreePath: `/workspace/t/${input.id}`, + createdAt: requestedAt, + updatedAt: completedAt ?? requestedAt, + messages: [ + userMessage(input.id, input.prompt, requestedAt, turnId), + ...(input.reply && completedAt + ? [ + { + id: replyId, + role: "assistant" as const, + text: input.reply, + turnId, + streaming: false, + createdAt: completedAt, + updatedAt: completedAt, + }, + ] + : []), + ], + activities: input.activities ?? [], + latestTurn: { + turnId, + state: input.turnState, + requestedAt, + startedAt: requestedAt, + completedAt, + assistantMessageId: input.reply && completedAt ? replyId : null, + }, + session: base.session + ? { + ...base.session, + threadId, + status: input.sessionStatus, + activeTurnId: input.turnState === "running" ? turnId : null, + lastError: input.lastError ?? null, + updatedAt: completedAt ?? requestedAt, + } + : null, + }, + }; + }; + + return [ + make({ + id: "aldo-demo-login-redirect", + title: "Fix the login redirect loop", + branch: "aldo/fix-login-redirect", + minutesAgo: 0.4, + prompt: "Signing in on Safari bounces back to /login forever. Find out why and fix it.", + reply: null, + turnState: "running", + sessionStatus: "running", + machine: { state: "starting", detail: "Cloning aldo-demo/northstar" }, + }), + make({ + id: "aldo-demo-pricing", + title: "Add a pricing page", + branch: "aldo/pricing-page", + minutesAgo: 95, + prompt: "Add a pricing page with three tiers and a monthly/yearly toggle.", + reply: + "Added `pricing.html` with Starter, Team, and Enterprise tiers, a monthly/yearly toggle, and a link from the nav. The build and lint both pass.", + turnState: "completed", + sessionStatus: "ready", + machine: { state: "paused", detail: null }, + }), + make({ + id: "aldo-demo-react-upgrade", + title: "Upgrade the site to React 20", + branch: "aldo/react-20", + minutesAgo: 12, + prompt: "Upgrade the marketing site to React 20 and fix anything that breaks.", + reply: null, + turnState: "error", + sessionStatus: "error", + lastError: "Thread machine is unavailable", + activities: [ + { + id: EventId.make("aldo-demo-react-upgrade-failed"), + tone: "error", + kind: "thread-machine.failed", + summary: "Thread machine is unavailable", + payload: { detail: "No machine capacity in this region right now." }, + turnId: null, + createdAt: isoMinutesAgo(11), + }, + ], + machine: { state: "failed", detail: "No machine capacity in this region right now." }, + }), + ]; +} + +export const hubDemoRefs: VcsListRefsResult = { + refs: [ + { name: "main", current: false, isDefault: true, worktreePath: null }, + { name: "aldo/pricing-page", current: false, isDefault: false, worktreePath: null }, + { name: "release/2026-09", current: false, isDefault: false, worktreePath: null }, + ], + isRepo: true, + hasPrimaryRemote: true, + nextCursor: null, + totalCount: 3, +}; + +const githubDiscovery = { + kind: "github" as const, + label: "GitHub", + status: "available" as const, + version: Option.none(), + installHint: "Connect GitHub in Aldo.", + detail: Option.none(), + auth: { + status: "authenticated" as const, + account: Option.some("aldo-demo"), + host: Option.some("github.com"), + detail: Option.none(), + }, +}; + +export const hubDemoDiscovery: SourceControlDiscoveryResult = { + versionControlSystems: [], + sourceControlProviders: [githubDiscovery], +}; + +function repository( + name: string, + description: string, + flags: Partial> = {}, +): SourceControlRepositorySummary { + return { + provider: "github", + nameWithOwner: `aldo-demo/${name}`, + name, + owner: "aldo-demo", + url: `https://github.com/aldo-demo/${name}`, + sshUrl: `git@github.com:aldo-demo/${name}.git`, + description, + isPrivate: flags.isPrivate ?? true, + isArchived: false, + isFork: flags.isFork ?? false, + }; +} + +export const hubDemoRepositories: ReadonlyArray = [ + repository("northstar", "Marketing site and docs"), + repository("api", "Public API and webhooks"), + repository("mobile", "iOS and Android apps"), + repository("design-system", "Shared components and tokens", { isPrivate: false }), + repository("infra", "Terraform and deploy scripts"), +]; diff --git a/apps/web/src/landingDemo/mode.test.ts b/apps/web/src/landingDemo/mode.test.ts index 245739e5b427..cd4ba8294a6a 100644 --- a/apps/web/src/landingDemo/mode.test.ts +++ b/apps/web/src/landingDemo/mode.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vite-plus/test"; -import { isLandingDemo } from "./mode"; +import { isLandingDemo, isLandingDemoHub } from "./mode"; describe("landing demo mode", () => { it("is enabled only by the explicit anonymous demo flag", () => { @@ -9,3 +9,13 @@ describe("landing demo mode", () => { expect(isLandingDemo(new URL("https://aldoagent.com/demo/t3?aldoDemo=0"))).toBe(false); }); }); + +describe("landing demo hub mode", () => { + it("needs both the demo flag and the hub flag", () => { + expect( + isLandingDemoHub(new URL("https://aldoagent.com/demo/t3?aldoDemo=1&aldoDemoHub=1")), + ).toBe(true); + expect(isLandingDemoHub(new URL("https://aldoagent.com/demo/t3?aldoDemo=1"))).toBe(false); + expect(isLandingDemoHub(new URL("https://aldoagent.com/demo/t3?aldoDemoHub=1"))).toBe(false); + }); +}); diff --git a/apps/web/src/landingDemo/mode.ts b/apps/web/src/landingDemo/mode.ts index b0c1b7a3f71e..270fe706ec65 100644 --- a/apps/web/src/landingDemo/mode.ts +++ b/apps/web/src/landingDemo/mode.ts @@ -13,3 +13,10 @@ export function isLandingDemo(url?: URL): boolean { } return resolved?.searchParams.get("aldoDemo") === "1"; } + +/** The demo served as a hub: repository projects, each thread on its own machine. */ +export function isLandingDemoHub(url?: URL): boolean { + if (!isLandingDemo(url)) return false; + const resolved = url ?? new URL(window.location.href, "http://localhost"); + return resolved.searchParams.get("aldoDemoHub") === "1"; +} diff --git a/apps/web/src/landingDemo/runtime.ts b/apps/web/src/landingDemo/runtime.ts index f659d1ab400a..653e23b69e9c 100644 --- a/apps/web/src/landingDemo/runtime.ts +++ b/apps/web/src/landingDemo/runtime.ts @@ -15,6 +15,7 @@ import { type OrchestrationShellStreamItem, type OrchestrationThread, type OrchestrationThreadDetailSnapshot, + type OrchestrationThreadShell, type OrchestrationThreadStreamItem, type ServerConfig, } from "@t3tools/contracts"; @@ -30,10 +31,18 @@ import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; +import { + hubDemoDiscovery, + hubDemoProjects, + hubDemoRefs, + hubDemoRepositories, + hubDemoThreads, +} from "./hubScenario"; import { LANDING_DEMO_ENVIRONMENT_ID, LANDING_DEMO_PROJECT_ID, LANDING_DEMO_THREAD_ID, + isLandingDemoHub, } from "./mode"; export const demoEnvironmentId = EnvironmentId.make(LANDING_DEMO_ENVIRONMENT_ID); @@ -41,6 +50,7 @@ export const demoProjectId = ProjectId.make(LANDING_DEMO_PROJECT_ID); export const demoThreadId = ThreadId.make(LANDING_DEMO_THREAD_ID); const demoProviderId = ProviderInstanceId.make("codex"); const STARTED_AT = "2026-07-31T12:00:00.000Z"; +const hubDemo = isLandingDemoHub(); const initialFiles = { "index.html": ` @@ -91,8 +101,8 @@ let thread: OrchestrationThread = { modelSelection: { instanceId: demoProviderId, model: "openai/gpt-oss-20b:free" }, runtimeMode: "full-access", interactionMode: "default", - branch: "main", - worktreePath: null, + branch: hubDemo ? "aldo/homepage" : "main", + worktreePath: hubDemo ? `/workspace/t/${LANDING_DEMO_THREAD_ID}` : null, latestTurn: null, createdAt: STARTED_AT, updatedAt: STARTED_AT, @@ -127,42 +137,54 @@ const project: OrchestrationProjectShell = { updatedAt: STARTED_AT, }; -function threadShell() { +const hubThreads = hubDemo ? hubDemoThreads(thread) : []; + +function threadShell( + source: OrchestrationThread = thread, + machine: OrchestrationThreadShell["machine"] = hubDemo + ? { state: "running", detail: null, updatedAt: STARTED_AT } + : undefined, +): OrchestrationThreadShell { return { - id: thread.id, - projectId: thread.projectId, - title: thread.title, - modelSelection: thread.modelSelection, - runtimeMode: thread.runtimeMode, - interactionMode: thread.interactionMode, - branch: thread.branch, - worktreePath: thread.worktreePath, - latestTurn: thread.latestTurn, - createdAt: thread.createdAt, - updatedAt: thread.updatedAt, - archivedAt: thread.archivedAt, - settledOverride: thread.settledOverride, - settledAt: thread.settledAt, - session: thread.session, + id: source.id, + projectId: source.projectId, + title: source.title, + modelSelection: source.modelSelection, + runtimeMode: source.runtimeMode, + interactionMode: source.interactionMode, + branch: source.branch, + worktreePath: source.worktreePath, + latestTurn: source.latestTurn, + createdAt: source.createdAt, + updatedAt: source.updatedAt, + archivedAt: source.archivedAt, + settledOverride: source.settledOverride, + settledAt: source.settledAt, + session: source.session, latestUserMessageAt: - thread.messages.findLast((message) => message.role === "user")?.createdAt ?? null, + source.messages.findLast((message) => message.role === "user")?.createdAt ?? null, hasPendingApprovals: false, hasPendingUserInput: false, hasActionableProposedPlan: false, - } as const; + ...(machine ? { machine } : {}), + }; } function shellSnapshot(): OrchestrationShellSnapshot { return { snapshotSequence, - projects: [project], - threads: [threadShell()], + projects: hubDemo ? [...hubDemoProjects(project)] : [project], + threads: [ + threadShell(), + ...hubThreads.map((entry) => threadShell(entry.thread, entry.machine)), + ], updatedAt: thread.updatedAt, }; } -function threadSnapshot(): OrchestrationThreadDetailSnapshot { - return { snapshotSequence, thread }; +function threadSnapshot(threadId?: string): OrchestrationThreadDetailSnapshot { + const hubThread = hubThreads.find((entry) => entry.thread.id === threadId)?.thread; + return { snapshotSequence, thread: hubThread ?? thread }; } function broadcastThread() { @@ -184,10 +206,12 @@ function callbackStream(listeners: Set<(item: T) => void>) { ); } -function subscribeThread(): Stream.Stream { +function subscribeThread(input?: { + readonly threadId?: string; +}): Stream.Stream { return Stream.concat( Stream.fromIterable([ - { kind: "snapshot", snapshot: threadSnapshot() }, + { kind: "snapshot", snapshot: threadSnapshot(input?.threadId) }, { kind: "synchronized" }, ]), callbackStream(threadListeners), @@ -208,8 +232,12 @@ export const demoServerConfig = { environment: { id: demoEnvironmentId, label: "Aldo browser demo", + platform: { os: "linux", arch: "x64" }, serverVersion: "0.0.28", - capabilities: { connectionProbe: false }, + capabilities: { + connectionProbe: false, + ...(hubDemo ? { repositoryIdentity: true, threadMachines: true } : {}), + }, }, auth: { mode: "none" }, cwd: "/demo/northstar", @@ -523,14 +551,25 @@ const client = new Proxy( [ORCHESTRATION_WS_METHODS.subscribeThread]: subscribeThread, [ORCHESTRATION_WS_METHODS.dispatchCommand]: dispatchCommand, [WS_METHODS.vcsListRefs]: () => - Effect.succeed({ - refs: [], - isRepo: true, - hasPrimaryRemote: false, - nextCursor: null, - totalCount: 0, - }), + Effect.succeed( + hubDemo + ? hubDemoRefs + : { + refs: [], + isRepo: true, + hasPrimaryRemote: false, + nextCursor: null, + totalCount: 0, + }, + ), + ...(hubDemo + ? { + [WS_METHODS.serverDiscoverSourceControl]: () => Effect.succeed(hubDemoDiscovery), + [WS_METHODS.sourceControlListRepositories]: () => Effect.succeed(hubDemoRepositories), + } + : {}), [ORCHESTRATION_WS_METHODS.getFullThreadDiff]: () => Effect.succeed({ patch: "", files: [] }), + [ORCHESTRATION_WS_METHODS.searchThreads]: () => Effect.succeed({ matches: [] }), [ORCHESTRATION_WS_METHODS.getTurnDiff]: () => Effect.succeed({ patch: "", files: [] }), } as Record, { From 0a407a45d534c783a945458bcbe5a9210b61ee76 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 01:25:26 +0000 Subject: [PATCH 45/58] docs: describe the hub-mode web client Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/internals/thread-machines.md | 35 ++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/internals/thread-machines.md b/docs/internals/thread-machines.md index 0829014e914e..127eab6e5941 100644 --- a/docs/internals/thread-machines.md +++ b/docs/internals/thread-machines.md @@ -348,6 +348,35 @@ mode only through the `serverModeHooks.ts` references (deterministic ingestion i behavior. `t3 runner` is composed in `runner/RunnerServer.ts` without orchestration, projections or the client API. +## Web client + +The web client decides hub mode per environment, at render time, from +`capabilities.threadMachines` (`apps/web/src/hubMode.ts`); before a managed primary's config +arrives it trusts the managed bootstrap (`serverMode: "hub"`) or the cached environment +descriptor, so a hub never flashes local-checkout controls. On a hub: + +- New threads have no local/worktree choice. A draft shows "New machine" and a base branch + listed from the repository (`vcs.listRefs` on the project root); with none picked it sends + `prepareWorktree.baseBranch: "HEAD"`. Drafts never ask the hub for status, files, + terminals or scripts, since their machine does not exist yet. Blank projects send no + `prepareWorktree`. +- Projects are added as repositories (URL, provider lookup, or the GitHub list) or blank + projects: `project.create` with `repositoryIdentity` and the virtual root, deduplicated by + remote. Nothing is cloned by the client. +- The thread shell's `machine` drives a header status pill, the timeline's waking row + (with the platform's detail), the composer notice (asleep, or failed with a retry that + re-drives the failed turn), and the sidebar (starting reads as working, failed as failed). + `thread-machine.*` activities stay out of the work log unless they are errors. +- Surfaces that need a checkout adapt: no Open In, Copy path, t3.json reads or new-thread + workspace settings; files show "Machine asleep" while the machine sleeps; the diff panel + opens on the latest captured turn; previews and the machine browser use the bootstrap's + per-thread templates (`threadPreviewUrlTemplate`, `threadBrowserUrlTemplate`); provider + sign-in opens the session's `workspaceBrowserUrl` when present. The managed workspace + status, `/settings/workspace` and the durable wake queue are skipped. + +An explicit wake (`threadMachines.wake`) is not wired yet: the client hides that affordance +(`components/hub/threadMachineActions.ts`). + ## Gaps Not available on a hub (typed errors, never the hub's disk): filesystem browsing for the @@ -360,9 +389,9 @@ and pull-request listing (which needs provider credentials). Not built yet: -- No client records a project's repository identity; the platform (or the import, from a - checkout) must dispatch `project.create` / `project.meta.update` with `repositoryIdentity`. - A hub without a database records none. +- Only the web client's add-repository flow and the import (from a checkout) record a + project's repository identity (`project.create` / `project.meta.update` with + `repositoryIdentity`). A hub without a database records none. - A hub behind a runner's `firstRetainedSequence` (more than 200,000 unacknowledged events) only logs the gap; resynchronizing from `readThread` is future work. - Attachment bytes live in `bytea` within the upload limits; object storage can replace the From e726e0f0c2bd691556597011f53d61d9f9dde81f Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 01:26:01 +0000 Subject: [PATCH 46/58] feat(server): run provider sign-in on the reserved sign-in machine in hub mode On a hub, server.startAuthConnector (and get, submit, cancel) runs T3's auth connector on the machine of the reserved thread aldo-provider-sign-in (blank project, no repository) through new runner.auth.* RPCs: the session answers at once while the machine starts, then mirrors the runner's prompts, URLs and fields, and stays verifying until the platform lists the provider's stored sign-in at a higher version; the provider's status is then refreshed on the machine and the sign-in machine is reported idle. Browser flows carry workspaceBrowserUrl from T3CODE_HUB_THREAD_BROWSER_URL_TEMPLATE (default /_devpc/threads/{threadId}/browser). New RPCs server.listProviderSignIns and server.signOutProvider read and delete stored sign-ins; standalone servers answer them with AuthConnectorError and keep their local auth connector. Runners serve sign-in only on the sign-in machine, and the remote provider driver never starts agent sessions there. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/auth/RpcAuthorization.ts | 2 + .../src/authConnector/AuthConnectorManager.ts | 10 + apps/server/src/cli/config.ts | 6 + apps/server/src/config.ts | 6 + apps/server/src/hub/HubLayers.ts | 2 + apps/server/src/hub/HubProviderSignIn.test.ts | 220 +++++++++ apps/server/src/hub/HubProviderSignIn.ts | 425 ++++++++++++++++++ apps/server/src/hub/HubProviderSnapshots.ts | 3 + .../src/hub/RemoteProviderDriver.test.ts | 10 + apps/server/src/hub/RemoteProviderDriver.ts | 23 +- apps/server/src/runner/RunnerHandlers.test.ts | 14 + apps/server/src/runner/RunnerHandlers.ts | 43 ++ apps/server/src/server.test.ts | 12 +- apps/server/src/serverModeHooks.ts | 32 ++ apps/server/src/ws.ts | 60 ++- packages/contracts/src/authConnector.ts | 33 ++ packages/contracts/src/rpc.ts | 21 + packages/contracts/src/runner.ts | 43 ++ 18 files changed, 953 insertions(+), 12 deletions(-) create mode 100644 apps/server/src/hub/HubProviderSignIn.test.ts create mode 100644 apps/server/src/hub/HubProviderSignIn.ts diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index 3575ae6fa1e0..bd3fd1f22152 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -44,6 +44,8 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.serverGetAuthConnector]: AuthOrchestrationReadScope, [WS_METHODS.serverSubmitAuthConnector]: AuthOrchestrationOperateScope, [WS_METHODS.serverCancelAuthConnector]: AuthOrchestrationOperateScope, + [WS_METHODS.serverListProviderSignIns]: AuthOrchestrationReadScope, + [WS_METHODS.serverSignOutProvider]: AuthOrchestrationOperateScope, [WS_METHODS.serverGetTraceDiagnostics]: AuthOrchestrationReadScope, [WS_METHODS.serverGetProcessDiagnostics]: AuthOrchestrationReadScope, [WS_METHODS.serverGetProcessResourceHistory]: AuthOrchestrationReadScope, diff --git a/apps/server/src/authConnector/AuthConnectorManager.ts b/apps/server/src/authConnector/AuthConnectorManager.ts index a0231bfbd61f..1f526392e4f6 100644 --- a/apps/server/src/authConnector/AuthConnectorManager.ts +++ b/apps/server/src/authConnector/AuthConnectorManager.ts @@ -1259,6 +1259,16 @@ async function submitBitbucket( scheduleSessionRemoval(session); } +/** + * The flow a connector method uses, known before its login command starts + * (a hub shows it while a machine starts the command). Bitbucket asks for a + * secret; anything without a launch spec defaults to a browser flow. + */ +export function authConnectorFlow(input: AuthConnectorStartInput): AuthConnectorSession["flow"] { + if (input.connector === "bitbucket") return "secret"; + return launchSpec(input)?.flow ?? "browser"; +} + export const start = Effect.fn("AuthConnectorManager.start")(function* ( input: AuthConnectorStartInput, ): Effect.fn.Return< diff --git a/apps/server/src/cli/config.ts b/apps/server/src/cli/config.ts index 95f28c549b3c..f71d7e1f928e 100644 --- a/apps/server/src/cli/config.ts +++ b/apps/server/src/cli/config.ts @@ -167,6 +167,10 @@ const EnvServerConfig = Config.all({ Config.option, Config.map(Option.getOrUndefined), ), + hubThreadBrowserUrlTemplate: Config.string("T3CODE_HUB_THREAD_BROWSER_URL_TEMPLATE").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), runnerUrl: Config.string("T3CODE_RUNNER_URL").pipe( Config.option, Config.map(Option.getOrUndefined), @@ -278,6 +282,7 @@ const resolveHubServerConfig = (env: { readonly hubMachinesToken?: string | undefined; readonly hubPublicUrl?: string | undefined; readonly hubCheckoutRoot?: string | undefined; + readonly hubThreadBrowserUrlTemplate?: string | undefined; readonly runnerUrl?: string | undefined; }) => Effect.gen(function* () { @@ -308,6 +313,7 @@ const resolveHubServerConfig = (env: { machinesToken: env.hubMachinesToken, publicUrl: env.hubPublicUrl, checkoutRoot: env.hubCheckoutRoot, + threadBrowserUrlTemplate: env.hubThreadBrowserUrlTemplate, } satisfies ServerConfig.HubServerConfig; }); diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index d46f09b60194..a3772afc6e52 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -86,6 +86,12 @@ export interface HubServerConfig { * which only a runner on the same host can reach. */ readonly publicUrl?: string | undefined; + /** + * Same-origin URL template of a thread machine's browser page, with + * `{threadId}`; provider sign-in flows that finish in a browser on the + * sign-in machine link to it. Defaults to `/_devpc/threads/{threadId}/browser`. + */ + readonly threadBrowserUrlTemplate?: string | undefined; /** * Root of thread checkouts on thread machines (`/workspace/t` by default). * Hub and machines must agree on it; only tests and local development diff --git a/apps/server/src/hub/HubLayers.ts b/apps/server/src/hub/HubLayers.ts index cdab10d50d5c..36c2bad96a37 100644 --- a/apps/server/src/hub/HubLayers.ts +++ b/apps/server/src/hub/HubLayers.ts @@ -44,6 +44,7 @@ import { hubWorkspaceFileSystemLayer, hubWorkspacePathsLayer, } from "./HubWorkspace.ts"; +import * as HubProviderSignIn from "./HubProviderSignIn.ts"; import * as HubProviderSnapshots from "./HubProviderSnapshots.ts"; import * as MachineDirectory from "./MachineDirectory.ts"; import { makeRemoteProviderDriver } from "./RemoteProviderDriver.ts"; @@ -153,6 +154,7 @@ export const hubReactorHooksLayer = Layer.mergeAll( hubCheckoutGitProbeLayer, HubThreadCheckouts.layer, ThreadMachineControls.layer, + HubProviderSignIn.layer, ); /** Background work that needs the reactors: delivery acks, resume, machine release. */ diff --git a/apps/server/src/hub/HubProviderSignIn.test.ts b/apps/server/src/hub/HubProviderSignIn.test.ts new file mode 100644 index 000000000000..8b404a62d655 --- /dev/null +++ b/apps/server/src/hub/HubProviderSignIn.test.ts @@ -0,0 +1,220 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { it } from "@effect/vitest"; +import { + type AuthConnectorSession, + ProviderDriverKind, + ProviderInstanceId, + type ServerProvider, +} from "@t3tools/contracts"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { describe, expect } from "vite-plus/test"; + +import { ServerConfig } from "../config.ts"; +import { HubThreadMachineStateSqliteLive } from "../persistence/Layers/HubThreadMachineState.ts"; +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; +import { ProviderSignInControls } from "../serverModeHooks.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; +import * as HubProviderSignIn from "./HubProviderSignIn.ts"; +import * as HubProviderSnapshots from "./HubProviderSnapshots.ts"; +import { MachineDirectory, makeFakeMachineDirectory } from "./MachineDirectory.ts"; +import { make as makePool, RunnerConnectionPool } from "./RunnerConnectionPool.ts"; +import { fakeRunner, fakeRunnerHello, serveRunner } from "./testUtils/runnerServer.ts"; + +const claude = ProviderDriverKind.make("claudeAgent"); +const claudeInstance = ProviderInstanceId.make("claudeAgent"); + +const remoteSession = (patch: Partial): AuthConnectorSession => ({ + id: "runner-session-1", + connector: "claude", + method: "account", + status: "waiting", + flow: "code", + stage: "authorize", + message: "Open the link, then paste the code.", + verificationUrl: "https://claude.ai/oauth/authorize?code=true", + userCode: null, + fields: [{ key: "callback", label: "Code", type: "text" }], + expiresAt: null, + ...patch, +}); + +const authenticated: ServerProvider = { + instanceId: claudeInstance, + driver: claude, + enabled: true, + installed: true, + version: "2.1.220", + status: "ready", + auth: { status: "authenticated", email: "user@example.com" }, + checkedAt: "2026-09-26T00:00:00.000Z", + models: [], + slashCommands: [], + skills: [], +}; + +const StoresLive = Layer.mergeAll( + HubThreadMachineStateSqliteLive, + ServerSettingsService.layerTest(), +).pipe(Layer.provideMerge(SqlitePersistenceMemory), Layer.provideMerge(NodeServices.layer)); + +/** + * A sign-in machine whose runner reports `waiting` until the credential is + * submitted (or at once, with `succeedWithoutInput`), then `succeeded`. + */ +const setup = ( + waiting: AuthConnectorSession, + options: { + readonly storeTimeout?: Duration.Input; + readonly succeedWithoutInput?: boolean; + } = {}, +) => + Effect.gen(function* () { + const calls: Array = []; + let submitted = options.succeedWithoutInput === true; + const succeeded = { ...waiting, status: "succeeded" as const, stage: "complete" as const }; + const runner = yield* serveRunner( + fakeRunner({ + "runner.hello": (input) => + Effect.succeed( + fakeRunnerHello({ threadId: input.threadId, instances: [claudeInstance] }), + ), + "runner.provider.configure": ({ instances }) => + Effect.sync(() => { + calls.push(`configure ${Object.keys(instances).join(",")}`); + return { instances: [claudeInstance] }; + }), + "runner.provider.getCapabilities": () => + Effect.succeed({ snapshot: authenticated, sessionModelSwitch: "in-session" as const }), + "runner.auth.start": (input) => + Effect.sync(() => { + calls.push(`start ${input.connector}`); + return waiting; + }), + "runner.auth.get": () => Effect.sync(() => (submitted ? succeeded : waiting)), + "runner.auth.submit": ({ sessionId, values }) => + Effect.sync(() => { + calls.push(`submit ${sessionId} ${Object.keys(values).join(",")}`); + submitted = true; + return remoteSession({ status: "starting", stage: "verifying", fields: [] }); + }), + "runner.auth.cancel": ({ sessionId }) => + Effect.sync(() => { + calls.push(`cancel ${sessionId}`); + return remoteSession({ status: "cancelled", stage: "error" }); + }), + }), + ); + const fake = yield* makeFakeMachineDirectory({ + onWake: () => ({ state: "running", runnerUrl: runner.url }), + }); + const snapshots = yield* HubProviderSnapshots.make; + const context = yield* Layer.build( + Layer.effect( + ProviderSignInControls, + HubProviderSignIn.make({ + pollInterval: "10 millis", + storePollInterval: "10 millis", + storeTimeout: options.storeTimeout ?? Duration.seconds(5), + }), + ).pipe( + Layer.provideMerge( + Layer.effect( + RunnerConnectionPool, + makePool({ wakePollInterval: "5 millis", idleCheckInterval: "1 hour" }), + ), + ), + Layer.provideMerge(Layer.succeed(MachineDirectory, fake.directory)), + Layer.provide(Layer.succeed(HubProviderSnapshots.HubProviderSnapshots, snapshots)), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-sign-in-" })), + ), + ); + const signIn = (yield* ProviderSignInControls.pipe(Effect.provide(context)))!; + const waitFor = (id: string, predicate: (session: AuthConnectorSession) => boolean) => + Effect.repeat(signIn.get(id), { until: predicate }).pipe(Effect.timeout("5 seconds")); + return { signIn, fake, calls, snapshots, waitFor }; + }); + +describe("hub provider sign-in", () => { + it.live("runs the login on the sign-in machine and completes once the sign-in is stored", () => + Effect.gen(function* () { + const { signIn, fake, calls, snapshots, waitFor } = yield* setup(remoteSession({})); + const started = yield* signIn.start({ connector: "claude", method: "account" }); + expect(started).toMatchObject({ status: "starting", stage: "preparing", flow: "code" }); + + const waiting = yield* waitFor(started.id, (session) => session.status === "waiting"); + expect(waiting).toMatchObject({ + id: started.id, + flow: "code", + verificationUrl: "https://claude.ai/oauth/authorize?code=true", + workspaceBrowserUrl: null, + }); + yield* signIn.submit({ sessionId: started.id, values: { callback: "code#state" } }); + + // The login succeeded on the machine; the upload has not landed yet. + const verifying = yield* waitFor( + started.id, + (session) => session.message === "Saving your sign-in…", + ); + expect(verifying).toMatchObject({ status: "starting", stage: "verifying" }); + yield* fake.setProviderHome("claude", 1); + const done = yield* waitFor(started.id, (session) => session.status !== "starting"); + expect(done).toMatchObject({ status: "succeeded", stage: "complete", message: "Signed in." }); + + expect(calls).toContain("configure claudeAgent"); + expect(calls).toContain("start claude"); + expect(calls).toContain("submit runner-session-1 callback"); + const [status] = yield* snapshots.all; + expect(status?.auth.status).toBe("authenticated"); + const directoryCalls = (yield* fake.calls).map((call) => call.method); + expect(directoryCalls[0]).toBe("ensure"); + expect(directoryCalls.at(-1)).toBe("idle"); + }).pipe(Effect.scoped, Effect.provide(StoresLive)), + ); + + it.live("links browser flows to the machine's browser and fails when nothing is stored", () => + Effect.gen(function* () { + const { signIn, waitFor } = yield* setup( + remoteSession({ connector: "cursor", flow: "browser", fields: [] }), + { storeTimeout: "100 millis", succeedWithoutInput: true }, + ); + const started = yield* signIn.start({ connector: "cursor", method: "account" }); + expect(started).toMatchObject({ flow: "browser", workspaceBrowserUrl: null }); + const browser = yield* waitFor(started.id, (session) => session.workspaceBrowserUrl != null); + expect(browser.workspaceBrowserUrl).toBe("/_devpc/threads/aldo-provider-sign-in/browser"); + const failed = yield* waitFor(started.id, (session) => session.status === "failed"); + expect(failed.message).toContain("was not saved"); + }).pipe(Effect.scoped, Effect.provide(StoresLive)), + ); + + it.live("cancels on the machine, signs out, lists stored sign-ins, and refuses others", () => + Effect.gen(function* () { + const { signIn, fake, calls, waitFor } = yield* setup(remoteSession({})); + const started = yield* signIn.start({ connector: "claude", method: "account" }); + yield* waitFor(started.id, (session) => session.status === "waiting"); + const cancelled = yield* signIn.cancel(started.id); + expect(cancelled.status).toBe("cancelled"); + expect(calls).toContain("cancel runner-session-1"); + + yield* fake.setProviderHome("claude", 4); + yield* fake.setProviderHome("prime", 2); + expect(yield* signIn.list).toEqual({ + signIns: [ + { connector: "claude", version: 4, updatedAt: null }, + { connector: "prime-agent", version: 2, updatedAt: null }, + ], + }); + expect(yield* signIn.signOut({ connector: "claude" })).toEqual({ + connector: "claude", + signedOut: true, + }); + expect(yield* fake.platformCalls).toContain("deleteProviderHome claude"); + + const github = yield* signIn + .start({ connector: "github", method: "token" }) + .pipe(Effect.flip); + expect(github._tag).toBe("AuthConnectorError"); + }).pipe(Effect.scoped, Effect.provide(StoresLive)), + ); +}); diff --git a/apps/server/src/hub/HubProviderSignIn.ts b/apps/server/src/hub/HubProviderSignIn.ts new file mode 100644 index 000000000000..8703d62f0e16 --- /dev/null +++ b/apps/server/src/hub/HubProviderSignIn.ts @@ -0,0 +1,425 @@ +/** + * HubProviderSignIn - subscription sign-in when the server is a hub. + * + * A hub runs no provider CLI. Starting a provider sign-in in Settings runs the + * provider's own login command, through T3's unchanged auth connector, on the + * machine of the reserved thread `aldo-provider-sign-in` (a blank project, no + * repository, checkout `/workspace/t/aldo-provider-sign-in`): + * + * 1. `start` answers at once with a session that is starting a machine; a + * background fiber records the provider's stored sign-in version, ensures + * and wakes the sign-in machine, pushes the instance's settings, and + * starts the connector there (`runner.auth.start`). The session mirrors the + * runner's session (prompts, URLs, codes, fields) as it progresses; + * `submit` and `cancel` are forwarded. Flows that finish in a browser on the + * machine carry `workspaceBrowserUrl`. + * 2. When the login command succeeds, the session stays `verifying` until the + * platform lists the provider's stored sign-in at a higher version (the + * machine's supervisor uploads the CLI's files within about 20 s), then + * succeeds; the instance's provider status is refreshed on the machine. + * 3. Once no sign-in is active the sign-in machine is reported idle. Agent + * sessions never run on it (the remote provider driver refuses them). + * + * Sign-out deletes the stored sign-in (`DELETE provider-homes/{provider}`); + * running machines drop the provider's files within about 15 s. + * + * @module hub/HubProviderSignIn + */ +import { + AuthConnectorError, + type AuthConnectorKind, + type AuthConnectorSession, + type AuthConnectorStartInput, + defaultInstanceIdForDriver, + ProjectId, + PROVIDER_SIGN_IN_THREAD_ID, + ProviderDriverKind, + type ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import { + type ProviderHomeId, + RUNNER_PROTOCOL_PROVIDER_SETTINGS, + ThreadMachineUnavailableError, +} from "@t3tools/contracts/runner"; +import * as Clock from "effect/Clock"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import { authConnectorFlow } from "../authConnector/AuthConnectorManager.ts"; +import { ServerConfig } from "../config.ts"; +import { ProviderSignInControls, type ProviderSignInControlsShape } from "../serverModeHooks.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; +import { HubProviderSnapshots } from "./HubProviderSnapshots.ts"; +import { MachineDirectory } from "./MachineDirectory.ts"; +import { effectiveInstanceSettings } from "./RemoteProviderDriver.ts"; +import { + RunnerConnectionPool, + type RunnerConnection, + type ThreadMachineContext, +} from "./RunnerConnectionPool.ts"; + +export const SIGN_IN_THREAD_ID = ThreadId.make(PROVIDER_SIGN_IN_THREAD_ID); +const SIGN_IN_CONTEXT: ThreadMachineContext = { + projectId: ProjectId.make(PROVIDER_SIGN_IN_THREAD_ID), + repository: null, + branch: null, + providerInstanceId: null, +}; +export const DEFAULT_THREAD_BROWSER_URL_TEMPLATE = "/_devpc/threads/{threadId}/browser"; + +/** Provider connectors whose sign-in is stored for thread machines, and their drivers. */ +const PROVIDER_CONNECTORS: Partial< + Record +> = { + claude: { home: "claude", driver: "claudeAgent" }, + codex: { home: "codex", driver: "codex" }, + opencode: { home: "opencode", driver: "opencode" }, + grok: { home: "grok", driver: "grok" }, + cursor: { home: "cursor", driver: "cursor" }, + "prime-agent": { home: "prime", driver: "primeAgent" }, + muse: { home: "muse", driver: "muse" }, +}; + +const connectorForHome = (home: string): AuthConnectorKind | undefined => + ( + Object.entries(PROVIDER_CONNECTORS) as ReadonlyArray< + readonly [AuthConnectorKind, { readonly home: ProviderHomeId }] + > + ).find(([, entry]) => entry.home === home)?.[0]; + +const SESSION_TTL_MINUTES = 15; +const TERMINAL_RETENTION = Duration.minutes(5); + +export interface HubProviderSignInOptions { + /** How often the runner's session is read while it is active. */ + readonly pollInterval?: Duration.Input; + /** How often stored sign-ins are listed while waiting for the upload. */ + readonly storePollInterval?: Duration.Input; + /** How long to wait for the stored sign-in after the login succeeded. */ + readonly storeTimeout?: Duration.Input; +} + +interface HubSignInSession { + snapshot: AuthConnectorSession; + runnerSessionId: string | null; + fiber: Fiber.Fiber | null; + terminal: boolean; +} + +const isThreadMachineUnavailable = Schema.is(ThreadMachineUnavailableError); +const isAuthConnectorError = Schema.is(AuthConnectorError); + +const describe = (cause: unknown): string => + cause && typeof cause === "object" && "message" in cause + ? String((cause as { readonly message: unknown }).message) + : String(cause); + +const failure = (operation: string, detail: string) => + new AuthConnectorError({ operation, detail }); + +export const make = (options: HubProviderSignInOptions = {}) => + Effect.gen(function* () { + const pool = yield* RunnerConnectionPool; + const directory = yield* MachineDirectory; + const snapshots = yield* HubProviderSnapshots; + const serverSettings = yield* ServerSettingsService; + const config = yield* ServerConfig; + const crypto = yield* Crypto.Crypto; + const scope = yield* Effect.scope; + const pollInterval = Duration.fromInputUnsafe(options.pollInterval ?? "1 second"); + const storePollInterval = Duration.fromInputUnsafe(options.storePollInterval ?? "2 seconds"); + const storeTimeout = Duration.fromInputUnsafe(options.storeTimeout ?? "2 minutes"); + const browserUrl = ( + config.hub?.threadBrowserUrlTemplate ?? DEFAULT_THREAD_BROWSER_URL_TEMPLATE + ).replaceAll("{threadId}", encodeURIComponent(PROVIDER_SIGN_IN_THREAD_ID)); + + const sessions = new Map(); + + const update = (session: HubSignInSession, patch: Partial) => { + session.snapshot = { ...session.snapshot, ...patch }; + }; + + /** The hub's view of a runner session: the hub's id, and the machine's browser when needed. */ + const mirror = (session: HubSignInSession, remote: AuthConnectorSession) => { + session.snapshot = { + ...remote, + id: session.snapshot.id, + workspaceBrowserUrl: remote.flow === "browser" ? browserUrl : null, + }; + }; + + const fail = (session: HubSignInSession, message: string) => + update(session, { + status: "failed", + stage: "error", + fields: [], + verificationUrl: null, + userCode: null, + message, + }); + + const storedVersion = (home: ProviderHomeId) => + directory.providerHomes.pipe( + Effect.map( + (homes) => homes.providers.find((entry) => entry.provider === home)?.version ?? 0, + ), + ); + + /** A call on the sign-in machine's runner; never wakes it. */ + const onSignInRunner = ( + operation: string, + f: (connection: RunnerConnection) => Effect.Effect, + ) => pool.use(SIGN_IN_THREAD_ID, { wake: false, operation: `auth.${operation}` }, f); + + const activeCount = () => [...sessions.values()].filter((session) => !session.terminal).length; + + /** Ends a session; the machine is reported idle once no sign-in is active. */ + const finish = (id: string, session: HubSignInSession) => + Effect.gen(function* () { + if (session.terminal) return; + session.terminal = true; + yield* pool.setBusy(SIGN_IN_THREAD_ID, `sign-in:${id}`, false); + if (activeCount() === 0) yield* pool.idle(SIGN_IN_THREAD_ID); + yield* Effect.sleep(TERMINAL_RETENTION).pipe( + Effect.andThen(Effect.sync(() => sessions.delete(id))), + Effect.forkIn(scope), + ); + }); + + /** After a successful login: read the provider's status again on the machine. */ + const refreshProviderStatus = (instanceId: ProviderInstanceId, driver: string) => + onSignInRunner("refreshStatus", (connection) => + connection.client["runner.provider.getCapabilities"]({ instanceId, refresh: true }), + ).pipe( + Effect.flatMap((result) => snapshots.put({ ...result.snapshot, instanceId })), + Effect.catchCause(() => + markAuthUnknown(driver, "Signed in; checked when a thread machine starts."), + ), + ); + + const markAuthUnknown = (driver: string, message: string) => + Effect.gen(function* () { + for (const snapshot of yield* snapshots.all) { + if (snapshot.driver !== driver) continue; + yield* snapshots.put({ ...snapshot, auth: { status: "unknown" }, message }); + } + }); + + const run = ( + id: string, + session: HubSignInSession, + input: AuthConnectorStartInput, + provider: { readonly home: ProviderHomeId; readonly driver: string }, + instanceId: ProviderInstanceId, + ) => + Effect.gen(function* () { + const baseline = yield* storedVersion(provider.home); + const started = yield* pool.use( + SIGN_IN_THREAD_ID, + { wake: true, operation: "auth.start", context: SIGN_IN_CONTEXT }, + (connection) => + Effect.gen(function* () { + if (connection.hello.protocolVersion < RUNNER_PROTOCOL_PROVIDER_SETTINGS) { + return yield* failure( + "start", + "The sign-in machine runs a runner too old for provider sign-in.", + ); + } + const settings = yield* effectiveInstanceSettings(serverSettings, instanceId).pipe( + Effect.orElseSucceed(() => undefined), + ); + if (settings !== undefined) { + yield* connection.client["runner.provider.configure"]({ + instances: { [instanceId]: settings }, + }).pipe(Effect.ignore); + } + return yield* connection.client["runner.auth.start"](input); + }), + ); + session.runnerSessionId = started.id; + mirror(session, started); + + let current = started; + while (current.status === "starting" || current.status === "waiting") { + yield* Effect.sleep(pollInterval); + current = yield* onSignInRunner("get", (connection) => + connection.client["runner.auth.get"]({ sessionId: started.id }), + ); + if (current.status !== "succeeded") mirror(session, current); + } + if (current.status !== "succeeded") return; + + update(session, { + status: "starting", + stage: "verifying", + fields: [], + message: "Saving your sign-in…", + }); + const deadline = (yield* Clock.currentTimeMillis) + Duration.toMillis(storeTimeout); + while ((yield* storedVersion(provider.home)) <= baseline) { + if ((yield* Clock.currentTimeMillis) >= deadline) { + return fail( + session, + "The sign-in finished on the machine but was not saved. Try again.", + ); + } + yield* Effect.sleep(storePollInterval); + } + yield* refreshProviderStatus(instanceId, provider.driver); + mirror(session, current); + update(session, { stage: "complete", message: "Signed in." }); + }).pipe( + Effect.catch((error) => + Effect.sync(() => + fail( + session, + isAuthConnectorError(error) + ? error.detail + : isThreadMachineUnavailable(error) + ? `The sign-in machine is unavailable: ${error.detail}` + : `Sign-in failed: ${describe(error)}`, + ), + ), + ), + Effect.ensuring(Effect.suspend(() => finish(id, session))), + ); + + const sessionOf = ( + operation: string, + sessionId: string, + ): Effect.Effect => { + const session = sessions.get(sessionId); + return session + ? Effect.succeed(session) + : Effect.fail(failure(operation, "This connection attempt no longer exists. Start again.")); + }; + + const start: ProviderSignInControlsShape["start"] = (input) => + Effect.gen(function* () { + const provider = PROVIDER_CONNECTORS[input.connector]; + if (provider === undefined) { + return yield* failure( + "start", + "This sign-in is not available here yet; thread machines store provider sign-ins only.", + ); + } + const id = yield* crypto.randomUUIDv4.pipe(Effect.orDie); + const now = yield* DateTime.now; + const session: HubSignInSession = { + snapshot: { + id, + connector: input.connector, + method: input.method, + status: "starting", + flow: authConnectorFlow(input), + stage: "preparing", + message: "Starting a machine for sign-in…", + verificationUrl: null, + userCode: null, + fields: [], + expiresAt: DateTime.formatIso(DateTime.add(now, { minutes: SESSION_TTL_MINUTES })), + workspaceBrowserUrl: null, + }, + runnerSessionId: null, + fiber: null, + terminal: false, + }; + sessions.set(id, session); + yield* pool.setBusy(SIGN_IN_THREAD_ID, `sign-in:${id}`, true); + const instanceId = + input.providerInstanceId ?? + defaultInstanceIdForDriver(ProviderDriverKind.make(provider.driver)); + session.fiber = yield* run(id, session, input, provider, instanceId).pipe( + Effect.forkIn(scope), + ); + return session.snapshot; + }); + + const submit: ProviderSignInControlsShape["submit"] = (input) => + Effect.gen(function* () { + const session = yield* sessionOf("submit", input.sessionId); + if (session.runnerSessionId === null || session.snapshot.status !== "waiting") { + return yield* failure("submit", "This connection attempt is not waiting for input."); + } + const runnerSessionId = session.runnerSessionId; + const next = yield* onSignInRunner("submit", (connection) => + connection.client["runner.auth.submit"]({ ...input, sessionId: runnerSessionId }), + ).pipe( + Effect.mapError((error) => + isAuthConnectorError(error) + ? error + : failure("submit", "The sign-in machine is no longer running. Start again."), + ), + ); + if (next.status !== "succeeded") mirror(session, next); + return session.snapshot; + }); + + const cancel: ProviderSignInControlsShape["cancel"] = (sessionId) => + Effect.gen(function* () { + const session = yield* sessionOf("cancel", sessionId); + if (session.terminal) return session.snapshot; + const runnerSessionId = session.runnerSessionId; + if (runnerSessionId !== null) { + yield* onSignInRunner("cancel", (connection) => + connection.client["runner.auth.cancel"]({ sessionId: runnerSessionId }), + ).pipe(Effect.ignore); + } + if (session.fiber) yield* Fiber.interrupt(session.fiber); + update(session, { + status: "cancelled", + stage: "error", + fields: [], + verificationUrl: null, + userCode: null, + message: "Connection cancelled.", + }); + yield* finish(sessionId, session); + return session.snapshot; + }); + + const directoryFailure = (operation: string) => (error: { readonly message: string }) => + failure(operation, `Stored provider sign-ins are unavailable: ${error.message}`); + + return { + start, + get: (sessionId) => + sessionOf("get", sessionId).pipe(Effect.map((session) => session.snapshot)), + submit, + cancel, + list: directory.providerHomes.pipe( + Effect.map((homes) => ({ + signIns: homes.providers.flatMap((entry) => { + const connector = connectorForHome(entry.provider); + return connector === undefined + ? [] + : [{ connector, version: entry.version, updatedAt: entry.updatedAt ?? null }]; + }), + })), + Effect.mapError(directoryFailure("listProviderSignIns")), + ), + signOut: ({ connector }) => + Effect.gen(function* () { + const provider = PROVIDER_CONNECTORS[connector]; + if (provider === undefined) { + return yield* failure("signOutProvider", "This provider has no stored sign-in."); + } + const result = yield* directory + .deleteProviderHome(provider.home) + .pipe(Effect.mapError(directoryFailure("signOutProvider"))); + yield* markAuthUnknown( + provider.driver, + "Signed out; checked when a thread machine starts.", + ); + return { connector, signedOut: result.deleted }; + }), + } satisfies ProviderSignInControlsShape; + }); + +export const layer = Layer.effect(ProviderSignInControls, make()); diff --git a/apps/server/src/hub/HubProviderSnapshots.ts b/apps/server/src/hub/HubProviderSnapshots.ts index 3895c4012b79..a89e2f0c9b01 100644 --- a/apps/server/src/hub/HubProviderSnapshots.ts +++ b/apps/server/src/hub/HubProviderSnapshots.ts @@ -57,6 +57,8 @@ import { buildUnavailableProviderSnapshot } from "../provider/unavailableProvide export interface HubProviderSnapshotsShape { /** The latest snapshot a runner reported for the instance, if any. */ readonly get: (instanceId: ProviderInstanceId) => Effect.Effect>; + /** Every snapshot a runner reported. */ + readonly all: Effect.Effect>; /** Records (and persists) a runner-reported snapshot and publishes it. */ readonly put: (snapshot: ServerProvider) => Effect.Effect; readonly changes: Stream.Stream; @@ -85,6 +87,7 @@ export const make = Effect.gen(function* () { return HubProviderSnapshots.of({ get: (instanceId) => Effect.sync(() => Option.fromNullishOr(snapshots.get(instanceId))), + all: Effect.sync(() => [...snapshots.values()]), put: (snapshot) => Effect.gen(function* () { snapshots.set(snapshot.instanceId, snapshot); diff --git a/apps/server/src/hub/RemoteProviderDriver.test.ts b/apps/server/src/hub/RemoteProviderDriver.test.ts index 9454505e2edb..1b55cbc3e7ab 100644 --- a/apps/server/src/hub/RemoteProviderDriver.test.ts +++ b/apps/server/src/hub/RemoteProviderDriver.test.ts @@ -199,6 +199,16 @@ describe("remote provider adapter session answers", () => { yield* adapter.stopSession(threadId); expect(yield* adapter.hasSession(threadId)).toBe(false); yield* adapter.stopAll(); + + // The provider sign-in machine never runs agent sessions. + const refused = yield* adapter + .startSession({ + threadId: ThreadId.make("aldo-provider-sign-in"), + provider, + runtimeMode: "full-access", + }) + .pipe(Effect.flip); + expect(refused._tag).toBe("ProviderAdapterValidationError"); expect(yield* fake.calls).toEqual([]); }), ).pipe(Effect.provide(StoresLive)), diff --git a/apps/server/src/hub/RemoteProviderDriver.ts b/apps/server/src/hub/RemoteProviderDriver.ts index 87ba7574dc83..55030db1aa5d 100644 --- a/apps/server/src/hub/RemoteProviderDriver.ts +++ b/apps/server/src/hub/RemoteProviderDriver.ts @@ -37,6 +37,7 @@ import * as NodeFS from "node:fs"; import { type ChatAttachment, + PROVIDER_SIGN_IN_THREAD_ID, type ProviderDriverKind, type ProviderInstanceConfig, type ProviderInstanceId, @@ -72,6 +73,7 @@ import { type ProviderAdapterError, ProviderAdapterRequestError, ProviderAdapterSessionNotFoundError, + ProviderAdapterValidationError, } from "../provider/Errors.ts"; import { defaultProviderContinuationIdentity, @@ -341,6 +343,18 @@ function makeRemoteAdapter( const sessionNotFound = (threadId: ThreadId) => new ProviderAdapterSessionNotFoundError({ provider: driverKind, threadId }); + /** The provider sign-in machine never runs agent sessions. */ + const refuseSignInThread = (threadId: ThreadId, operation: string) => + threadId === PROVIDER_SIGN_IN_THREAD_ID + ? Effect.fail( + new ProviderAdapterValidationError({ + provider: driverKind, + operation, + issue: "The provider sign-in machine does not run agent sessions.", + }), + ) + : Effect.void; + const recordSession = (session: ProviderSession, connection: RunnerConnection) => registry.upsert({ threadId: session.threadId, @@ -376,11 +390,16 @@ function makeRemoteAdapter( return { ...currentCapabilities(), sessionsOutliveServer: true }; }, startSession: (input) => - onRunner(input.threadId, "startSession", true, (connection) => - startOnRunner(connection, input), + refuseSignInThread(input.threadId, "startSession").pipe( + Effect.andThen( + onRunner(input.threadId, "startSession", true, (connection) => + startOnRunner(connection, input), + ), + ), ), sendTurn: (input) => Effect.gen(function* () { + yield* refuseSignInThread(input.threadId, "sendTurn"); // Captured before waking: if the machine was recreated while asleep, // reconciliation settles the old session and this restarts it. const before = yield* hostedSession(input.threadId); diff --git a/apps/server/src/runner/RunnerHandlers.test.ts b/apps/server/src/runner/RunnerHandlers.test.ts index dceb1634c489..0f3268d69554 100644 --- a/apps/server/src/runner/RunnerHandlers.test.ts +++ b/apps/server/src/runner/RunnerHandlers.test.ts @@ -294,4 +294,18 @@ describe("runner handlers", () => { }), ), ); + + it.live("serves provider sign-in only on the sign-in machine", () => + Effect.scoped( + Effect.gen(function* () { + const { pool } = yield* setup; + const refused = yield* pool + .use(threadId, { wake: false, operation: "test" }, (connection) => + connection.client["runner.auth.start"]({ connector: "claude", method: "account" }), + ) + .pipe(Effect.flip); + expect(refused).toMatchObject({ _tag: "AuthConnectorError", operation: "start" }); + }), + ), + ); }); diff --git a/apps/server/src/runner/RunnerHandlers.ts b/apps/server/src/runner/RunnerHandlers.ts index 3e79d81bd01e..84b9704b7f6e 100644 --- a/apps/server/src/runner/RunnerHandlers.ts +++ b/apps/server/src/runner/RunnerHandlers.ts @@ -16,8 +16,10 @@ import * as NodeFS from "node:fs"; import * as NodePath from "node:path"; import { + AuthConnectorError, GitCommandError, GitManagerError, + PROVIDER_SIGN_IN_THREAD_ID, type ChatAttachment, type GitManagerServiceError, type ProviderInstanceId, @@ -53,6 +55,7 @@ import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { resolveAttachmentPath } from "../attachmentStore.ts"; +import * as AuthConnectorManager from "../authConnector/AuthConnectorManager.ts"; import * as CheckpointStore from "../checkpointing/CheckpointStore.ts"; import { ServerConfig } from "../config.ts"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; @@ -300,6 +303,25 @@ export const RunnerRpcHandlersLive = RunnerRpcGroup.toLayer( const decodeSendTurnInput = Schema.decodeUnknownEffect(ProviderSendTurnInput); + /** Provider sign-in runs only on the reserved sign-in machine. */ + const signInMachineOnly = (operation: string) => + binding.threadId === PROVIDER_SIGN_IN_THREAD_ID + ? Effect.void + : Effect.fail( + new AuthConnectorError({ + operation, + detail: "Provider sign-in runs only on the provider sign-in machine.", + }), + ); + /** The login command sees the hub's settings for the instance (environment, paths). */ + const withHubProviderSettings = (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(ServerSettingsService, { + ...serverSettings, + getSettings: providerSettings.effectiveSettings, + }), + ); + return RunnerRpcGroup.of({ "runner.hello": (input) => Effect.gen(function* () { @@ -347,6 +369,27 @@ export const RunnerRpcHandlersLive = RunnerRpcGroup.toLayer( { instances: Object.keys(instances) }, providerSettings.apply(instances).pipe(Effect.map((hosted) => ({ instances: hosted }))), ), + // ── Provider sign-in (credentials in `submit` are never logged) ── + "runner.auth.start": (input) => + signInMachineOnly("start").pipe( + Effect.andThen( + logged( + "auth.start", + { connector: input.connector, method: input.method }, + withHubProviderSettings(AuthConnectorManager.start(input)), + ), + ), + ), + "runner.auth.get": ({ sessionId }) => + signInMachineOnly("get").pipe(Effect.andThen(AuthConnectorManager.get(sessionId))), + "runner.auth.submit": (input) => + signInMachineOnly("submit").pipe( + Effect.andThen(logged("auth.submit", {}, AuthConnectorManager.submit(input))), + ), + "runner.auth.cancel": ({ sessionId }) => + signInMachineOnly("cancel").pipe( + Effect.andThen(logged("auth.cancel", {}, AuthConnectorManager.cancel(sessionId))), + ), "runner.provider.startSession": ({ instanceId, input, mcp }) => guardThread("startSession", input.threadId).pipe( Effect.andThen( diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index b6cdd6c12c75..644b20d15677 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -4762,7 +4762,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); - it.effect("answers thread machine controls as unsupported on a standalone server", () => + it.effect("answers hub-only thread machine and sign-in calls as unsupported standalone", () => Effect.gen(function* () { yield* buildAppUnderTest(); const wsUrl = yield* getWsServerUrl("/ws"); @@ -4779,6 +4779,16 @@ it.layer(NodeServices.layer)("server router seam", (it) => { assert.equal(error._tag, "ThreadMachineControlError"); assert.equal(error._tag === "ThreadMachineControlError" && error.reason, "unsupported"); } + const [list, signOut] = yield* Effect.scoped( + withWsRpcClient(wsUrl, (client) => + Effect.all([ + client[WS_METHODS.serverListProviderSignIns]({}).pipe(Effect.flip), + client[WS_METHODS.serverSignOutProvider]({ connector: "claude" }).pipe(Effect.flip), + ]), + ), + ); + assert.equal(list._tag, "AuthConnectorError"); + assert.equal(signOut._tag, "AuthConnectorError"); }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); diff --git a/apps/server/src/serverModeHooks.ts b/apps/server/src/serverModeHooks.ts index 71b0f369bb47..9847e20bd70b 100644 --- a/apps/server/src/serverModeHooks.ts +++ b/apps/server/src/serverModeHooks.ts @@ -9,9 +9,16 @@ * @module serverModeHooks */ import type { + AuthConnectorError, + AuthConnectorSession, + AuthConnectorStartInput, + AuthConnectorSubmitInput, ClientOrchestrationCommand, OrchestrationDispatchCommandError, ProjectId, + ProviderSignInList, + ProviderSignOutInput, + ProviderSignOutResult, ThreadId, ThreadMachineControlError, ThreadMachineStatus, @@ -112,3 +119,28 @@ export class ThreadMachineControls extends Context.Reference null }, ) {} + +export interface ProviderSignInControlsShape { + readonly start: ( + input: AuthConnectorStartInput, + ) => Effect.Effect; + readonly get: (sessionId: string) => Effect.Effect; + readonly submit: ( + input: AuthConnectorSubmitInput, + ) => Effect.Effect; + readonly cancel: (sessionId: string) => Effect.Effect; + readonly list: Effect.Effect; + readonly signOut: ( + input: ProviderSignOutInput, + ) => Effect.Effect; +} + +/** + * Provider sign-in on a hub: the auth connector runs on the provider sign-in + * machine and sign-ins are stored by the platform. `null` (standalone) keeps + * the local `AuthConnectorManager`. + */ +export class ProviderSignInControls extends Context.Reference( + "t3/serverModeHooks/ProviderSignInControls", + { defaultValue: () => null }, +) {} diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 7e2e7df9f713..160dd3b39d1d 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -18,6 +18,7 @@ import { ClientOs, ClientSurface, ClientWebDeployment, + AuthConnectorError, CommandId, type DiscoveredLocalServerList, EventId, @@ -130,7 +131,7 @@ import * as AzureDevOpsCli from "./sourceControl/AzureDevOpsCli.ts"; import * as BitbucketApi from "./sourceControl/BitbucketApi.ts"; import * as GitHubCli from "./sourceControl/GitHubCli.ts"; import * as GitLabCli from "./sourceControl/GitLabCli.ts"; -import { ThreadMachineControls } from "./serverModeHooks.ts"; +import { ProviderSignInControls, ThreadMachineControls } from "./serverModeHooks.ts"; import * as AuthConnectorManager from "./authConnector/AuthConnectorManager.ts"; import * as SourceControlProviderRegistry from "./sourceControl/SourceControlProviderRegistry.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; @@ -424,6 +425,13 @@ function readClientAnalyticsProps(request: HttpServerRequest.HttpServerRequest) }; } +/** Stored provider sign-ins exist only on a hub (the `threadMachines` capability). */ +const noStoredProviderSignIns = (operation: string) => + new AuthConnectorError({ + operation, + detail: "Stored provider sign-ins exist only on servers that run thread machines.", + }); + /** Thread-machine controls exist only on a hub (the `threadMachines` capability). */ const unsupportedThreadMachineControl = (operation: string) => new ThreadMachineControlError({ @@ -485,6 +493,8 @@ const makeWsRpcLayer = ( const serverSelfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; const config = yield* ServerConfig.ServerConfig; const threadMachineControls = yield* ThreadMachineControls; + // Hub mode runs provider sign-in on the sign-in machine; standalone runs it here. + const providerSignIn = yield* ProviderSignInControls; const lifecycleEvents = yield* ServerLifecycleEvents.ServerLifecycleEvents; const serverSettings = yield* ServerSettings.ServerSettingsService; const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; @@ -1367,17 +1377,31 @@ const makeWsRpcLayer = ( }, ), [WS_METHODS.serverStartAuthConnector]: (input) => - observeRpcEffect(WS_METHODS.serverStartAuthConnector, AuthConnectorManager.start(input), { - "rpc.aggregate": "server", - }), + observeRpcEffect( + WS_METHODS.serverStartAuthConnector, + providerSignIn === null + ? AuthConnectorManager.start(input) + : providerSignIn.start(input), + { + "rpc.aggregate": "server", + }, + ), [WS_METHODS.serverGetAuthConnector]: ({ sessionId }) => - observeRpcEffect(WS_METHODS.serverGetAuthConnector, AuthConnectorManager.get(sessionId), { - "rpc.aggregate": "server", - }), + observeRpcEffect( + WS_METHODS.serverGetAuthConnector, + providerSignIn === null + ? AuthConnectorManager.get(sessionId) + : providerSignIn.get(sessionId), + { + "rpc.aggregate": "server", + }, + ), [WS_METHODS.serverSubmitAuthConnector]: (input) => observeRpcEffect( WS_METHODS.serverSubmitAuthConnector, - AuthConnectorManager.submit(input), + providerSignIn === null + ? AuthConnectorManager.submit(input) + : providerSignIn.submit(input), { "rpc.aggregate": "server", }, @@ -1385,11 +1409,29 @@ const makeWsRpcLayer = ( [WS_METHODS.serverCancelAuthConnector]: ({ sessionId }) => observeRpcEffect( WS_METHODS.serverCancelAuthConnector, - AuthConnectorManager.cancel(sessionId), + providerSignIn === null + ? AuthConnectorManager.cancel(sessionId) + : providerSignIn.cancel(sessionId), { "rpc.aggregate": "server", }, ), + [WS_METHODS.serverListProviderSignIns]: (_input) => + observeRpcEffect( + WS_METHODS.serverListProviderSignIns, + providerSignIn === null + ? Effect.fail(noStoredProviderSignIns("listProviderSignIns")) + : providerSignIn.list, + { "rpc.aggregate": "server" }, + ), + [WS_METHODS.serverSignOutProvider]: (input) => + observeRpcEffect( + WS_METHODS.serverSignOutProvider, + providerSignIn === null + ? Effect.fail(noStoredProviderSignIns("signOutProvider")) + : providerSignIn.signOut(input), + { "rpc.aggregate": "server" }, + ), [WS_METHODS.serverGetTraceDiagnostics]: (_input) => observeRpcEffect( WS_METHODS.serverGetTraceDiagnostics, diff --git a/packages/contracts/src/authConnector.ts b/packages/contracts/src/authConnector.ts index 08750859918c..c024dcf78af8 100644 --- a/packages/contracts/src/authConnector.ts +++ b/packages/contracts/src/authConnector.ts @@ -83,6 +83,12 @@ export const AuthConnectorSession = Schema.Struct({ userCode: Schema.NullOr(Schema.String), fields: Schema.Array(AuthConnectorField), expiresAt: Schema.NullOr(Schema.String), + /** + * Hub mode: sign-in runs on a thread machine. For flows that finish in a + * browser on that machine (`flow: "browser"`), the same-origin page showing + * the machine's browser. Absent on standalone servers. + */ + workspaceBrowserUrl: Schema.optional(Schema.NullOr(Schema.String)), }); export type AuthConnectorSession = typeof AuthConnectorSession.Type; @@ -116,3 +122,30 @@ export class AuthConnectorError extends Schema.TaggedErrorClass Date: Sat, 26 Sep 2026 01:30:20 +0000 Subject: [PATCH 47/58] feat(server): keep a hub's MCP credentials across hub restarts Provider sessions on thread machines outlive the hub process, so the hub now persists each MCP credential it mints (the token's hash and scope, never the token; hub migration 051, SQLite in development) through the McpCredentialPersistence hook. A restarted hub loads unexpired credentials and keeps answering the sessions' MCP calls; revocation deletes them, liveness is written through at most every ten minutes, and revokeAll on shutdown clears only the process's memory. Standalone servers keep credentials in memory only, as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/hub/HubLayers.ts | 11 +- apps/server/src/hub/HubMcpCredentials.ts | 47 ++++++++ .../server/src/mcp/McpSessionRegistry.test.ts | 57 ++++++++++ apps/server/src/mcp/McpSessionRegistry.ts | 105 ++++++++++++++++-- .../Layers/HubThreadMachineState.test.ts | 12 ++ .../Layers/HubThreadMachineState.ts | 68 ++++++++++++ .../Postgres/HubThreadMachineState.test.ts | 41 ++++++- .../Postgres/HubThreadMachineState.ts | 64 +++++++++++ .../051_HubThreadMachineServices.ts | 15 +++ .../Services/HubThreadMachineState.ts | 31 ++++++ .../hubThreadMachineServicesScenario.ts | 99 +++++++++++++++++ apps/server/src/serverModeHooks.ts | 30 +++++ 12 files changed, 567 insertions(+), 13 deletions(-) create mode 100644 apps/server/src/hub/HubMcpCredentials.ts create mode 100644 apps/server/src/persistence/hubThreadMachineServicesScenario.ts diff --git a/apps/server/src/hub/HubLayers.ts b/apps/server/src/hub/HubLayers.ts index 36c2bad96a37..646f5ddd726f 100644 --- a/apps/server/src/hub/HubLayers.ts +++ b/apps/server/src/hub/HubLayers.ts @@ -44,6 +44,7 @@ import { hubWorkspaceFileSystemLayer, hubWorkspacePathsLayer, } from "./HubWorkspace.ts"; +import * as HubMcpCredentials from "./HubMcpCredentials.ts"; import * as HubProviderSignIn from "./HubProviderSignIn.ts"; import * as HubProviderSnapshots from "./HubProviderSnapshots.ts"; import * as MachineDirectory from "./MachineDirectory.ts"; @@ -70,7 +71,7 @@ export const makeHubStateStoresLayer = ( /** * Hub-only services underneath the whole runtime: stores, machine states, - * provider snapshots, machine directory (observed by the machine states), connection pool, + * provider snapshots, persisted MCP credentials, machine directory (observed by the machine states), connection pool, * session registry, event delivery and git status cache. `persistence` is * the server's persistence layer (SQLite, or the hub's Postgres client in hub * mode with a database). @@ -89,7 +90,13 @@ export const makeHubInfrastructureLayer = (parts: { ThreadMachineStates.observedMachineDirectoryLayer.pipe(Layer.provide(MachineDirectory.layer)), ), Layer.provideMerge(ThreadMachineStates.readerLayer), - Layer.provideMerge(Layer.mergeAll(ThreadMachineStates.layer, HubProviderSnapshots.layer)), + Layer.provideMerge( + Layer.mergeAll( + ThreadMachineStates.layer, + HubProviderSnapshots.layer, + HubMcpCredentials.layer, + ), + ), Layer.provideMerge(makeHubStateStoresLayer(parts.persistence)), ); diff --git a/apps/server/src/hub/HubMcpCredentials.ts b/apps/server/src/hub/HubMcpCredentials.ts new file mode 100644 index 000000000000..ecde92e2caac --- /dev/null +++ b/apps/server/src/hub/HubMcpCredentials.ts @@ -0,0 +1,47 @@ +/** + * MCP credentials that survive hub restarts. + * + * The hub mints an MCP credential for every provider session it starts and + * the runner hands it to the provider CLI. Those sessions run on thread + * machines and outlive the hub process, so the hub persists each credential's + * hash and scope (never the token) in `McpCredentialStore`; a restarted hub + * loads them and keeps answering the sessions' MCP calls. Persistence failures + * are logged: the credential still works for this process. + * + * @module hub/HubMcpCredentials + */ +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import { + McpCredentialStore, + type McpCredentialRow, +} from "../persistence/Services/HubThreadMachineState.ts"; +import { McpCredentialPersistence } from "../serverModeHooks.ts"; + +export const layer = Layer.effect( + McpCredentialPersistence, + Effect.gen(function* () { + const store = yield* McpCredentialStore; + const logFailure = + (operation: string) => + (error: { readonly message: string }): Effect.Effect => + Effect.logWarning("MCP credential persistence failed", { + operation, + detail: error.message, + }); + return { + load: store + .list() + .pipe( + Effect.catch((error) => + logFailure("load")(error).pipe(Effect.as([] as ReadonlyArray)), + ), + ), + save: (record) => store.put(record).pipe(Effect.catch(logFailure("save"))), + touch: (tokenHashes, lastAliveAt) => + store.touch(tokenHashes, lastAliveAt).pipe(Effect.catch(logFailure("touch"))), + remove: (tokenHashes) => store.remove(tokenHashes).pipe(Effect.catch(logFailure("remove"))), + }; + }), +); diff --git a/apps/server/src/mcp/McpSessionRegistry.test.ts b/apps/server/src/mcp/McpSessionRegistry.test.ts index dc038987b17b..19c88467fb89 100644 --- a/apps/server/src/mcp/McpSessionRegistry.test.ts +++ b/apps/server/src/mcp/McpSessionRegistry.test.ts @@ -5,6 +5,11 @@ import * as Effect from "effect/Effect"; import { HttpServer } from "effect/unstable/http"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import { + McpCredentialPersistence, + type McpCredentialPersistenceShape, + type McpCredentialRecord, +} from "../serverModeHooks.ts"; import * as McpSessionRegistry from "./McpSessionRegistry.ts"; const environmentId = EnvironmentId.make("environment-1"); @@ -142,3 +147,55 @@ it.effect("does not keep credentials of other threads alive", () => expect(yield* registry.resolve(token)).toBeUndefined(); }), ); + +it.effect("keeps persisted credentials across restarts until they are revoked or expire", () => + Effect.gen(function* () { + const rows = new Map(); + const persistence: McpCredentialPersistenceShape = { + load: Effect.sync(() => [...rows.values()]), + save: (record) => Effect.sync(() => void rows.set(record.tokenHash, record)), + touch: (hashes, lastAliveAt) => + Effect.sync(() => { + for (const hash of hashes) { + const row = rows.get(hash); + if (row) rows.set(hash, { ...row, lastAliveAt }); + } + }), + remove: (hashes) => Effect.sync(() => hashes.forEach((hash) => rows.delete(hash))), + }; + let timestamp = 1_000; + const makePersistent = () => + makeRegistry(() => timestamp).pipe( + Effect.provideService(McpCredentialPersistence, persistence), + ); + + const first = yield* makePersistent(); + const threadId = ThreadId.make("thread-persisted"); + const issued = yield* first.issue({ + threadId, + providerInstanceId: ProviderInstanceId.make("claudeAgent"), + grantReview: true, + }); + const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); + // Only the hash is persisted. + expect([...rows.values()].some((row) => JSON.stringify(row).includes(token))).toBe(false); + yield* first.revokeAll; + + // A new process (a restarted hub) still resolves the running session's token. + const second = yield* makePersistent(); + const resolved = yield* second.resolve(token); + expect(resolved?.threadId).toBe(threadId); + expect(resolved?.capabilities).toEqual(new Set(["preview", "review"])); + + yield* second.revokeThread(threadId); + expect(rows.size).toBe(0); + + // Credentials past the liveness window are dropped when loaded. + yield* second.issue({ threadId, providerInstanceId: ProviderInstanceId.make("codex") }); + expect(rows.size).toBe(1); + timestamp += 1_000; + const third = yield* makePersistent(); + expect(rows.size).toBe(0); + expect(yield* third.resolve(token)).toBeUndefined(); + }), +); diff --git a/apps/server/src/mcp/McpSessionRegistry.ts b/apps/server/src/mcp/McpSessionRegistry.ts index b6c0176b7264..604e3376417b 100644 --- a/apps/server/src/mcp/McpSessionRegistry.ts +++ b/apps/server/src/mcp/McpSessionRegistry.ts @@ -8,6 +8,7 @@ import * as SynchronizedRef from "effect/SynchronizedRef"; import { HttpServer } from "effect/unstable/http"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import { McpCredentialPersistence, type McpCredentialRecord } from "../serverModeHooks.ts"; import * as McpInvocationContext from "./McpInvocationContext.ts"; import * as McpProviderSession from "./McpProviderSession.ts"; @@ -73,6 +74,14 @@ export interface McpSessionRegistryOptions { */ const DEFAULT_LIVENESS_WINDOW_MS = 24 * 60 * 60 * 1_000; +/** + * With persistence (a hub), how stale a persisted liveness may get before a + * sign of life is written through. Well inside the liveness window. + */ +const PERSISTED_LIVENESS_REFRESH_MS = 10 * 60 * 1_000; + +const MCP_CAPABILITIES: ReadonlySet = new Set(["preview", "review"]); + const bytesToHex = (bytes: Uint8Array): string => Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); @@ -96,9 +105,64 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( const environment = yield* ServerEnvironment.ServerEnvironment; const environmentId = yield* environment.getEnvironmentId; const httpServer = yield* HttpServer.HttpServer; - const state = yield* SynchronizedRef.make({ records: new Map() }); const currentTimeMillis = options.now ? Effect.sync(options.now) : Clock.currentTimeMillis; const livenessWindowMs = options.livenessWindowMs ?? DEFAULT_LIVENESS_WINDOW_MS; + // A hub persists credentials: its provider sessions outlive the process. + const persistence = yield* McpCredentialPersistence; + const persistedAliveAt = new Map(); + const restored = new Map(); + if (persistence !== null) { + const loadedAt = yield* currentTimeMillis; + const expired: Array = []; + for (const row of yield* persistence.load) { + if (row.environmentId !== environmentId || loadedAt - row.lastAliveAt > livenessWindowMs) { + expired.push(row.tokenHash); + continue; + } + restored.set(row.tokenHash, { + tokenHash: row.tokenHash, + lastAliveAt: row.lastAliveAt, + scope: { + environmentId, + threadId: ThreadId.make(row.threadId), + providerSessionId: row.providerSessionId, + providerInstanceId: ProviderInstanceId.make(row.providerInstanceId), + capabilities: new Set( + row.capabilities.filter( + (capability): capability is McpInvocationContext.McpCapability => + MCP_CAPABILITIES.has(capability), + ), + ), + issuedAt: row.issuedAt, + }, + }); + persistedAliveAt.set(row.tokenHash, row.lastAliveAt); + } + if (expired.length > 0) yield* persistence.remove(expired); + } + const state = yield* SynchronizedRef.make({ records: restored }); + + /** Writes liveness through for credentials whose persisted value is stale. */ + const persistLiveness = (tokenHashes: ReadonlyArray, timestamp: number) => { + if (persistence === null) return Effect.void; + const stale = tokenHashes.filter( + (tokenHash) => + timestamp - (persistedAliveAt.get(tokenHash) ?? 0) >= PERSISTED_LIVENESS_REFRESH_MS, + ); + if (stale.length === 0) return Effect.void; + for (const tokenHash of stale) persistedAliveAt.set(tokenHash, timestamp); + return persistence.touch(stale, timestamp); + }; + const toPersisted = (record: CredentialRecord): McpCredentialRecord => ({ + tokenHash: record.tokenHash, + environmentId: record.scope.environmentId, + threadId: record.scope.threadId, + providerSessionId: record.scope.providerSessionId, + providerInstanceId: record.scope.providerInstanceId, + capabilities: [...record.scope.capabilities], + issuedAt: record.scope.issuedAt, + lastAliveAt: record.lastAliveAt, + }); const endpoint = httpServer.address._tag === "TcpAddress" ? `http://${getHttpMcpEndpointHost(httpServer.address.hostname)}:${httpServer.address.port}/mcp` @@ -135,11 +199,16 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( ]), issuedAt, }; + const record: CredentialRecord = { tokenHash, scope, lastAliveAt: issuedAt }; yield* SynchronizedRef.update(state, ({ records }) => { const next = new Map(pruneDead(records, issuedAt)); - next.set(tokenHash, { tokenHash, scope, lastAliveAt: issuedAt }); + next.set(tokenHash, record); return { records: next }; }); + if (persistence !== null) { + persistedAliveAt.set(tokenHash, issuedAt); + yield* persistence.save(toPersisted(record)); + } return { config: { environmentId, @@ -158,7 +227,7 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( if (rawToken.length === 0) return undefined; const tokenHash = yield* hashToken(rawToken); const timestamp = yield* currentTimeMillis; - return yield* SynchronizedRef.modify(state, ({ records }) => { + const scope = yield* SynchronizedRef.modify(state, ({ records }) => { const current = pruneDead(records, timestamp); const record = current.get(tokenHash); if (!record) return [undefined, { records: current }] as const; @@ -166,29 +235,46 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( next.set(tokenHash, { ...record, lastAliveAt: timestamp }); return [record.scope, { records: next }] as const; }); + if (scope !== undefined) yield* persistLiveness([tokenHash], timestamp); + return scope; }, ); const touch: McpSessionRegistryShape["touch"] = Effect.fn("McpSessionRegistry.touch")( function* (threadId) { const timestamp = yield* currentTimeMillis; - yield* SynchronizedRef.update(state, ({ records }) => { + const touched = yield* SynchronizedRef.modify(state, ({ records }) => { const current = pruneDead(records, timestamp); const next = new Map(current); + const hashes: Array = []; for (const [tokenHash, record] of current) { if (record.scope.threadId === threadId) { next.set(tokenHash, { ...record, lastAliveAt: timestamp }); + hashes.push(tokenHash); } } - return { records: next }; + return [hashes, { records: next }] as const; }); + yield* persistLiveness(touched, timestamp); }, ); const revokeWhere = (predicate: (record: CredentialRecord) => boolean) => - SynchronizedRef.update(state, ({ records }) => ({ - records: new Map(Array.from(records).filter(([, record]) => !predicate(record))), - })); + SynchronizedRef.modify(state, ({ records }) => { + const revoked = Array.from(records.values()).filter(predicate); + return [ + revoked.map((record) => record.tokenHash), + { + records: new Map(Array.from(records).filter(([, record]) => !predicate(record))), + }, + ] as const; + }).pipe( + Effect.flatMap((revoked) => { + if (persistence === null || revoked.length === 0) return Effect.void; + for (const tokenHash of revoked) persistedAliveAt.delete(tokenHash); + return persistence.remove(revoked); + }), + ); return McpSessionRegistry.of({ issue, @@ -202,6 +288,9 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( revokeThread: Effect.fn("McpSessionRegistry.revokeThread")(function* (threadId) { yield* revokeWhere((record) => record.scope.threadId === threadId); }), + // Clears this process's credentials only (the shutdown path). Persisted + // credentials belong to sessions that outlive the server (a hub's thread + // machines) and are loaded again by the next process. revokeAll: SynchronizedRef.set(state, { records: new Map() }), }); }); diff --git a/apps/server/src/persistence/Layers/HubThreadMachineState.test.ts b/apps/server/src/persistence/Layers/HubThreadMachineState.test.ts index a92aebd61d9d..2285a805fc44 100644 --- a/apps/server/src/persistence/Layers/HubThreadMachineState.test.ts +++ b/apps/server/src/persistence/Layers/HubThreadMachineState.test.ts @@ -10,6 +10,11 @@ import { RunnerCursorStore, ThreadVcsStatusStore, } from "../Services/HubThreadMachineState.ts"; +import { + expectedThreadMachineServices, + readThreadMachineServices, + writeThreadMachineServices, +} from "../hubThreadMachineServicesScenario.ts"; import { HubThreadMachineStateSqliteLive } from "./HubThreadMachineState.ts"; import { SqlitePersistenceMemory } from "./Sqlite.ts"; @@ -114,4 +119,11 @@ describe("hub thread-machine state on SQLite", () => { expect(yield* statuses.list()).toEqual([]); }).pipe(Effect.provide(TestLayer)), ); + + it.effect("keeps machine states, provider snapshots and MCP credentials", () => + Effect.gen(function* () { + yield* writeThreadMachineServices("sqlite"); + expect(yield* readThreadMachineServices).toEqual(expectedThreadMachineServices("sqlite")); + }).pipe(Effect.provide(TestLayer)), + ); }); diff --git a/apps/server/src/persistence/Layers/HubThreadMachineState.ts b/apps/server/src/persistence/Layers/HubThreadMachineState.ts index 4540bace6be4..a89588907a44 100644 --- a/apps/server/src/persistence/Layers/HubThreadMachineState.ts +++ b/apps/server/src/persistence/Layers/HubThreadMachineState.ts @@ -26,6 +26,8 @@ import { CheckpointTurnDiff, CheckpointTurnDiffKey, CheckpointTurnDiffStore, + McpCredentialRow, + McpCredentialStore, ProviderSnapshotRow, ProviderSnapshotStore, RunnerCursor, @@ -71,6 +73,16 @@ const HUB_SQLITE_TABLES = [ snapshot_json TEXT NOT NULL, updated_at TEXT NOT NULL )`, + `CREATE TABLE IF NOT EXISTS hub_mcp_credentials ( + token_hash TEXT PRIMARY KEY, + environment_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + provider_session_id TEXT NOT NULL, + provider_instance_id TEXT NOT NULL, + capabilities_json TEXT NOT NULL, + issued_at INTEGER NOT NULL, + last_alive_at INTEGER NOT NULL + )`, ] as const; const ensureTables = Effect.gen(function* () { @@ -308,6 +320,61 @@ export const makeProviderSnapshotStore = Effect.gen(function* () { }); }); +const McpCredentialDbRow = McpCredentialRow.mapFields((fields) => ({ + ...fields, + capabilities: Schema.fromJsonString(fields.capabilities), +})); + +export const makeMcpCredentialStore = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* ensureTables; + + const selectAll = SqlSchema.findAll({ + Request: Schema.Void, + Result: McpCredentialDbRow, + execute: () => sql` + SELECT token_hash AS "tokenHash", environment_id AS "environmentId", + thread_id AS "threadId", provider_session_id AS "providerSessionId", + provider_instance_id AS "providerInstanceId", capabilities_json AS capabilities, + issued_at AS "issuedAt", last_alive_at AS "lastAliveAt" + FROM hub_mcp_credentials + `, + }); + const insert = SqlSchema.void({ + Request: McpCredentialDbRow, + execute: (row) => sql` + INSERT INTO hub_mcp_credentials ( + token_hash, environment_id, thread_id, provider_session_id, provider_instance_id, + capabilities_json, issued_at, last_alive_at + ) + VALUES ( + ${row.tokenHash}, ${row.environmentId}, ${row.threadId}, ${row.providerSessionId}, + ${row.providerInstanceId}, ${row.capabilities}, ${row.issuedAt}, ${row.lastAliveAt} + ) + ON CONFLICT (token_hash) DO UPDATE SET last_alive_at = excluded.last_alive_at + `, + }); + + return McpCredentialStore.of({ + list: () => selectAll(undefined).pipe(Effect.mapError(sqlOrDecode("McpCredentialStore.list"))), + put: (row) => insert(row).pipe(Effect.mapError(sqlOrDecode("McpCredentialStore.put"))), + touch: (tokenHashes, lastAliveAt) => + tokenHashes.length === 0 + ? Effect.void + : sql` + UPDATE hub_mcp_credentials SET last_alive_at = ${lastAliveAt} + WHERE token_hash IN ${sql.in(tokenHashes)} + `.pipe(Effect.asVoid, Effect.mapError(toPersistenceSqlError("McpCredentialStore.touch"))), + remove: (tokenHashes) => + tokenHashes.length === 0 + ? Effect.void + : sql`DELETE FROM hub_mcp_credentials WHERE token_hash IN ${sql.in(tokenHashes)}`.pipe( + Effect.asVoid, + Effect.mapError(toPersistenceSqlError("McpCredentialStore.remove")), + ), + }); +}); + /** Every repository over the configured SQLite client. */ export const HubThreadMachineStateSqliteLive = Layer.mergeAll( Layer.effect(RunnerCursorStore, makeRunnerCursorStore), @@ -315,4 +382,5 @@ export const HubThreadMachineStateSqliteLive = Layer.mergeAll( Layer.effect(ThreadVcsStatusStore, makeThreadVcsStatusStore), Layer.effect(ThreadMachineStatusStore, makeThreadMachineStatusStore), Layer.effect(ProviderSnapshotStore, makeProviderSnapshotStore), + Layer.effect(McpCredentialStore, makeMcpCredentialStore), ); diff --git a/apps/server/src/persistence/Postgres/HubThreadMachineState.test.ts b/apps/server/src/persistence/Postgres/HubThreadMachineState.test.ts index 9075cb49fcd1..2c818a266d35 100644 --- a/apps/server/src/persistence/Postgres/HubThreadMachineState.test.ts +++ b/apps/server/src/persistence/Postgres/HubThreadMachineState.test.ts @@ -21,7 +21,18 @@ import { import { HubDatabase } from "./HubDatabase.ts"; import { HubThreadMachineStatePostgresLive } from "./HubThreadMachineState.ts"; import { hubTenantLayer } from "./HubTenant.ts"; -import { HUB_THREAD_MACHINE_STATE_TABLES } from "./migrations/050_HubThreadMachineState.ts"; +import { + expectedThreadMachineServices, + readThreadMachineServices, + writeThreadMachineServices, +} from "../hubThreadMachineServicesScenario.ts"; +import { HUB_THREAD_MACHINE_STATE_TABLES as MIGRATION_050_TABLES } from "./migrations/050_HubThreadMachineState.ts"; +import { HUB_THREAD_MACHINE_SERVICE_TABLES } from "./migrations/051_HubThreadMachineServices.ts"; + +const HUB_THREAD_MACHINE_STATE_TABLES = [ + ...MIGRATION_050_TABLES, + ...HUB_THREAD_MACHINE_SERVICE_TABLES, +] as const; import { hubTestDatabaseLayer, hubTestDatabaseUrl, @@ -140,7 +151,28 @@ describe.skipIf(hubTestDatabaseUrl === undefined)("hub thread-machine state on P ), ); - it.effect("applies migration 050 with the forced tenant policy on every table", () => + it.effect("keeps machine states, provider snapshots and MCP credentials per tenant", () => + withSchema((schema) => + Effect.gen(function* () { + yield* writeThreadMachineServices("user-a").pipe( + Effect.provide(storesFor(schema, "user-a")), + ); + yield* writeThreadMachineServices("user-b").pipe( + Effect.provide(storesFor(schema, "user-b")), + ); + assert.deepStrictEqual( + yield* readThreadMachineServices.pipe(Effect.provide(storesFor(schema, "user-a"))), + expectedThreadMachineServices("user-a"), + ); + assert.deepStrictEqual( + yield* readThreadMachineServices.pipe(Effect.provide(storesFor(schema, "user-b"))), + expectedThreadMachineServices("user-b"), + ); + }), + ), + ); + + it.effect("applies migrations 050 and 051 with the forced tenant policy on every table", () => withSchema((schema) => Effect.gen(function* () { yield* writeAs(1, "user-a").pipe(Effect.provide(storesFor(schema, "user-a"))); @@ -152,7 +184,10 @@ describe.skipIf(hubTestDatabaseUrl === undefined)("hub thread-machine state on P `; assert.deepStrictEqual( migrations.map((row) => [row.id, row.name]), - [[50, "HubThreadMachineState"]], + [ + [50, "HubThreadMachineState"], + [51, "HubThreadMachineServices"], + ], ); const tables = yield* sql<{ readonly table: string; diff --git a/apps/server/src/persistence/Postgres/HubThreadMachineState.ts b/apps/server/src/persistence/Postgres/HubThreadMachineState.ts index 9d0ecf66ddd2..3d56f98a494a 100644 --- a/apps/server/src/persistence/Postgres/HubThreadMachineState.ts +++ b/apps/server/src/persistence/Postgres/HubThreadMachineState.ts @@ -23,6 +23,8 @@ import { CheckpointTurnDiff, CheckpointTurnDiffKey, CheckpointTurnDiffStore, + McpCredentialRow, + McpCredentialStore, ProviderSnapshotRow, ProviderSnapshotStore, RunnerCursor, @@ -274,6 +276,67 @@ export const makeProviderSnapshotStore = Effect.gen(function* () { }); }); +const McpCredentialDbRow = McpCredentialRow.mapFields((fields) => ({ + ...fields, + capabilities: Schema.fromJsonString(fields.capabilities), + issuedAt: Schema.NumberFromString, + lastAliveAt: Schema.NumberFromString, +})); + +export const makeMcpCredentialStore = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const { userId } = yield* HubTenant; + + const selectAll = SqlSchema.findAll({ + Request: Schema.Void, + Result: McpCredentialDbRow, + execute: () => sql` + SELECT token_hash AS "tokenHash", environment_id AS "environmentId", + thread_id AS "threadId", provider_session_id AS "providerSessionId", + provider_instance_id AS "providerInstanceId", capabilities_json AS capabilities, + issued_at::text AS "issuedAt", last_alive_at::text AS "lastAliveAt" + FROM hub_mcp_credentials WHERE user_id = ${userId} + `, + }); + const insert = SqlSchema.void({ + Request: McpCredentialDbRow, + execute: (row) => sql` + INSERT INTO hub_mcp_credentials ( + user_id, token_hash, environment_id, thread_id, provider_session_id, + provider_instance_id, capabilities_json, issued_at, last_alive_at + ) + VALUES ( + ${userId}, ${row.tokenHash}, ${row.environmentId}, ${row.threadId}, + ${row.providerSessionId}, ${row.providerInstanceId}, ${row.capabilities}, + ${row.issuedAt}::bigint, ${row.lastAliveAt}::bigint + ) + ON CONFLICT (user_id, token_hash) DO UPDATE SET last_alive_at = excluded.last_alive_at + `, + }); + + return McpCredentialStore.of({ + list: () => selectAll(undefined).pipe(Effect.mapError(sqlOrDecode("McpCredentialStore.list"))), + put: (row) => insert(row).pipe(Effect.mapError(sqlOrDecode("McpCredentialStore.put"))), + touch: (tokenHashes, lastAliveAt) => + tokenHashes.length === 0 + ? Effect.void + : sql` + UPDATE hub_mcp_credentials SET last_alive_at = ${String(lastAliveAt)}::bigint + WHERE user_id = ${userId} AND token_hash IN ${sql.in(tokenHashes)} + `.pipe(Effect.asVoid, Effect.mapError(toPersistenceSqlError("McpCredentialStore.touch"))), + remove: (tokenHashes) => + tokenHashes.length === 0 + ? Effect.void + : sql` + DELETE FROM hub_mcp_credentials + WHERE user_id = ${userId} AND token_hash IN ${sql.in(tokenHashes)} + `.pipe( + Effect.asVoid, + Effect.mapError(toPersistenceSqlError("McpCredentialStore.remove")), + ), + }); +}); + /** Every repository; requires the hub Postgres `SqlClient` and `HubTenant`. */ export const HubThreadMachineStatePostgresLive = Layer.mergeAll( Layer.effect(RunnerCursorStore, makeRunnerCursorStore), @@ -281,4 +344,5 @@ export const HubThreadMachineStatePostgresLive = Layer.mergeAll( Layer.effect(ThreadVcsStatusStore, makeThreadVcsStatusStore), Layer.effect(ThreadMachineStatusStore, makeThreadMachineStatusStore), Layer.effect(ProviderSnapshotStore, makeProviderSnapshotStore), + Layer.effect(McpCredentialStore, makeMcpCredentialStore), ); diff --git a/apps/server/src/persistence/Postgres/migrations/051_HubThreadMachineServices.ts b/apps/server/src/persistence/Postgres/migrations/051_HubThreadMachineServices.ts index 18f15f8a6968..4381bf73dfee 100644 --- a/apps/server/src/persistence/Postgres/migrations/051_HubThreadMachineServices.ts +++ b/apps/server/src/persistence/Postgres/migrations/051_HubThreadMachineServices.ts @@ -5,6 +5,8 @@ * per thread, so thread shells show it without asking the directory again. * - `hub_provider_snapshots`: the last provider snapshot (status, auth, models) * a runner reported per provider instance. + * - `hub_mcp_credentials`: hashes and scopes of the MCP credentials minted for + * provider sessions, which run on thread machines and outlive the hub. * * Every table leads its key with `user_id` and carries the forced tenant policy * from the baseline (`hubTenantPolicyStatements`). The statements are @@ -22,6 +24,7 @@ const c = `COLLATE "C"`; export const HUB_THREAD_MACHINE_SERVICE_TABLES = [ "hub_thread_machine_status", "hub_provider_snapshots", + "hub_mcp_credentials", ] as const; const statements: ReadonlyArray = [ @@ -41,6 +44,18 @@ const statements: ReadonlyArray = [ updated_at text ${c} NOT NULL, PRIMARY KEY (user_id, instance_id) )`, + `CREATE TABLE IF NOT EXISTS hub_mcp_credentials ( + user_id text ${c} NOT NULL, + token_hash text ${c} NOT NULL, + environment_id text ${c} NOT NULL, + thread_id text ${c} NOT NULL, + provider_session_id text ${c} NOT NULL, + provider_instance_id text ${c} NOT NULL, + capabilities_json text NOT NULL, + issued_at bigint NOT NULL, + last_alive_at bigint NOT NULL, + PRIMARY KEY (user_id, token_hash) + )`, ...HUB_THREAD_MACHINE_SERVICE_TABLES.flatMap(hubTenantPolicyStatements), ]; diff --git a/apps/server/src/persistence/Services/HubThreadMachineState.ts b/apps/server/src/persistence/Services/HubThreadMachineState.ts index 0afac55e3d9f..ef2ff6c119c5 100644 --- a/apps/server/src/persistence/Services/HubThreadMachineState.ts +++ b/apps/server/src/persistence/Services/HubThreadMachineState.ts @@ -14,6 +14,8 @@ * per thread, shown on thread shells without asking the directory again. * - `ProviderSnapshotStore`: the last provider snapshot a runner reported per * provider instance, so a hub shows providers before any machine runs. + * - `McpCredentialStore`: the hashes and scopes of the MCP credentials the hub + * minted for provider sessions on thread machines, which outlive the hub. * * Postgres implementations live in `persistence/Postgres/HubThreadMachineState.ts` * (hub migrations 050 and 051); the SQLite implementations back hub mode without a @@ -161,3 +163,32 @@ export class ProviderSnapshotStore extends Context.Service< ProviderSnapshotStore, ProviderSnapshotStoreShape >()("t3/persistence/Services/HubThreadMachineState/ProviderSnapshotStore") {} + +export const McpCredentialRow = Schema.Struct({ + tokenHash: TrimmedNonEmptyString, + environmentId: Schema.String, + threadId: Schema.String, + providerSessionId: Schema.String, + providerInstanceId: Schema.String, + capabilities: Schema.Array(Schema.String), + issuedAt: Schema.Number, + lastAliveAt: Schema.Number, +}); +export type McpCredentialRow = typeof McpCredentialRow.Type; + +export interface McpCredentialStoreShape { + readonly list: () => Effect.Effect, ProjectionRepositoryError>; + readonly put: (row: McpCredentialRow) => Effect.Effect; + readonly touch: ( + tokenHashes: ReadonlyArray, + lastAliveAt: number, + ) => Effect.Effect; + readonly remove: ( + tokenHashes: ReadonlyArray, + ) => Effect.Effect; +} + +export class McpCredentialStore extends Context.Service< + McpCredentialStore, + McpCredentialStoreShape +>()("t3/persistence/Services/HubThreadMachineState/McpCredentialStore") {} diff --git a/apps/server/src/persistence/hubThreadMachineServicesScenario.ts b/apps/server/src/persistence/hubThreadMachineServicesScenario.ts new file mode 100644 index 000000000000..081148ac8b1f --- /dev/null +++ b/apps/server/src/persistence/hubThreadMachineServicesScenario.ts @@ -0,0 +1,99 @@ +/** + * One scenario over the stores behind the hub's thread-machine services + * (migration 051 on Postgres, first-use tables on SQLite): machine states, + * provider snapshots, and MCP credentials. Shared by the SQLite and Postgres + * store tests so both implementations answer identically. + * + * @module hubThreadMachineServicesScenario + */ +import { + ProviderDriverKind, + ProviderInstanceId, + type ServerProvider, + ThreadId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; + +import { + McpCredentialStore, + ProviderSnapshotStore, + ThreadMachineStatusStore, +} from "./Services/HubThreadMachineState.ts"; + +const snapshot = (label: string): ServerProvider => ({ + instanceId: ProviderInstanceId.make("claudeAgent"), + driver: ProviderDriverKind.make("claudeAgent"), + enabled: true, + installed: true, + version: label, + status: "ready", + auth: { status: "authenticated" }, + checkedAt: "2026-09-26T00:00:00.000Z", + models: [], + slashCommands: [], + skills: [], +}); + +/** Writes one row of each kind labelled `label`. */ +export const writeThreadMachineServices = (label: string) => + Effect.gen(function* () { + const threadId = ThreadId.make("thread-services"); + const states = yield* ThreadMachineStatusStore; + yield* states.put({ + threadId, + state: "starting", + detail: `starting ${label}`, + bootId: null, + updatedAt: "t1", + }); + yield* states.put({ + threadId, + state: "running", + detail: null, + bootId: `boot-${label}`, + updatedAt: "t2", + }); + yield* (yield* ProviderSnapshotStore).put({ + instanceId: ProviderInstanceId.make("claudeAgent"), + snapshot: snapshot(label), + updatedAt: "t2", + }); + const credentials = yield* McpCredentialStore; + for (const token of ["a", "b"]) { + yield* credentials.put({ + tokenHash: `${label}-${token}`, + environmentId: "env", + threadId, + providerSessionId: `session-${token}`, + providerInstanceId: "claudeAgent", + capabilities: ["preview", "review"], + issuedAt: 1_700_000_000_000, + lastAliveAt: 1_700_000_000_000, + }); + } + yield* credentials.touch([`${label}-a`], 1_700_000_600_000); + yield* credentials.remove([`${label}-b`]); + }); + +/** What `writeThreadMachineServices(label)` left, as plain values. */ +export const readThreadMachineServices = Effect.gen(function* () { + const states = yield* (yield* ThreadMachineStatusStore).list(); + const snapshots = yield* (yield* ProviderSnapshotStore).list(); + const credentials = yield* (yield* McpCredentialStore).list(); + return { + states: states.map((row) => [row.threadId, row.state, row.bootId]), + snapshots: snapshots.map((row) => [row.instanceId, row.snapshot.version]), + credentials: credentials.map((row) => [ + row.tokenHash, + row.capabilities, + row.issuedAt, + row.lastAliveAt, + ]), + }; +}); + +export const expectedThreadMachineServices = (label: string) => ({ + states: [["thread-services", "running", `boot-${label}`]], + snapshots: [["claudeAgent", label]], + credentials: [[`${label}-a`, ["preview", "review"], 1_700_000_000_000, 1_700_000_600_000]], +}); diff --git a/apps/server/src/serverModeHooks.ts b/apps/server/src/serverModeHooks.ts index 9847e20bd70b..b40bec9288c9 100644 --- a/apps/server/src/serverModeHooks.ts +++ b/apps/server/src/serverModeHooks.ts @@ -144,3 +144,33 @@ export class ProviderSignInControls extends Context.Reference null }, ) {} + +/** A minted MCP credential as persisted: the token's hash, never the token. */ +export interface McpCredentialRecord { + readonly tokenHash: string; + readonly environmentId: string; + readonly threadId: string; + readonly providerSessionId: string; + readonly providerInstanceId: string; + readonly capabilities: ReadonlyArray; + readonly issuedAt: number; + readonly lastAliveAt: number; +} + +export interface McpCredentialPersistenceShape { + readonly load: Effect.Effect>; + readonly save: (record: McpCredentialRecord) => Effect.Effect; + readonly touch: (tokenHashes: ReadonlyArray, lastAliveAt: number) => Effect.Effect; + readonly remove: (tokenHashes: ReadonlyArray) => Effect.Effect; +} + +/** + * Persists the MCP credentials this server mints. A hub's provider sessions + * run on thread machines and outlive the hub process, so their credentials + * must still resolve after a hub restart. `null` (standalone) keeps them in + * memory only. + */ +export class McpCredentialPersistence extends Context.Reference( + "t3/serverModeHooks/McpCredentialPersistence", + { defaultValue: () => null }, +) {} From a146271d88b15efd0c36ca88555038632e0c327b Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 01:37:45 +0000 Subject: [PATCH 48/58] test(server): confirm hub pairing through the auth CLI and the hub boot capabilities t3 auth pairing create --base-dir --json in hub mode writes the pairing link to the tenant's Postgres rows (nothing to the CLI's base directory), and a running hub exchanges it once for a browser session. The hub boot test also checks the threadMachines capability, machine: null on shells of threads without a machine, and providers that are selectable before any runner reported them. The concurrent-migration test no longer assumes which start applies which migration now that there are three. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../persistence/Postgres/HubDatabase.test.ts | 14 +- apps/server/src/server.hub.test.ts | 136 +++++++++++++++++- 2 files changed, 146 insertions(+), 4 deletions(-) diff --git a/apps/server/src/persistence/Postgres/HubDatabase.test.ts b/apps/server/src/persistence/Postgres/HubDatabase.test.ts index e7a18fdf1a42..04db67f153b8 100644 --- a/apps/server/src/persistence/Postgres/HubDatabase.test.ts +++ b/apps/server/src/persistence/Postgres/HubDatabase.test.ts @@ -47,10 +47,22 @@ describe.skipIf(hubTestDatabaseUrl === undefined)("hub database", () => { ), { concurrency: "unbounded" }, ); + // Each migration is applied exactly once across the concurrent starts, + // and each start applies what it applied in id order. assert.deepStrictEqual( - applied.flat().map((entry) => entry.id), + applied + .flat() + .map((entry) => entry.id) + .toSorted((left, right) => left - right), hubMigrations.map((entry) => entry.id), ); + for (const run of applied) { + const ids = run.map((entry) => entry.id); + assert.deepStrictEqual( + ids, + ids.toSorted((left, right) => left - right), + ); + } // A later migration with a lower id than an applied one still applies. const pool = yield* makeHubPool({ url: schema.adminUrl, maxConnections: 1 }); diff --git a/apps/server/src/server.hub.test.ts b/apps/server/src/server.hub.test.ts index f1e30c9c1352..cbd1ecbb0ada 100644 --- a/apps/server/src/server.hub.test.ts +++ b/apps/server/src/server.hub.test.ts @@ -26,18 +26,24 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as TestConsole from "effect/testing/TestConsole"; +import { Command } from "effect/unstable/cli"; import * as Reactivity from "effect/unstable/reactivity/Reactivity"; -import { FetchHttpClient, HttpServer } from "effect/unstable/http"; +import { FetchHttpClient, HttpClient, HttpClientRequest, HttpServer } from "effect/unstable/http"; import * as HttpApiClient from "effect/unstable/httpapi/HttpApiClient"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; +import { makeCli } from "./bin.ts"; import * as ServerConfig from "./config.ts"; import { ServerEnvironment } from "./environment/ServerEnvironment.ts"; import { OrchestrationEngineService } from "./orchestration/Services/OrchestrationEngine.ts"; +import { ProviderRegistry } from "./provider/Services/ProviderRegistry.ts"; import { AuthAdministrativeScopes } from "@t3tools/contracts"; import { HubDatabase } from "./persistence/Postgres/HubDatabase.ts"; import { makeHubDocuments } from "./persistence/Postgres/HubDocuments.ts"; import { + HUB_TEST_SECRET_KEY, hubTestDatabaseLayer, hubTestDatabaseUrl, hubTestServerConfigLayer, @@ -108,6 +114,7 @@ const startHub = (schema: HubTestSchema, baseDir: string) => engine: Context.getUnsafe(context, OrchestrationEngineService), auth: Context.getUnsafe(context, EnvironmentAuth.EnvironmentAuth), environment: Context.getUnsafe(context, ServerEnvironment), + providers: Context.getUnsafe(context, ProviderRegistry), stop: Fiber.interrupt(fiber), }; }); @@ -198,8 +205,19 @@ describe.skipIf(hubTestDatabaseUrl === undefined)("hub server", () => { [[projectId, "main"]], ); assert.deepStrictEqual( - shell.threads.map((thread) => thread.id), - [threadId], + shell.threads.map((thread) => [thread.id, thread.machine]), + [[threadId, null]], + ); + // Hub capabilities, and providers selectable before any machine reported them. + const descriptor = yield* first.environment.getDescriptor; + assert.strictEqual(descriptor.capabilities.threadMachines, true); + assert.isUndefined(descriptor.capabilities.pullRequests); + const codex = (yield* first.providers.getProviders).find( + (provider) => provider.instanceId === "codex", + ); + assert.deepStrictEqual( + codex && [codex.enabled, codex.installed, codex.status, codex.auth.status], + [true, true, "ready", "unknown"], ); const detail = yield* client.orchestration.threadSnapshot({ headers, @@ -271,4 +289,116 @@ describe.skipIf(hubTestDatabaseUrl === undefined)("hub server", () => { ), ), ); + + it.effect("mints a pairing link with `t3 auth pairing create` that a running hub exchanges", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const schema = yield* makeHubTestSchema(hubTestDatabaseUrl!); + yield* Effect.gen(function* () { + const database = yield* HubDatabase; + yield* makeHubDocuments(database!).write( + "settings.json", + '{ "enableProviderUpdateChecks": false }\n', + ); + }).pipe(Effect.provide(hubTestDatabaseLayer(schema, TENANT))); + const hub = yield* startHub( + schema, + yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-hub-pair-server-" }), + ); + + // The hub host mints pairing credentials with the CLI on its own base + // directory, configured for the tenant's hub database. + const cliBaseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-hub-pair-cli-", + }); + const env: Record = { + T3CODE_SERVER_MODE: "hub", + T3CODE_HUB_DATABASE_URL: schema.runtimeUrl, + ...(schema.separateRuntimeRole ? { T3CODE_HUB_DATABASE_ADMIN_URL: schema.adminUrl } : {}), + T3CODE_HUB_TENANT_ID: TENANT, + T3CODE_HUB_SECRET_KEY: HUB_TEST_SECRET_KEY, + T3CODE_HUB_MACHINES_URL: "http://127.0.0.1:9", + T3CODE_HUB_MACHINES_TOKEN: "unused", + }; + const output = yield* Effect.gen(function* () { + yield* Command.runWith(makeCli(), { version: "0.0.0" })([ + "auth", + "pairing", + "create", + "--base-dir", + cliBaseDir, + "--json", + ]); + return (yield* TestConsole.logLines).findLast( + (line): line is string => typeof line === "string", + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + TestConsole.layer, + NetService.layer, + ConfigProvider.layer(ConfigProvider.fromEnv({ env })), + ), + ), + ); + // @effect-diagnostics-next-line preferSchemaOverJson:off + const issued = JSON.parse(output ?? "{}") as { + readonly id: string; + readonly credential: string; + }; + assert.isString(issued.credential); + + // The pairing link lives in the tenant's Postgres rows, not on disk. + const stored = yield* Effect.gen(function* () { + const { sql } = (yield* HubDatabase)!; + return yield* sql<{ readonly id: string }>` + SELECT id FROM auth_pairing_links + WHERE user_id = ${TENANT} AND revoked_at IS NULL + `; + }).pipe(Effect.provide(hubTestDatabaseLayer(schema, TENANT))); + assert.include( + stored.map((row) => row.id), + issued.id, + ); + assert.notInclude( + yield* fileSystem.readDirectory(cliBaseDir, { recursive: true }), + "userdata/state.sqlite", + ); + + // The running hub exchanges it for a browser session. + const http = yield* HttpClient.HttpClient; + const exchange = (credential: string) => + HttpClientRequest.post(`${hub.origin}/api/auth/browser-session`).pipe( + HttpClientRequest.bodyJson({ credential }), + Effect.flatMap(http.execute), + ); + const exchanged = yield* exchange(issued.credential); + assert.strictEqual(exchanged.status, 200); + const setCookie = exchanged.headers["set-cookie"]; + const cookie = (Array.isArray(setCookie) ? setCookie[0] : setCookie)?.split(";")[0] ?? ""; + assert.isAbove(cookie.length, 0); + const session = yield* http + .execute( + HttpClientRequest.get(`${hub.origin}/api/auth/session`).pipe( + HttpClientRequest.setHeader("cookie", cookie), + ), + ) + .pipe(Effect.flatMap((response) => response.json)); + assert.isTrue((session as { readonly authenticated: boolean }).authenticated); + + // One-time: the same credential cannot be exchanged again. + assert.notStrictEqual((yield* exchange(issued.credential)).status, 200); + yield* hub.stop; + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NetService.layer, + FetchHttpClient.layer, + Reactivity.layer, + ), + ), + ), + ); }); From 8378e0e6cd26dcc5989887626a03eb317a73f15d Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 01:44:29 +0000 Subject: [PATCH 49/58] fix(server): take provider status only from runners that have the hub's settings A runner reports an instance's status when its thread uses that instance (after the hub pushed the instance's settings), or when it predates the push. Other runners would probe with their local defaults and could persist a misleading unauthenticated status for API-key instances; the provider sign-in machine never reports. Refreshing asks such a runner. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/hub/HubProviderSnapshots.test.ts | 10 ++++++ apps/server/src/hub/RemoteProviderDriver.ts | 31 ++++++++++++++----- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/apps/server/src/hub/HubProviderSnapshots.test.ts b/apps/server/src/hub/HubProviderSnapshots.test.ts index 740d3f847529..f1bb9f326a49 100644 --- a/apps/server/src/hub/HubProviderSnapshots.test.ts +++ b/apps/server/src/hub/HubProviderSnapshots.test.ts @@ -130,6 +130,7 @@ describe("hub provider snapshots", () => { snapshot: reported("authenticated"), sessionModelSwitch: "in-session", }), + "runner.provider.configure": () => Effect.succeed({ instances: [claudeInstance] }), }), ); const fake = yield* makeFakeMachineDirectory({ @@ -140,6 +141,15 @@ describe("hub provider snapshots", () => { wakePollInterval: "5 millis", idleCheckInterval: "1 hour", }).pipe(Effect.provideService(MachineDirectory, fake.directory)); + // Only runners whose thread uses the instance report it. + yield* pool.setContextResolver(() => + Effect.succeed({ + projectId: null, + repository: null, + branch: null, + providerInstanceId: claudeInstance, + }), + ); const registry = yield* RemoteSessionRegistry.make; const delivery = yield* RunnerEventDelivery.make.pipe( Effect.provideService(RunnerConnectionPool, pool), diff --git a/apps/server/src/hub/RemoteProviderDriver.ts b/apps/server/src/hub/RemoteProviderDriver.ts index 55030db1aa5d..d15e94280ad7 100644 --- a/apps/server/src/hub/RemoteProviderDriver.ts +++ b/apps/server/src/hub/RemoteProviderDriver.ts @@ -575,21 +575,38 @@ const makeRemoteSnapshot = (input: { return (yield* context.pool.contextOf(threadId))?.providerInstanceId ?? null; }); + /** + * Whether a runner's report of this instance reflects the hub's settings: + * the runner's thread uses the instance (so its settings were pushed), or + * the runner predates the push and only has its own settings anyway. The + * provider sign-in machine never reports: it runs no sessions. + */ + const reportsFor = (connection: RunnerConnection) => + Effect.gen(function* () { + if (connection.threadId === PROVIDER_SIGN_IN_THREAD_ID) return false; + if (connection.hello.protocolVersion < RUNNER_PROTOCOL_PROVIDER_SETTINGS) { + return connection.hello.instances.includes(context.instanceId); + } + return (yield* threadInstance(connection.threadId)) === context.instanceId; + }); + // Scoped to the instance: a rebuilt instance (settings changed) replaces it. yield* context.pool.onConnectionScoped((connection) => Effect.gen(function* () { - let hosted = connection.hello.instances; - if ((yield* threadInstance(connection.threadId)) === context.instanceId) { - const pushed = yield* context.pushSettings(connection); - if (Option.isSome(pushed)) hosted = pushed.value; - } + if (!(yield* reportsFor(connection))) return; + const pushed = yield* context.pushSettings(connection); + const hosted = Option.getOrElse(pushed, () => connection.hello.instances); if (hosted.includes(context.instanceId)) yield* fetchFrom(connection, false, true); }), ); + // Refreshing asks a connected runner whose thread uses this instance. const refresh = Effect.gen(function* () { - const [connection] = yield* context.pool.connections; - if (connection) yield* fetchFrom(connection, true); + for (const connection of yield* context.pool.connections) { + if (!(yield* reportsFor(connection))) continue; + yield* fetchFrom(connection, true, true); + break; + } return yield* Ref.get(current); }); From db9b3299dd5d57e4ff5a99b9b049fe65c8729151 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 01:52:19 +0000 Subject: [PATCH 50/58] fix(server): probe provider status on connect and never cut off work when pausing The end-to-end run showed two defects. A runner that had just booted (or an instance the settings push had just rebuilt) reported its pending, unchecked snapshot, which replaced the persisted signed-in status with auth unknown; the hub now asks for a fresh probe when a runner connects. And threadMachines.pause closed the connection while a title was still being generated on the machine; the pool now refuses to release a machine while any call or stream is using it, not only while a turn runs. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/hub/HubProviderSnapshots.test.ts | 13 +++++++++---- apps/server/src/hub/RemoteProviderDriver.ts | 4 +++- apps/server/src/hub/RunnerConnectionPool.ts | 6 ++++-- apps/server/src/hub/ThreadMachineControls.test.ts | 13 ++++++++++++- apps/server/src/hub/ThreadMachineControls.ts | 5 +++-- packages/contracts/src/rpc.ts | 2 +- packages/contracts/src/threadMachine.ts | 2 +- 7 files changed, 33 insertions(+), 12 deletions(-) diff --git a/apps/server/src/hub/HubProviderSnapshots.test.ts b/apps/server/src/hub/HubProviderSnapshots.test.ts index f1bb9f326a49..000d4020746d 100644 --- a/apps/server/src/hub/HubProviderSnapshots.test.ts +++ b/apps/server/src/hub/HubProviderSnapshots.test.ts @@ -206,9 +206,9 @@ describe("hub provider snapshots", () => { instances: Object.keys(instances).map((id) => ProviderInstanceId.make(id)), }; }), - "runner.provider.getCapabilities": () => + "runner.provider.getCapabilities": ({ refresh }) => Effect.sync(() => { - calls.push("getCapabilities"); + calls.push(refresh ? "getCapabilities refresh" : "getCapabilities"); return { snapshot: { ...reported("authenticated"), instanceId: claudeWork }, sessionModelSwitch: "in-session" as const, @@ -269,7 +269,7 @@ describe("hub provider snapshots", () => { yield* Effect.repeat(instance.snapshot.getSnapshot, { until: (snapshot) => snapshot.auth.status === "authenticated", }).pipe(Effect.timeout("5 seconds")); - expect(calls).toEqual(["configure", "getCapabilities"]); + expect(calls).toEqual(["configure", "getCapabilities refresh"]); yield* instance.adapter.startSession({ threadId, @@ -277,7 +277,12 @@ describe("hub provider snapshots", () => { providerInstanceId: claudeWork, runtimeMode: "full-access", }); - expect(calls).toEqual(["configure", "getCapabilities", "configure", "startSession"]); + expect(calls).toEqual([ + "configure", + "getCapabilities refresh", + "configure", + "startSession", + ]); expect(pushed[1]).toEqual({ claude_work: { driver: "claudeAgent", diff --git a/apps/server/src/hub/RemoteProviderDriver.ts b/apps/server/src/hub/RemoteProviderDriver.ts index d15e94280ad7..996ed7644f68 100644 --- a/apps/server/src/hub/RemoteProviderDriver.ts +++ b/apps/server/src/hub/RemoteProviderDriver.ts @@ -596,7 +596,9 @@ const makeRemoteSnapshot = (input: { if (!(yield* reportsFor(connection))) return; const pushed = yield* context.pushSettings(connection); const hosted = Option.getOrElse(pushed, () => connection.hello.instances); - if (hosted.includes(context.instanceId)) yield* fetchFrom(connection, false, true); + // Probe: a runner that just booted, or an instance the push just + // rebuilt, would otherwise report its pending, unchecked snapshot. + if (hosted.includes(context.instanceId)) yield* fetchFrom(connection, true, true); }), ); diff --git a/apps/server/src/hub/RunnerConnectionPool.ts b/apps/server/src/hub/RunnerConnectionPool.ts index 3b6647ad77ee..13d77321156c 100644 --- a/apps/server/src/hub/RunnerConnectionPool.ts +++ b/apps/server/src/hub/RunnerConnectionPool.ts @@ -139,7 +139,8 @@ export interface RunnerConnectionPoolShape { readonly release: (threadId: ThreadId) => Effect.Effect; /** * Closes the connection now and tells the directory the machine is idle, - * unless a busy mark (a running turn) holds it. Never wakes a machine. + * unless a busy mark (a running turn) or an in-flight call or stream holds + * it: closing would cut that work off. Never wakes a machine. */ readonly idle: (threadId: ThreadId) => Effect.Effect<"idle" | "busy">; /** @@ -735,7 +736,8 @@ export const make = (options: RunnerConnectionPoolOptions = {}) => }), idle: (threadId) => Effect.gen(function* () { - if ((activity.get(threadId)?.busy.size ?? 0) > 0) return "busy" as const; + const entry = activity.get(threadId); + if (entry && (entry.busy.size > 0 || entry.inFlight > 0)) return "busy" as const; const slot = slots.get(threadId); if (slot) yield* closeSlot(threadId, slot, true); yield* reportIdle(threadId); diff --git a/apps/server/src/hub/ThreadMachineControls.test.ts b/apps/server/src/hub/ThreadMachineControls.test.ts index ac6c24a19e3b..5aa09a7d71bb 100644 --- a/apps/server/src/hub/ThreadMachineControls.test.ts +++ b/apps/server/src/hub/ThreadMachineControls.test.ts @@ -2,6 +2,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { it } from "@effect/vitest"; import { type OrchestrationThreadShell, ThreadId } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import { describe, expect } from "vite-plus/test"; @@ -96,12 +97,22 @@ describe("thread machine controls", () => { }).pipe(Effect.scoped, Effect.provide(StoreLayer)), ); - it.live("refuses to release a machine while a turn runs", () => + it.live("refuses to release a machine while a turn or a call uses it", () => Effect.gen(function* () { const { controls, pool } = yield* setup((url) => ({ state: "running", runnerUrl: url })); yield* pool.setBusy(threadId, "turn", true); const error = yield* controls.pause(threadId).pipe(Effect.flip); expect(error.reason).toBe("busy"); + yield* pool.setBusy(threadId, "turn", false); + + // A call still using the machine (a title being generated) holds it too. + const call = yield* pool + .use(threadId, { wake: true, operation: "test" }, () => Effect.never) + .pipe(Effect.forkChild); + yield* Effect.sleep("50 millis"); + expect((yield* controls.pause(threadId).pipe(Effect.flip)).reason).toBe("busy"); + yield* Fiber.interrupt(call); + expect((yield* controls.pause(threadId))?.state).toBe("running"); }).pipe(Effect.scoped, Effect.provide(StoreLayer)), ); diff --git a/apps/server/src/hub/ThreadMachineControls.ts b/apps/server/src/hub/ThreadMachineControls.ts index 0dabc4383b34..911bded48a5b 100644 --- a/apps/server/src/hub/ThreadMachineControls.ts +++ b/apps/server/src/hub/ThreadMachineControls.ts @@ -9,7 +9,8 @@ * the wake continues in the background and is visible on the thread shell. * - `pause` releases the hub's hold: it closes the runner connection and * reports the machine idle, after which the platform pauses it. A running - * turn refuses with `busy`. + * turn, or any call or stream still using the machine (a title being + * generated, an attached terminal), refuses with `busy`. * * Only existing, unarchived threads are controlled; the provider sign-in * machine is managed by the sign-in flow. @@ -97,7 +98,7 @@ export const make = Effect.gen(function* () { return yield* new ThreadMachineControlError({ operation: "threadMachines.pause", reason: "busy", - detail: "A turn is running on this thread's machine.", + detail: "This thread's machine is in use (a turn, a terminal, or another call).", }); } return current(threadId); diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 084f1840ad13..37ce0b8867db 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -527,7 +527,7 @@ export const WsThreadMachinesWakeRpc = Rpc.make(WS_METHODS.threadMachinesWake, { /** * Hub mode: releases the hub's hold on a thread's machine (closes its runner * connection and reports it idle); the platform pauses it after its idle - * delay. Refused with `busy` while a turn runs. + * delay. Refused with `busy` while a turn, a terminal, or another call uses it. */ export const WsThreadMachinesPauseRpc = Rpc.make(WS_METHODS.threadMachinesPause, { payload: ThreadMachineControlInput, diff --git a/packages/contracts/src/threadMachine.ts b/packages/contracts/src/threadMachine.ts index 8b9130c3ff20..1ba80ef27954 100644 --- a/packages/contracts/src/threadMachine.ts +++ b/packages/contracts/src/threadMachine.ts @@ -68,7 +68,7 @@ export const ThreadMachineControlErrorReason = Schema.Literals([ "unsupported", /** The thread does not exist, is archived, or is reserved. */ "not-found", - /** A turn is running on the machine; it cannot be released now. */ + /** A turn, a terminal, or another call is using the machine; it cannot be released now. */ "busy", /** The machine directory or the machine failed; `state` and `detail` say how. */ "unavailable", From 7ae0bcf088ecc1331c32dc40d3281bdde82a5f7e Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 01:53:21 +0000 Subject: [PATCH 51/58] docs: describe the hub client contract and the thread-machine services thread-machines.md now covers machine state on thread shells and the thread-machine activity kinds, threadMachines.wake/pause, branch lists without a machine, provider status and the settings push (runner protocol 2), provider sign-in on the reserved sign-in machine, persisted MCP credentials, migration 051, the remaining gaps, and a Client contract section summarizing everything a web client relies on in hub mode. The e2e script also checks that contract (capability, pending provider snapshots, settings push, shell machine state, listRefs, pause/wake) without extra turns, waits for the checkpoint and the title instead of racing them, and passes 22/22 on a Postgres hub. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/scripts/thread-machines-e2e.mjs | 101 +++++- docs/internals/thread-machines.md | 339 +++++++++++++++++--- 2 files changed, 391 insertions(+), 49 deletions(-) diff --git a/apps/server/scripts/thread-machines-e2e.mjs b/apps/server/scripts/thread-machines-e2e.mjs index a2eb417008d3..6f3516c1287f 100644 --- a/apps/server/scripts/thread-machines-e2e.mjs +++ b/apps/server/scripts/thread-machines-e2e.mjs @@ -15,6 +15,13 @@ * e. a runner restart (new boot), then a turn that resumes the session * f. a hub restart; history and diffs survive * + * Along the way it checks the hub-mode client contract: the threadMachines + * capability, selectable provider snapshots before any runner reported (and + * the persisted report after a restart), the provider settings push, the + * thread shell's machine state and thread-machine.state activities, + * vcs.listRefs on the virtual project root and on the thread's checkout, and + * threadMachines.pause / threadMachines.wake. + * * By default the hub persists to SQLite in its base directory. With * `--hub-database-url` it runs as production hubs do: on Postgres, for a fresh * tenant with a random secret key, seeded through `t3 hub import` from a @@ -379,6 +386,25 @@ async function main() { }); const created = await thread(); check("thread checkout path", created.worktreePath === checkout, created.worktreePath); + + // The hub-mode client contract before any machine ran. + const config = await rpc("server.getConfig", {}); + check( + "threadMachines capability without pullRequests", + config.environment.capabilities.threadMachines === true && + config.environment.capabilities.pullRequests === undefined, + ); + const pending = config.providers.find((provider) => provider.instanceId === "claudeAgent"); + check( + "provider selectable before any runner reported", + pending?.enabled === true && pending.installed && pending.status === "ready", + `${pending?.status} / auth ${pending?.auth.status}`, + ); + const projectRefs = await rpc("vcs.listRefs", { cwd: `/workspace/p/${projectId}` }); + check( + "blank project lists no refs without a machine", + projectRefs.isRepo === false && projectRefs.refs.length === 0, + ); const shell = await http("GET", "/api/orchestration/shell"); const project = (shell.projects ?? shell.snapshot?.projects ?? []).find( (entry) => entry.id === projectId, @@ -395,13 +421,59 @@ async function main() { ); const notes = NodeFS.readFileSync(NodePath.join(checkout, "notes.txt"), "utf8"); check("turn 1 edited the runner checkout", notes.includes("edited by the runner")); - const checkpoint = first.checkpoints.find((entry) => entry.checkpointTurnCount === 1); + // The checkpoint is captured just after the turn settles. + const captured = await waitFor("turn 1 checkpoint", async () => { + const current = await thread(); + return current.checkpoints.some( + (entry) => entry.checkpointTurnCount === 1 && entry.status === "ready", + ) + ? current + : undefined; + }); + const checkpoint = captured.checkpoints.find((entry) => entry.checkpointTurnCount === 1); check( "turn 1 checkpoint captured", checkpoint?.status === "ready" && checkpoint.files.some((file) => file.path === "notes.txt"), JSON.stringify(checkpoint?.files ?? []), ); + // The machine, the settings push, and the runner's provider report. + const shellAfterTurn = await http("GET", "/api/orchestration/shell"); + const shellThread = (shellAfterTurn.threads ?? shellAfterTurn.snapshot?.threads ?? []).find( + (entry) => entry.id === threadId, + ); + check( + "thread shell carries the machine state", + shellThread?.machine?.state === "running", + JSON.stringify(shellThread?.machine ?? null), + ); + check( + "thread-machine.state activity recorded", + first.activities.some( + (activity) => + activity.kind === "thread-machine.state" && activity.payload?.state === "running", + ), + ); + check( + "provider settings pushed to the runner", + /runner rpc configure[\s\S]{0,120}claudeAgent/.test( + NodeFS.readFileSync(NodePath.join(logs, "runner.log"), "utf8"), + ), + ); + const reported = (await rpc("server.getConfig", {})).providers.find( + (provider) => provider.instanceId === "claudeAgent", + ); + check( + "runner reported the provider", + reported?.auth.status === "authenticated", + reported?.auth.status, + ); + const threadRefs = await rpc("vcs.listRefs", { cwd: checkout }); + check( + "thread checkout lists its refs", + threadRefs.refs.some((ref) => ref.name === "main"), + threadRefs.refs.map((ref) => ref.name).join(","), + ); // c. The diff through the hub. const t0 = Date.now(); const diff = await rpc("orchestration.getTurnDiff", { @@ -416,6 +488,12 @@ async function main() { ); // d. With the runner stopped, the hub still serves the diff and git status. + // The first turn's title is generated on the runner; let it finish first. + await waitFor( + "title generation", + async () => ((await thread()).title !== "New thread" ? true : undefined), + 90_000, + ).catch((error) => log(`title not generated before the runner stopped: ${error.message}`)); await stop("runner"); const t1 = Date.now(); const cached = await rpc("orchestration.getTurnDiff", { @@ -442,6 +520,19 @@ async function main() { /runner rpc startSession[\s\S]{0,200}resume: true/.test(runnerLog), ); + // Releasing the machine waits for work still using it (the title, for one). + const paused = await waitFor( + "threadMachines.pause", + () => rpc("threadMachines.pause", { threadId }), + 60_000, + ); + const woken = await rpc("threadMachines.wake", { threadId }); + check( + "threadMachines.pause and threadMachines.wake", + paused?.state === "running" && woken?.state === "running", + `${paused?.state} / ${woken?.state}`, + ); + // f. A hub restart keeps history and diffs. const beforeRestart = await thread(); await stop("hub"); @@ -459,6 +550,14 @@ async function main() { toTurnCount: 1, }); check("turn diff after hub restart", afterDiff.diff === diff.diff); + const restartedProvider = (await rpc("server.getConfig", {})).providers.find( + (provider) => provider.instanceId === "claudeAgent", + ); + check( + "provider status after the hub restart", + restartedProvider?.auth.status === "authenticated", + restartedProvider?.auth.status, + ); check( "title generated on the runner", afterRestart.title !== "New thread", diff --git a/docs/internals/thread-machines.md b/docs/internals/thread-machines.md index 0829014e914e..bec7222af560 100644 --- a/docs/internals/thread-machines.md +++ b/docs/internals/thread-machines.md @@ -26,19 +26,20 @@ checkout path. A runner never persists orchestration state. ### Hub -| Variable | Meaning | -| ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `T3CODE_SERVER_MODE=hub` | Enables hub mode | -| `T3CODE_HUB_DATABASE_URL` | Postgres URL of the hub schema, for a role without `BYPASSRLS`. libpq's `sslrootcert=system` is accepted. Without it the hub persists to SQLite (tests, development) | -| `T3CODE_HUB_DATABASE_ADMIN_URL` | Optional; the schema owner, used only to apply hub migrations and grant the runtime role. Defaults to the runtime URL | -| `T3CODE_HUB_TENANT_ID` | The Aldo user ID whose rows this process owns; one hub process serves one user. Required with a database URL | -| `T3CODE_HUB_SECRET_KEY` | Base64 32-byte key encrypting the tenant's secrets at rest. Required with a database URL | -| `T3CODE_HUB_MACHINES_URL` | Base URL of the machine directory (served by the platform host process) | -| `T3CODE_HUB_MACHINES_TOKEN` | Bearer for the machine directory | -| `T3CODE_HUB_PUBLIC_URL` | Base URL at which thread machines reach the hub. Runners are given `/mcp` as the `t3-code` MCP endpoint; without it only a runner on the hub's host can reach MCP | -| `T3CODE_HUB_CHECKOUT_ROOT` | Root of thread checkouts (`/workspace/t`). Hub and machines must agree; only tests and development change it | -| `T3CODE_RUNNER_URL` | Development only: a single `t3 runner` WebSocket URL used instead of the machine directory (static directory, always `running`) | -| `T3CODE_RUNNER_TOKEN` | Development only: the bearer presented to that runner | +| Variable | Meaning | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `T3CODE_SERVER_MODE=hub` | Enables hub mode | +| `T3CODE_HUB_DATABASE_URL` | Postgres URL of the hub schema, for a role without `BYPASSRLS`. libpq's `sslrootcert=system` is accepted. Without it the hub persists to SQLite (tests, development) | +| `T3CODE_HUB_DATABASE_ADMIN_URL` | Optional; the schema owner, used only to apply hub migrations and grant the runtime role. Defaults to the runtime URL | +| `T3CODE_HUB_TENANT_ID` | The Aldo user ID whose rows this process owns; one hub process serves one user. Required with a database URL | +| `T3CODE_HUB_SECRET_KEY` | Base64 32-byte key encrypting the tenant's secrets at rest. Required with a database URL | +| `T3CODE_HUB_MACHINES_URL` | Base URL of the machine directory (served by the platform host process) | +| `T3CODE_HUB_MACHINES_TOKEN` | Bearer for the machine directory | +| `T3CODE_HUB_PUBLIC_URL` | Base URL at which thread machines reach the hub. Runners are given `/mcp` as the `t3-code` MCP endpoint; without it only a runner on the hub's host can reach MCP | +| `T3CODE_HUB_CHECKOUT_ROOT` | Root of thread checkouts (`/workspace/t`). Hub and machines must agree; only tests and development change it | +| `T3CODE_HUB_THREAD_BROWSER_URL_TEMPLATE` | Same-origin URL of a thread machine's browser page with `{threadId}`, linked from provider sign-in flows that finish in a browser. Default `/_devpc/threads/{threadId}/browser` | +| `T3CODE_RUNNER_URL` | Development only: a single `t3 runner` WebSocket URL used instead of the machine directory (static directory, always `running`) | +| `T3CODE_RUNNER_TOKEN` | Development only: the bearer presented to that runner | A hub needs the machine directory (URL and token) or a development runner URL. The database, tenant, and secret key go together: with them the hub is production-shaped and its @@ -108,6 +109,20 @@ never wakes. Both return: } ``` +The same base URL serves the platform's machine-free lookups (the hub host forwards them to +the control plane; see `docs/hub-host.md` in the remote-dev repository): + +``` +GET /repositories/refs?url= → { defaultBranch, refs: [{ name, sha }], truncated } +GET /provider-homes → { providers: [{ provider, version, updatedAt }] } +DELETE /provider-homes/{provider} → { provider, deleted } +``` + +Errors carry `{ "error": "CODE" }` (`REPOSITORY_ACCESS_REQUIRED`, `REPOSITORY_NOT_FOUND`, +`NOT_HUB_HOSTED`, `REPOSITORY_UNSUPPORTED`, `REPOSITORY_REFS_UNAVAILABLE`), surfaced as +`MachineDirectoryError.code`. The development directory (`T3CODE_RUNNER_URL`) lists no +repositories and no provider homes. + `runner` is present only in `running`. A changed `bootId` means the machine restarted; the hub reconciles sessions and turns that were active on the previous boot. `idle` tells the platform no session or turn needs the machine; the platform decides when to pause or @@ -133,8 +148,9 @@ Hub behavior (`hub/MachineDirectory.ts`, `hub/RunnerConnectionPool.ts`): ## Runner protocol `packages/contracts/src/runner.ts` (`@t3tools/contracts/runner`) defines `RunnerRpcGroup`, -versioned independently of the client protocol: `RUNNER_PROTOCOL_VERSION = 1`, -`RUNNER_MIN_PROTOCOL_VERSION = 1`. `runner.hello` sends the hub's range and the thread; a +versioned independently of the client protocol: `RUNNER_PROTOCOL_VERSION = 2`, +`RUNNER_MIN_PROTOCOL_VERSION = 1`. Version 2 adds `runner.provider.configure` and +`runner.auth.*`; a hub never calls them on a runner that negotiated 1. `runner.hello` sends the hub's range and the thread; a runner whose range does not overlap refuses with `RunnerProtocolMismatchError`, and one bound to another thread with `RunnerThreadMismatchError`. The reply carries the negotiated version, `runnerId` (the outbox identity, stable across restarts), `bootId` (per process), @@ -150,7 +166,8 @@ with the server's schema and decoded back on the hub. | Group | RPCs | | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Handshake | `runner.hello` | -| Provider | `runner.provider.{startSession, sendTurn, interruptTurn, respondToRequest, respondToUserInput, stopSession, listSessions, readThread, rollbackThread, getCapabilities}` | +| Provider | `runner.provider.{startSession, sendTurn, interruptTurn, respondToRequest, respondToUserInput, stopSession, listSessions, readThread, rollbackThread, getCapabilities, configure}` | +| Sign-in | `runner.auth.{start, get, submit, cancel}` (only on the provider sign-in machine) | | Text | `runner.text.{generateThreadTitle, generateBranchName}` | | Events | `runner.events.subscribe` (stream), `runner.events.ack` | | Checkout | `runner.checkout.prepare` | @@ -171,8 +188,23 @@ with its own. Commit and PR text are generated on the runner inside git actions. origin`, with the machine's own git configuration and credential helper) or, without a repository, initialized; an existing one is never reset. When `branch` is not checked out it is switched to if it exists locally, else created from `origin/` (pushed from an -earlier machine), else from `baseRef` (remote-tracking first), else `HEAD`. Network access -happens only when cloning or creating a branch. +earlier machine), else from `baseRef` (remote-tracking first), else `HEAD`. A `baseRef` of +`HEAD` (`DEFAULT_BRANCH_BASE_REF`, what a client sends as `prepareWorktree.baseBranch` when +no base was picked) means the repository's default branch: `origin/HEAD`, recorded with +`git remote set-head origin --auto` when the checkout lacks it; the directory is then asked +for the default ref (`repository.ref: null`). Network access happens only when cloning or +creating a branch. + +`runner.provider.configure` carries the hub's effective settings for provider instances +(explicit `providerInstances` or the legacy `providers.` mirror), sensitive +environment values decrypted from the hub's secret store. The hub pushes the thread's +instance when its runner connects, and the session's instance before every session start +and before title or branch-name generation. The runner layers them over its own settings in +memory only (`runner/RunnerProviderSettings.ts`): nothing is written to the machine beyond +the CLI's own files, a restarted runner starts from its local settings until the next push, +and identical settings are a no-op (changed ones rebuild the instance, as a settings change +does on any server). A runner serves one thread, so the settings apply to that thread's +sessions only. Only instance ids are logged. ## Wake semantics @@ -181,14 +213,81 @@ Whether a call may resume or recreate a sleeping machine is decided per call: | Wakes the machine | Never wakes it | | ----------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | | Provider session start, `sendTurn` (turn start), approval and user-input responses, interrupting an active turn, rollback (revert) and `readThread` | `hasSession` / `listSessions` (answered by the hub; a session on a sleeping machine is resumable), `stopSession`, provider capability refresh | -| Thread bootstrap and checkout preparation before each turn, setup scripts | Git status: `getStatus`, `streamStatus`, refreshes (hub cache; a connected runner refreshes it), `listRefs`, invalidations | +| Thread bootstrap and checkout preparation before each turn, setup scripts | Git status: `getStatus`, `streamStatus`, refreshes (hub cache; a connected runner refreshes it), invalidations | +| `threadMachines.wake` | `listRefs` (project roots and sleeping checkouts list the repository from the platform), `threadMachines.pause` | | Terminal `open`, `attach`, `restart` | Terminal `write`, `resize`, `clear` (fail with `TerminalNotRunningError`), `close` (no-op), terminal events and metadata | | File writes; `vcs.init`; git actions (pull, commit/push/PR stacked actions, refs, worktrees, branch rename, PR resolve and thread preparation, fetch) | File list, search, content search and read; review diff preview and file contents; workspace index refresh | | Title and branch-name generation (only issued at turn start) | Turn and full-thread diffs that were captured (hub cache); a diff never captured wakes once and is then cached | A read that needs a running machine fails with its service's own error type whose cause is `ThreadMachineUnavailableError { reason: "asleep", state }`, which clients can render as -a resumable state. Clients should offer an explicit action (a turn, a terminal) to wake. +a resumable state. Clients should offer an explicit action (a turn, a terminal, or +`threadMachines.wake`) to wake. + +`threadMachines.wake({ threadId })` resumes or recreates the machine through the +connection pool (the same path a turn takes, so the runner is connected once it is up) and +answers within two seconds with the state reached so far; the rest continues in the +background and shows on the thread shell. `threadMachines.pause({ threadId })` closes the +runner connection and reports the machine idle, after which the platform pauses it after +its idle delay; a running turn, an attached terminal or any other call still using the machine +refuses with `busy`. Both answer +`ThreadMachineControlError { reason: "unsupported" | "not-found" | "busy" | "unavailable", +state?, detail }`; standalone servers answer `unsupported`, and the provider sign-in thread +is `not-found` (the sign-in flow manages it). + +## Machine state on thread shells + +Every machine directory response (ensure, status, wake polls) passes through +`hub/ThreadMachineStates.ts`, and a release is recorded as `none`. A change of state or boot, +or new progress detail while a machine is `preparing`, `starting` or `failed`, is persisted +(`hub_thread_machine_status`) and published. `HubRunnerLifecycle` records each change as a +`thread-machine.state` activity; that domain event is what re-publishes the thread shell, +and shells read the state through the `ThreadMachineStatusReader` hook, so the shell +snapshot and every upsert carry `machine: { state, detail, updatedAt }` (`null` when no +machine is known). Machines the hub believes awake but holds no connection to are re-checked +every minute without waking them, so a machine the platform paused after an idle report +shows as `paused`. Standalone shells never carry the field. + +Thread-machine activity kinds (`THREAD_MACHINE_ACTIVITY_KINDS`): + +| Kind | Tone | Payload | When | +| ----------------------------------- | -------------------- | ---------------------------------- | ------------------------------------------ | +| `thread-machine.state` | `error` for `failed` | `{ state, detail, bootId }` | Every machine state transition | +| `thread-machine.checkout.preparing` | `info` | `{ checkout, branch, repository }` | Bootstrap asks the runner for the checkout | +| `thread-machine.failed` | `error` | `{ detail }` | Bootstrap or checkout preparation failed | + +The earlier one-off `thread-machine.starting` activity is folded into `thread-machine.state`. + +## Branch lists without a machine + +`vcs.listRefs` on a project's virtual root (`/workspace/p/`) is served from the +platform's repository refs for the project's recorded repository +(`hub/HubRepositoryRefs.ts`), mapped to the ordinary result: branch names as local refs (or +`origin/` remote refs for `refKind: "remote"`), the default branch `isDefault`, +current first, then default, filtered and paginated like a checkout, `isRepo: true`, +`hasPrimaryRemote: true`. Listings are cached for 30 s per repository; `refresh: true` +bypasses the cache. A blank project lists nothing (`isRepo: false`), a repository the +platform cannot list (`REPOSITORY_UNSUPPORTED`, not GitHub) lists nothing with `isRepo: +true`, and other platform errors fail with a `GitCommandError` whose detail says what to do +(for example, connect the repository to the Aldo GitHub App). A thread checkout with a +running machine lists the runner's refs; a sleeping one gets the platform listing with the +thread's branch marked current (and added when it was never pushed), so listing never wakes +a machine. + +## Provider status and settings + +A hub cannot probe provider CLIs. `hub/HubProviderSnapshots.ts` keeps the last snapshot a +runner reported per provider instance, persisted in `hub_provider_snapshots`, so a restarted +hub, or one whose machines are asleep, shows real status, auth and models. A runner reports +an instance when its thread uses that instance (on connect, right after the hub pushed the +instance's settings, and on refresh), so a report always reflects the hub's settings; the +provider sign-in machine reports only the instance it just signed in. Before any runner reported an instance, its snapshot is the +driver's pending one (built-in and custom models; manifest models when the driver lists +none) marked `installed: true`, `status: "ready"`, `auth: { status: "unknown" }`, so the +composer can select it and the first turn starts a machine. The hub's settings stay +authoritative for identity and enablement, and update advisories are dropped (a hub cannot +update CLIs on machines). Settings reach runners through `runner.provider.configure` (see +the runner protocol). ## Durable delivery and crash reconciliation @@ -249,11 +348,59 @@ bindings live (`ProviderAdapterCapabilities.sessionsOutliveServer`). Bootstrap in hub mode (`hub/HubThreadCheckouts.ts`, used by `CommandDispatcher`) creates the thread with its checkout path, ensures and wakes its machine with the project's repository identity and the branch, and calls `runner.checkout.prepare`, then runs the setup script in a -routed terminal. Progress and failures are thread activities: `thread-machine.starting` -("Starting machine", while waking), `thread-machine.checkout.preparing` ("Preparing -checkout") and `thread-machine.failed` (tone `error`). A failed bootstrap deletes the new -thread as in standalone mode. Before every turn the checkout is prepared again (skipped when -the same runner boot already did). +routed terminal. Machine progress is `thread-machine.state` activities; checkout progress +and failures are `thread-machine.checkout.preparing` ("Preparing checkout") and +`thread-machine.failed` (tone `error`). The base ref is `prepareWorktree.baseBranch`, where +`HEAD` means the repository's default branch. A failed bootstrap deletes the new thread as +in standalone mode. Before every turn the checkout is prepared again (skipped when the same +runner boot already did). + +## Provider sign-in + +On a hub, `server.startAuthConnector` (and `get`, `submit`, `cancel`) runs T3's unchanged +auth connector on the machine of the reserved thread `aldo-provider-sign-in` +(`PROVIDER_SIGN_IN_THREAD_ID`; project `aldo-provider-sign-in`, repository `null`, checkout +`/workspace/t/aldo-provider-sign-in`), through `hub/HubProviderSignIn.ts`: + +1. `start` answers at once with `status: "starting"`, `stage: "preparing"`, message + "Starting a machine for sign-in…" and the flow the method will use. In the background + the hub reads the provider's stored sign-in version, ensures and wakes the sign-in + machine, pushes the instance's settings, and starts the connector there + (`runner.auth.start`). The session then mirrors the runner's (prompts, verification URLs, + codes, fields) under the hub's session id; `submit` and `cancel` are forwarded. Flows that + finish in a browser on the machine (`flow: "browser"`) carry `workspaceBrowserUrl` + (`T3CODE_HUB_THREAD_BROWSER_URL_TEMPLATE` with the sign-in thread). +2. When the login command succeeds the session stays `starting`/`verifying` ("Saving your + sign-in…") until `GET provider-homes` lists the provider at a higher version (the + supervisor uploads within about 20 s; the hub waits up to two minutes, then fails with + "was not saved"). The instance's status is then read again on the machine and the session + succeeds. +3. Once no sign-in is active the sign-in machine is reported idle. The remote provider + driver refuses agent sessions on that thread, and runners serve `runner.auth.*` only when + bound to it. + +Connectors map to provider homes as `claude`→`claude`, `codex`→`codex`, `opencode`→`opencode`, +`grok`→`grok`, `cursor`→`cursor`, `prime-agent`→`prime`, `muse`→`muse`. Source-control +connectors (GitHub, GitLab, Azure DevOps, Bitbucket) answer `AuthConnectorError` on a hub. +`server.listProviderSignIns` lists stored sign-ins (`{ signIns: [{ connector, version, +updatedAt }] }`) and `server.signOutProvider({ connector })` deletes one +(`DELETE provider-homes/{provider}`; running machines drop the files within about 15 s, and +the provider's snapshots return to auth `unknown`). Standalone servers keep the local auth +connector and answer the two new calls with `AuthConnectorError`. A stored sign-in is not +proof of a valid login; the provider's status on a machine is. + +## MCP credentials + +The hub mints an MCP credential for every provider session it starts, and the runner hands +it to the provider CLI (`T3CODE_HUB_PUBLIC_URL/mcp`). Those sessions outlive the hub, so a hub +persists each credential's SHA-256 hash and scope, never the token, in +`hub_mcp_credentials` (`McpCredentialPersistence` hook, `hub/HubMcpCredentials.ts`). A +restarted hub loads the unexpired ones and keeps answering the sessions' MCP calls; +revocation deletes them, liveness is written through at most every ten minutes, and the +shutdown `revokeAll` clears only the process's memory. Standalone servers keep credentials +in memory, as before. A session a recreated machine lost is restarted without an MCP +credential when the hub restarted since it was minted (the raw token is not kept); the next +explicit session start mints a new one. ## Hub persistence @@ -269,7 +416,7 @@ with `user_id`: | `secrets/*.bin` | `hub_secrets`, AES-256-GCM with `T3CODE_HUB_SECRET_KEY`; the authenticated data binds tenant and secret name | | `attachments/` | `hub_attachments` (`bytea`); the local directory is a cache filled on lookup | | Repository identity from `git remote` on every read | `projection_projects.repository_identity_json`, recorded by `project.create` / `project.meta.update` | -| (none) | Thread-machine state, migration 050: see below | +| (none) | Thread-machine state, migrations 050 and 051: see below | Thread-machine state (`persistence/Services/HubThreadMachineState.ts`): @@ -279,6 +426,12 @@ Thread-machine state (`persistence/Services/HubThreadMachineState.ts`): whitespace modes. It is the hub's only diff table; standalone `checkpoint_diff_blobs` is never written, has no hub table, and is not imported. - `hub_thread_vcs_status` (`ThreadVcsStatusStore`): the last git status a runner reported. +- `hub_thread_machine_status` (`ThreadMachineStatusStore`, migration 051): the last machine + state per thread, shown on thread shells. +- `hub_provider_snapshots` (`ProviderSnapshotStore`, 051): the last provider snapshot a + runner reported per provider instance. +- `hub_mcp_credentials` (`McpCredentialStore`, 051): hashes and scopes of minted MCP + credentials. Machine-published themes stay local (a hub has no desktop to publish them). @@ -304,7 +457,7 @@ runner, so a hub on a fresh base directory still sends them. **Migrations.** Hub migrations are numbered entries in `apps/server/src/persistence/Postgres/migrations/index.ts`: 001–049 for hub persistence, -050–099 for thread-machine state (050 today). A hub applies pending ones at startup with +050–099 for thread-machine state (050 and 051 today). A hub applies pending ones at startup with `T3CODE_HUB_DATABASE_ADMIN_URL` (or the runtime URL), each in its own transaction under a transaction-scoped advisory lock, recorded in `hub_schema_migrations`. They are applied by id, not list position, so separately owned ranges can land in any order. Every new tenant table @@ -340,46 +493,69 @@ prints only equal/different with counts. `server.ts` builds one runtime for every mode from the `ServerModeLayers` record; standalone uses the local layers and a hub uses `hub/HubLayers.ts` for the checkout-bound groups (checkpointing, git, VCS, terminals, workspace, provider instances), the recorded repository -identity (`HubRepositoryIdentityResolver`), and hub-only infrastructure (machine directory, -connection pool, session registry, event delivery, state stores, git status cache) and -background work. `HubDatabase.layerConfig` sits under the whole runtime. Shared code sees hub -mode only through the `serverModeHooks.ts` references (deterministic ingestion ids, -`HubThreadCheckouts`), `CheckoutGitProbe`, and `HubDatabase`, whose defaults are standalone -behavior. `t3 runner` is composed in `runner/RunnerServer.ts` without orchestration, -projections or the client API. +identity (`HubRepositoryIdentityResolver`), and hub-only infrastructure (machine directory +observed by the machine states, connection pool, session registry, event delivery, state +stores, provider snapshots, persisted MCP credentials, git status cache) and background +work. `HubDatabase.layerConfig` sits under the whole runtime. Shared code sees hub mode only +through the `serverModeHooks.ts` references (deterministic ingestion ids, +`HubThreadCheckouts`, `ThreadMachineStatusReader` for shells, `ThreadMachineControls` and +`ProviderSignInControls` for the client API, `McpCredentialPersistence`), +`CheckoutGitProbe`, and `HubDatabase`, whose defaults are standalone behavior. `t3 runner` +is composed in `runner/RunnerServer.ts` without orchestration, projections or the client +API; its provider registry is `RunnerProviderSettings`. ## Gaps Not available on a hub (typed errors, never the hub's disk): filesystem browsing for the -project picker, repository clone and publish, and workspace-file, media and project-favicon -asset URLs. +project picker, repository clone and publish, workspace-file, media and project-favicon +asset URLs, source-control sign-in (GitHub, GitLab, Azure DevOps, Bitbucket connectors), +and the pull-request workspace (the `pullRequests` capability is absent until PR APIs work +without `gh` on a checkout). -Still running where the hub runs, to move to runners or the credential service: previews, -provider sign-in (auth connector), provider maintenance, usage scanning, workflow scripts, -and pull-request listing (which needs provider credentials). +Still running where the hub runs, to move to runners or the credential service: provider +maintenance (update advisories are hidden), usage scanning, and workflow scripts. Previews +and the thread browser are served by the platform's `/_devpc/threads/{threadId}/…` routes. Not built yet: -- No client records a project's repository identity; the platform (or the import, from a - checkout) must dispatch `project.create` / `project.meta.update` with `repositoryIdentity`. - A hub without a database records none. +- Repository identity is recorded only when a client or the platform sends + `repositoryIdentity` on `project.create` / `project.meta.update`; a hub without a + database records none, so its projects are blank. - A hub behind a runner's `firstRetainedSequence` (more than 200,000 unacknowledged events) only logs the gap; resynchronizing from `readThread` is future work. +- The platform does not produce `saved` yet (idle machines pause); a `paused` machine is + only noticed by the minute-long watch after the hub reported it idle, so the shell may + show `running` for up to a minute after the platform paused it. +- Thread detail (`OrchestrationThread`) does not carry `machine`; clients read it from the + shell. +- On connect the hub pushes settings only for the thread's instance; other instances on + that runner report status with the runner's local defaults until a session starts them. +- The raw MCP token is not persisted, so a session a recreated machine lost after a hub + restart restarts without MCP tools until the next explicit session start. - Attachment bytes live in `bytea` within the upload limits; object storage can replace the table behind the `HubAttachments` helpers. ## Testing - Unit: `apps/server/src/hub/*.test.ts` (wake semantics with a fake directory and fake - runner, delivery and reconciliation, routed services, the machine directory contract), - `apps/server/src/runner/*.test.ts` (outbox, checkout, handler guards), - `packages/contracts/src/runner.test.ts`, and the thread-machine stores on SQLite. + runner, delivery and reconciliation, routed services, the machine directory contract + including repository refs and provider homes, machine states and controls, repository + refs, provider snapshots and the settings push, provider sign-in), + `apps/server/src/runner/*.test.ts` (outbox, checkout including the `HEAD` base ref, + handler guards, provider settings), `apps/server/src/mcp/McpSessionRegistry.test.ts` + (persisted credentials), `packages/contracts/src/runner.test.ts`, and the thread-machine + stores on SQLite (`persistence/Layers/HubThreadMachineState.test.ts`). +- Standalone behavior: `src/environment/ServerEnvironment.test.ts` (capabilities) and + `src/server.test.ts` (hub-only RPCs answer `unsupported` / `AuthConnectorError`), plus + the rest of the server suite. - Postgres: set `T3_HUB_TEST_DATABASE_URL` to a disposable database the tests may create schemas in (for example a private loopback cluster as `postgres`). Each test gets its own schema and, when the connecting role bypasses RLS on loopback, a runtime role without `BYPASSRLS`, so row-level security is exercised. This enables the persistence tests - (`persistence/Postgres/*.test.ts`, including the thread-machine stores and migration 050), - the hub server boot (`server.hub.test.ts`), attachment restore before shipping to a + (`persistence/Postgres/*.test.ts`, including the thread-machine stores and migrations 050 + and 051), the hub server boot (`server.hub.test.ts`: capabilities, shells, pending provider + snapshots, and `t3 auth pairing create` in hub mode writing to Postgres with a browser + session exchange against the running hub), attachment restore before shipping to a runner, and the loopback integration on Postgres. Without the variable they skip, and the loopback runs on SQLite. - Loopback integration: `apps/server/integration/hubRunnerLoopback.integration.test.ts` runs @@ -389,4 +565,71 @@ Not built yet: [--hub-database-url [--hub-database-admin-url ]]` starts a runner (4422) and a hub (4421) on loopback and drives the hub's public API. With a database URL the hub runs on Postgres for a fresh tenant seeded through `t3 hub import`, and restarts on an empty - base directory. + base directory. Beyond the two turns it checks the client contract: the capability, + selectable provider snapshots before any runner reported, the settings push, the shell's + machine state and `thread-machine.state` activities, `vcs.listRefs` on the project root + and the checkout, and `threadMachines.pause` / `wake`. + +## Client contract + +Everything a web client relies on when an environment is a hub. All of it is additive and +optional in `packages/contracts`: standalone servers never send the new fields and answer the +new calls as unsupported, and older clients ignore them. + +**Detecting a hub.** `ExecutionEnvironmentDescriptor.capabilities.threadMachines === true`; +hubs omit `pullRequests`, so clients must not probe the pull-request APIs. Managed hubs also +set `serverMode: "hub"` in the platform bootstrap (with `threadPreviewUrlTemplate` and +`threadBrowserUrlTemplate`). + +**Projects and threads.** `project.create` on a hub gets the virtual root +`/workspace/p/` (whatever the client sends) and should carry +`repositoryIdentity` for a repository project; a project without one is blank. Every thread +gets `worktreePath` `/workspace/t/` on creation or bootstrap. Bootstrap uses +`prepareWorktree.branch ?? createThread.branch`, and `prepareWorktree.baseBranch` may be +`"HEAD"` (`DEFAULT_BRANCH_BASE_REF`) to mean the repository's default branch, so a client +need not list branches first. `projectCwd` is ignored; `startFromOrigin` has no effect. + +**Machine state.** `OrchestrationThreadShell.machine: { state, detail, updatedAt } | null` +on the shell snapshot and every `thread-upserted` event (`state` is `ThreadMachineState`: +`none`, `preparing`, `starting`, `running`, `paused`, `saved`, `failed`; `null` means no +machine is known). Activities: `thread-machine.state` `{ state, detail, bootId }` on every +transition (tone `error` when `failed`), `thread-machine.checkout.preparing`, and +`thread-machine.failed` `{ detail }` (tone `error`); see `THREAD_MACHINE_ACTIVITY_KINDS` and +`ThreadMachineStateActivityPayload`. Clients typically hide these activities unless they are +errors and render the shell's `machine` instead. + +**Controls.** `threadMachines.wake({ threadId })` and `threadMachines.pause({ threadId })` +return the shell's `ThreadMachineStatus | null` or fail with `ThreadMachineControlError` +(`unsupported`, `not-found`, `busy`, `unavailable` with `state?` and `detail`). Turns, +terminal opens, file writes and git actions wake a machine on their own; reads never do and +fail with their own error type caused by `ThreadMachineUnavailableError { reason: "asleep", +state }` (render as "asleep, wakes on send"). + +**Branches.** `vcs.listRefs({ cwd: "/workspace/p/" })` lists the repository's +branches from the platform (default branch `isDefault`, names without `origin/`), cached +30 s, `refresh: true` to bypass; blank projects answer `isRepo: false` with no refs, and a +missing GitHub App connection fails with a `GitCommandError` whose message says so. The same +call on a thread checkout lists the machine's refs, or the platform's with the thread's +branch `current` when the machine is asleep. `vcs.status` on a project root is not served +(a draft has no checkout). + +**Providers.** `server.getConfig` / `subscribeServerConfig` providers are available before +any machine ran: `installed: true`, `status: "ready"`, `auth.status: "unknown"` until a runner +reports, then the last reported status (persisted across hub restarts). Provider settings, +including API keys, stay hub settings; the hub delivers them to machines. + +**Sign-in.** The existing `server.startAuthConnector` / `getAuthConnector` / +`submitAuthConnector` / `cancelAuthConnector` flow works unchanged on a hub for provider +connectors (not source control). The first session answers `status: "starting"`, +`stage: "preparing"`, message "Starting a machine for sign-in…" while the sign-in machine +starts, then mirrors the provider's prompt; after the CLI finishes it stays `verifying` +("Saving your sign-in…") until the platform stored it, then `succeeded`. Sessions with +`flow: "browser"` carry `AuthConnectorSession.workspaceBrowserUrl` (the machine's browser +page, same origin, frameable). `server.listProviderSignIns({})` → +`{ signIns: [{ connector, version, updatedAt }] }` and +`server.signOutProvider({ connector })` → `{ connector, signedOut }` exist only on hubs +(standalone: `AuthConnectorError`). + +**Unchanged.** Previews and the thread browser are platform routes +(`/_devpc/threads/{threadId}/preview/{port}`, `/_devpc/threads/{threadId}/browser`); pairing +and sessions work as on any T3 server. From a3e999fdb44fd5ec0bb8cadf654cfa478c2bba82 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 02:08:34 +0000 Subject: [PATCH 52/58] feat(server): add t3 hub migrate and start tenant hubs without schema changes t3 hub migrate applies every pending hub migration with T3CODE_HUB_DATABASE_ADMIN_URL (the runtime role when absent), grants the runtime role access (failing if it cannot), starts no server, touches no tenant data, is idempotent, and needs no machines URL, gateway token or port; the hub host runs it once before activating tenants. A hub process without an admin URL now only verifies that the schema has every migration this build knows and fails with HubSchemaNotMigratedError naming the missing ones, never attempting DDL with the runtime role; with an admin URL (development, tests) it still migrates first. The e2e runs t3 hub migrate and gives tenant processes only the runtime URL. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/scripts/thread-machines-e2e.mjs | 24 +++- apps/server/src/cli/hub.test.ts | 103 +++++++++++++ apps/server/src/cli/hub.ts | 52 ++++++- .../persistence/Postgres/HubDatabase.test.ts | 54 +++++++ .../persistence/Postgres/HubDatabaseLive.ts | 136 ++++++++++++++---- .../src/persistence/Postgres/HubMigrator.ts | 17 +++ .../persistence/Postgres/hubTestDatabase.ts | 5 +- apps/server/src/server.hub.test.ts | 2 +- 8 files changed, 356 insertions(+), 37 deletions(-) create mode 100644 apps/server/src/cli/hub.test.ts diff --git a/apps/server/scripts/thread-machines-e2e.mjs b/apps/server/scripts/thread-machines-e2e.mjs index 6f3516c1287f..262b13c5e65e 100644 --- a/apps/server/scripts/thread-machines-e2e.mjs +++ b/apps/server/scripts/thread-machines-e2e.mjs @@ -25,9 +25,9 @@ * By default the hub persists to SQLite in its base directory. With * `--hub-database-url` it runs as production hubs do: on Postgres, for a fresh * tenant with a random secret key, seeded through `t3 hub import` from a - * standalone state directory, with a separate migration role when - * `--hub-database-admin-url` is given, and restarting on an empty base - * directory. Use a disposable database; the tenant's rows are left in it. + * standalone state directory after `t3 hub migrate` (with a separate + * migration role when `--hub-database-admin-url` is given; tenant processes + * never get it), and restarting on an empty base directory. Use a disposable database; the tenant's rows are left in it. * * It uses the real provider through T3's adapter on the runner, with this * machine's existing provider login, for two short turns (plus title @@ -165,11 +165,14 @@ async function startRunner() { log(`runner listening in ${Date.now() - started} ms`); } -/** Postgres hub settings: a fresh tenant and secret key per run (never printed). */ +/** + * Postgres hub settings: a fresh tenant and secret key per run (never printed). + * As on the hub host, tenant processes get only the runtime URL; the admin URL + * is used by `t3 hub migrate` alone. + */ const hubDatabaseEnv = HUB_DATABASE_URL ? { T3CODE_HUB_DATABASE_URL: HUB_DATABASE_URL, - ...(HUB_DATABASE_ADMIN_URL ? { T3CODE_HUB_DATABASE_ADMIN_URL: HUB_DATABASE_ADMIN_URL } : {}), T3CODE_HUB_TENANT_ID: `e2e-${NodeCrypto.randomUUID()}`, T3CODE_HUB_SECRET_KEY: NodeCrypto.randomBytes(32).toString("base64"), } @@ -195,6 +198,17 @@ const t3 = (argv, env) => * directory holding them (its state.sqlite is created by a standalone command). */ function seedHubDatabase() { + // The hub host migrates once with the migration role before any tenant starts. + const migrateDir = NodePath.join(root, "migrate"); + NodeFS.mkdirSync(migrateDir, { recursive: true }); + const migrated = t3(["hub", "migrate", "--base-dir", migrateDir], { + T3CODE_SERVER_MODE: "hub", + T3CODE_HUB_DATABASE_URL: HUB_DATABASE_URL, + ...(HUB_DATABASE_ADMIN_URL ? { T3CODE_HUB_DATABASE_ADMIN_URL: HUB_DATABASE_ADMIN_URL } : {}), + T3CODE_HUB_TENANT_ID: "00000000-0000-4000-8000-000000000000", + T3CODE_HUB_SECRET_KEY: hubDatabaseEnv.T3CODE_HUB_SECRET_KEY, + }); + check("t3 hub migrate", /hub migrations|nothing to apply/.test(migrated)); const seedDir = NodePath.join(root, "seed"); t3(["auth", "session", "list", "--base-dir", seedDir], {}); const report = t3(["hub", "import", NodePath.join(seedDir, "userdata")], hubDatabaseEnv); diff --git a/apps/server/src/cli/hub.test.ts b/apps/server/src/cli/hub.test.ts new file mode 100644 index 000000000000..67f0d3532e16 --- /dev/null +++ b/apps/server/src/cli/hub.test.ts @@ -0,0 +1,103 @@ +// @effect-diagnostics nodeBuiltinImport:off -- The test prepares a throwaway base directory like the hub host does. +/** + * `t3 hub migrate` as the hub host runs it: the exact allowlisted environment, + * no machines URL, no gateway token, no port. Runs only with + * T3_HUB_TEST_DATABASE_URL (see persistence/Postgres/hubTestDatabase.ts). + */ +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, it } from "@effect/vitest"; +import * as NetService from "@t3tools/shared/Net"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as TestConsole from "effect/testing/TestConsole"; +import { Command } from "effect/unstable/cli"; +import * as Reactivity from "effect/unstable/reactivity/Reactivity"; + +import { makeCli } from "../bin.ts"; +import { makeHubPool } from "../persistence/Postgres/HubClient.ts"; +import { hubMigrations } from "../persistence/Postgres/migrations/index.ts"; +import { + HUB_TEST_SECRET_KEY, + hubTestDatabaseUrl, + makeHubTestSchema, +} from "../persistence/Postgres/hubTestDatabase.ts"; + +const runMigrate = (env: Record, baseDir: string) => + Effect.gen(function* () { + yield* Command.runWith(makeCli(), { version: "0.0.0" })([ + "hub", + "migrate", + "--base-dir", + baseDir, + ]); + return (yield* TestConsole.logLines).filter((line): line is string => typeof line === "string"); + }).pipe( + Effect.provide( + Layer.mergeAll( + TestConsole.layer, + NodeServices.layer, + NetService.layer, + ConfigProvider.layer(ConfigProvider.fromEnv({ env })), + ), + ), + ); + +describe.skipIf(hubTestDatabaseUrl === undefined)("t3 hub migrate", () => { + it.effect("applies every hub migration once, grants the runtime role, and is idempotent", () => + Effect.gen(function* () { + const schema = yield* makeHubTestSchema(hubTestDatabaseUrl!); + const dir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-hub-migrate-")); + // The host's allowlist: no machines URL or token, no gateway token, no port. + const env: Record = { + PATH: process.env.PATH ?? "", + HOME: NodePath.join(dir, "home"), + LANG: "C.UTF-8", + NODE_ENV: "production", + T3CODE_HOME: dir, + T3CODE_SERVER_MODE: "hub", + T3CODE_HUB_DATABASE_URL: schema.runtimeUrl, + ...(schema.separateRuntimeRole ? { T3CODE_HUB_DATABASE_ADMIN_URL: schema.adminUrl } : {}), + T3CODE_HUB_TENANT_ID: "00000000-0000-4000-8000-000000000000", + T3CODE_HUB_SECRET_KEY: HUB_TEST_SECRET_KEY, + T3CODE_TELEMETRY_ENABLED: "false", + }; + + const first = (yield* runMigrate(env, dir)).join("\n"); + assert.include(first, `Applied ${hubMigrations.length} hub migrations`); + assert.include(first, "051_HubThreadMachineServices"); + assert.notInclude(first, schema.runtimeUrl); + assert.notInclude(first, schema.adminUrl); + assert.include((yield* runMigrate(env, dir)).join("\n"), "nothing to apply"); + + const admin = yield* makeHubPool({ url: schema.adminUrl, maxConnections: 1 }); + const applied = yield* admin<{ readonly id: number }>` + SELECT id FROM hub_schema_migrations ORDER BY id + `; + assert.deepStrictEqual( + applied.map((row) => Number(row.id)), + hubMigrations.map((entry) => entry.id), + ); + // It touched no tenant data. + const [documents] = yield* admin<{ readonly count: number }>` + SELECT count(*) AS count FROM hub_documents + `; + assert.strictEqual(Number(documents?.count), 0); + // The runtime role can use the tables it was granted. + const runtime = yield* makeHubPool({ url: schema.runtimeUrl, maxConnections: 1 }); + yield* runtime`SELECT count(*) FROM hub_thread_machine_status`; + + // An unreachable database fails the command. + const failed = yield* runMigrate( + { ...env, T3CODE_HUB_DATABASE_URL: "postgres://nobody@127.0.0.1:9/none" }, + dir, + ).pipe(Effect.exit); + assert.strictEqual(failed._tag, "Failure"); + NodeFS.rmSync(dir, { recursive: true, force: true }); + }).pipe(Effect.scoped, Effect.provide(Reactivity.layer)), + ); +}); diff --git a/apps/server/src/cli/hub.ts b/apps/server/src/cli/hub.ts index ebe0a873d8eb..7ef7532b031e 100644 --- a/apps/server/src/cli/hub.ts +++ b/apps/server/src/cli/hub.ts @@ -121,7 +121,57 @@ const importCommand = Command.make("import", { ), ); +const HubMigrateEnv = Config.all({ + databaseUrl: Config.redacted("T3CODE_HUB_DATABASE_URL"), + databaseAdminUrl: Config.redacted("T3CODE_HUB_DATABASE_ADMIN_URL").pipe(Config.option), +}); + +/** + * `t3 hub migrate`: the hub host runs this once before activating tenants, so + * tenant processes never need schema-change privileges. It applies every + * pending hub migration with `T3CODE_HUB_DATABASE_ADMIN_URL` (or the runtime + * role when that is absent), grants the runtime role access, starts no server, + * touches no tenant data, and is idempotent. Other `T3CODE_HUB_*` variables + * (tenant, secret key) are accepted and ignored. Prints no credentials. + */ +const migrateCommand = Command.make("migrate", { + baseDir: Flag.string("base-dir").pipe( + Flag.withDescription("Accepted for symmetry with other commands; nothing is written there."), + Flag.optional, + ), +}).pipe( + Command.withDescription( + "Apply the hub database migrations and grant the runtime role access. Reads T3CODE_HUB_DATABASE_URL and the optional T3CODE_HUB_DATABASE_ADMIN_URL; starts no server and touches no tenant data.", + ), + Command.withHandler(() => + Effect.gen(function* () { + const env = yield* HubMigrateEnv; + // Loaded on use so other `t3` commands never load the Postgres driver. + const { migrateHubDatabase } = yield* Effect.promise( + () => import("../persistence/Postgres/HubDatabaseLive.ts"), + ); + const report = yield* migrateHubDatabase({ + databaseUrl: Redacted.value(env.databaseUrl), + databaseAdminUrl: Option.getOrUndefined(Option.map(env.databaseAdminUrl, Redacted.value)), + }); + yield* Console.log( + [ + report.applied.length === 0 + ? "The hub database schema is current; nothing to apply." + : `Applied ${report.applied.length} hub migrations: ${report.applied + .map((entry) => `${String(entry.id).padStart(3, "0")}_${entry.name}`) + .join(", ")}.`, + report.grantedRole === null + ? "The runtime role migrated the schema itself; no grants needed." + : "Granted the runtime role access to the hub tables.", + "", + ].join("\n"), + ); + }), + ), +); + export const hubCommand = Command.make("hub").pipe( Command.withDescription("Manage thread-machine hub state."), - Command.withSubcommands([importCommand]), + Command.withSubcommands([importCommand, migrateCommand]), ); diff --git a/apps/server/src/persistence/Postgres/HubDatabase.test.ts b/apps/server/src/persistence/Postgres/HubDatabase.test.ts index 04db67f153b8..62cb53acd43f 100644 --- a/apps/server/src/persistence/Postgres/HubDatabase.test.ts +++ b/apps/server/src/persistence/Postgres/HubDatabase.test.ts @@ -10,6 +10,7 @@ import * as Reactivity from "effect/unstable/reactivity/Reactivity"; import * as SqlClient from "effect/unstable/sql/SqlClient"; import { HubDatabase } from "./HubDatabase.ts"; +import { makeHubDatabase, migrateHubDatabase } from "./HubDatabaseLive.ts"; import { makeHubPool, makeTenantSqlClient } from "./HubClient.ts"; import { orderHubMigrations, runHubMigrations, type HubMigrationEntry } from "./HubMigrator.ts"; import { hubMigrations } from "./migrations/index.ts"; @@ -98,6 +99,59 @@ describe.skipIf(hubTestDatabaseUrl === undefined)("hub database", () => { ), ); + it.effect( + "starts tenants without schema changes: verify first, `t3 hub migrate` applies and grants", + () => + withSchema((schema) => + Effect.gen(function* () { + const admin = yield* makeHubPool({ url: schema.adminUrl, maxConnections: 1 }); + const migrationsTable = admin<{ readonly exists: boolean }>` + SELECT to_regclass('hub_schema_migrations') IS NOT NULL AS exists + `.pipe(Effect.map((rows) => rows[0]?.exists ?? false)); + + // A tenant process (runtime role, no admin URL) refuses a schema + // that is behind, and changes nothing trying to fix it. + const behind = yield* makeHubDatabase({ + databaseUrl: schema.runtimeUrl, + tenantId: "tenant-verify", + }).pipe(Effect.scoped, Effect.flip); + assert.strictEqual(behind._tag, "HubSchemaNotMigratedError"); + assert.include(behind.message, "t3 hub migrate"); + assert.include(behind.message, "051_HubThreadMachineServices"); + assert.isFalse(yield* migrationsTable); + + const first = yield* migrateHubDatabase({ + databaseUrl: schema.runtimeUrl, + ...(schema.separateRuntimeRole ? { databaseAdminUrl: schema.adminUrl } : {}), + }); + assert.deepStrictEqual( + first.applied.map((entry) => entry.id), + hubMigrations.map((entry) => entry.id), + ); + assert.strictEqual(first.grantedRole !== null, schema.separateRuntimeRole); + const again = yield* migrateHubDatabase({ + databaseUrl: schema.runtimeUrl, + ...(schema.separateRuntimeRole ? { databaseAdminUrl: schema.adminUrl } : {}), + }); + assert.deepStrictEqual(again.applied, []); + + // Now the tenant process starts, and its runtime role can use the tables. + const rows = yield* Effect.scoped( + Effect.gen(function* () { + const tenant = yield* makeHubDatabase({ + databaseUrl: schema.runtimeUrl, + tenantId: "tenant-verify", + }); + return yield* tenant.sql<{ readonly count: number }>` + SELECT count(*) AS count FROM hub_documents + `; + }), + ); + assert.strictEqual(Number(rows[0]?.count), 0); + }), + ), + ); + it("rejects duplicate migration ids", () => { const migration = Effect.void; assert.throws(() => diff --git a/apps/server/src/persistence/Postgres/HubDatabaseLive.ts b/apps/server/src/persistence/Postgres/HubDatabaseLive.ts index 84fbfd73caf3..d4dbfd332b2e 100644 --- a/apps/server/src/persistence/Postgres/HubDatabaseLive.ts +++ b/apps/server/src/persistence/Postgres/HubDatabaseLive.ts @@ -1,11 +1,21 @@ /** - * Builds a hub's database: applies hub migrations (with the admin URL when one - * is configured), grants the runtime role access, and connects the - * tenant-scoped runtime client. Loaded only in hub mode. + * A hub's database. + * + * - `migrateHubDatabase` (`t3 hub migrate`): applies every pending hub + * migration with the admin URL (or the runtime URL when none is configured) + * and grants the runtime role access. It touches no tenant data. + * - `makeHubDatabase` connects the tenant-scoped runtime client. A tenant + * process normally has no admin URL: it then only verifies that the schema + * has every migration this build knows and fails clearly when it does not, + * never attempting DDL with the runtime role. Given an admin URL + * (development, tests) it migrates first, as `t3 hub migrate` would. + * + * Loaded only in hub mode. * * @module HubDatabaseLive */ import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; import * as Reactivity from "effect/unstable/reactivity/Reactivity"; import * as SqlClient from "effect/unstable/sql/SqlClient"; @@ -16,7 +26,7 @@ import { makeHubPool, makeTenantSqlClient, } from "./HubClient.ts"; -import { runHubMigrations } from "./HubMigrator.ts"; +import { type HubMigrationEntry, pendingHubMigrations, runHubMigrations } from "./HubMigrator.ts"; import { hubMigrations } from "./migrations/index.ts"; export interface HubDatabaseOptions { @@ -28,12 +38,18 @@ export interface HubDatabaseOptions { const quoteIdentifier = (identifier: string) => `"${identifier.replaceAll('"', '""')}"`; -/** - * Lets a separate runtime role use the tables the admin role owns. Idempotent; - * a failure is logged rather than fatal because a platform may manage grants - * itself, and a missing grant still fails loudly on first use. - */ -const grantRuntimeRole = (admin: SqlClient.SqlClient, runtimeRole: string) => +/** The hub schema lacks migrations this build needs; run `t3 hub migrate`. */ +export class HubSchemaNotMigratedError extends Schema.TaggedErrorClass()( + "HubSchemaNotMigratedError", + { missing: Schema.Array(Schema.String) }, +) { + override get message(): string { + return `The hub database schema is missing migrations ${this.missing.join(", ")}. Run \`t3 hub migrate\` with the migration role before starting hub processes.`; + } +} + +/** Grants the runtime role DML on the admin role's hub tables (see `grantRuntimeRole`). */ +const grantStatements = (admin: SqlClient.SqlClient, runtimeRole: string) => Effect.gen(function* () { const [row] = yield* admin<{ readonly adminRole: string; @@ -53,7 +69,16 @@ const grantRuntimeRole = (admin: SqlClient.SqlClient, runtimeRole: string) => yield* admin.unsafe( `REVOKE INSERT, UPDATE, DELETE ON ${schema}.hub_schema_migrations FROM ${role}`, ); - }).pipe( + }); + +/** + * Lets a separate runtime role use the tables the admin role owns. Idempotent; + * during a hub's own startup a failure is logged rather than fatal because a + * platform may manage grants itself, and a missing grant still fails loudly on + * first use. + */ +const grantRuntimeRole = (admin: SqlClient.SqlClient, runtimeRole: string) => + grantStatements(admin, runtimeRole).pipe( Effect.catchCause((cause) => Effect.logWarning("Could not grant the hub runtime role access to hub tables.", { cause, @@ -61,6 +86,52 @@ const grantRuntimeRole = (admin: SqlClient.SqlClient, runtimeRole: string) => ), ); +export interface HubMigrateOptions { + readonly databaseUrl: string; + /** The schema owner; the runtime role migrates when absent. */ + readonly databaseAdminUrl?: string | undefined; + /** For tests; defaults to every hub migration this build has. */ + readonly migrations?: ReadonlyArray; +} + +export interface HubMigrateReport { + readonly applied: ReadonlyArray>; + /** The runtime role that was granted access, when it differs from the admin role. */ + readonly grantedRole: string | null; +} + +/** + * `t3 hub migrate`: applies every pending hub migration and grants the runtime + * role access. Idempotent, tenant-independent, and strict: a failed grant + * fails the command, because tenant processes cannot migrate or grant. + */ +export const migrateHubDatabase = (options: HubMigrateOptions) => + Effect.gen(function* () { + const runtimePool = yield* makeHubPool({ + url: options.databaseUrl, + maxConnections: 1, + applicationName: "t3-hub-migrate", + }); + const [runtime] = yield* runtimePool<{ readonly role: string }>`SELECT current_user AS role`; + const admin = options.databaseAdminUrl + ? yield* makeHubPool({ + url: options.databaseAdminUrl, + maxConnections: 1, + applicationName: "t3-hub-migrate", + }) + : runtimePool; + const applied = yield* runHubMigrations(options.migrations ?? hubMigrations).pipe( + Effect.provideService(SqlClient.SqlClient, admin), + ); + const [adminRow] = yield* admin<{ readonly role: string }>`SELECT current_user AS role`; + const grantedRole = + options.databaseAdminUrl && runtime && adminRow && adminRow.role !== runtime.role + ? runtime.role + : null; + if (grantedRole !== null) yield* grantStatements(admin, grantedRole); + return { applied, grantedRole } satisfies HubMigrateReport; + }).pipe(Effect.scoped, Effect.provide(Reactivity.layer)); + export const makeHubDatabase = (options: HubDatabaseOptions) => Effect.gen(function* () { const tenantId = options.tenantId; @@ -71,23 +142,32 @@ export const makeHubDatabase = (options: HubDatabaseOptions) => const runtimePool = yield* makeHubPool({ url: options.databaseUrl }); const [runtime] = yield* runtimePool<{ readonly role: string }>`SELECT current_user AS role`; - yield* Effect.scoped( - Effect.gen(function* () { - const admin = options.databaseAdminUrl - ? yield* makeHubPool({ - url: options.databaseAdminUrl, - maxConnections: 1, - applicationName: "t3-hub-migrate", - }) - : runtimePool; - yield* runHubMigrations(hubMigrations).pipe( - Effect.provideService(SqlClient.SqlClient, admin), - ); - if (options.databaseAdminUrl && runtime) { - yield* grantRuntimeRole(admin, runtime.role); - } - }), - ); + if (options.databaseAdminUrl) { + // Development and tests: migrate here, as `t3 hub migrate` would. + yield* Effect.scoped( + Effect.gen(function* () { + const admin = yield* makeHubPool({ + url: options.databaseAdminUrl!, + maxConnections: 1, + applicationName: "t3-hub-migrate", + }); + yield* runHubMigrations(hubMigrations).pipe( + Effect.provideService(SqlClient.SqlClient, admin), + ); + if (runtime) yield* grantRuntimeRole(admin, runtime.role); + }), + ); + } else { + // A tenant process never changes the schema: it only checks it is current. + const pending = yield* pendingHubMigrations(hubMigrations).pipe( + Effect.provideService(SqlClient.SqlClient, runtimePool), + ); + if (pending.length > 0) { + return yield* new HubSchemaNotMigratedError({ + missing: pending.map((entry) => `${String(entry.id).padStart(3, "0")}_${entry.name}`), + }); + } + } const sql = yield* makeTenantSqlClient(runtimePool, tenantId); return { tenantId, sql } satisfies HubDatabaseShape; diff --git a/apps/server/src/persistence/Postgres/HubMigrator.ts b/apps/server/src/persistence/Postgres/HubMigrator.ts index 8770135d1165..d86ade24945d 100644 --- a/apps/server/src/persistence/Postgres/HubMigrator.ts +++ b/apps/server/src/persistence/Postgres/HubMigrator.ts @@ -82,3 +82,20 @@ export const runHubMigrations = (entries: ReadonlyArray) => yield* Effect.logInfo("Applied hub migration.", { id: next.id, name: next.name }); } }); + +/** + * The migrations `hub_schema_migrations` does not record yet (all of them + * when the table does not exist). Reads only; safe for the runtime role. + */ +export const pendingHubMigrations = (entries: ReadonlyArray) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const ordered = orderHubMigrations(entries); + const [table] = yield* sql<{ readonly exists: boolean }>` + SELECT to_regclass('hub_schema_migrations') IS NOT NULL AS exists + `; + if (!table?.exists) return ordered; + const rows = yield* sql<{ readonly id: number }>`SELECT id FROM hub_schema_migrations`; + const done = new Set(rows.map((row) => Number(row.id))); + return ordered.filter((entry) => !done.has(entry.id)); + }); diff --git a/apps/server/src/persistence/Postgres/hubTestDatabase.ts b/apps/server/src/persistence/Postgres/hubTestDatabase.ts index 8a4d2298e779..1649ace48637 100644 --- a/apps/server/src/persistence/Postgres/hubTestDatabase.ts +++ b/apps/server/src/persistence/Postgres/hubTestDatabase.ts @@ -118,7 +118,8 @@ export const hubTestDatabaseLayer = (schema: HubTestSchema, tenantId: string) => HubDatabase, makeHubDatabase({ databaseUrl: schema.runtimeUrl, - databaseAdminUrl: schema.separateRuntimeRole ? schema.adminUrl : undefined, + // Tests migrate at startup, as `t3 hub migrate` would before a tenant starts. + databaseAdminUrl: schema.adminUrl, tenantId, }), ); @@ -144,7 +145,7 @@ export const hubTestServerConfigLayer = ( serverMode: "hub" as const, hub: { databaseUrl: schema.runtimeUrl, - databaseAdminUrl: schema.separateRuntimeRole ? schema.adminUrl : undefined, + databaseAdminUrl: schema.adminUrl, tenantId, secretKey: options.secretKey ?? HUB_TEST_SECRET_KEY, }, diff --git a/apps/server/src/server.hub.test.ts b/apps/server/src/server.hub.test.ts index cbd1ecbb0ada..32d019b25a8e 100644 --- a/apps/server/src/server.hub.test.ts +++ b/apps/server/src/server.hub.test.ts @@ -313,8 +313,8 @@ describe.skipIf(hubTestDatabaseUrl === undefined)("hub server", () => { }); const env: Record = { T3CODE_SERVER_MODE: "hub", + // Like a tenant process: the runtime role only, never the migration role. T3CODE_HUB_DATABASE_URL: schema.runtimeUrl, - ...(schema.separateRuntimeRole ? { T3CODE_HUB_DATABASE_ADMIN_URL: schema.adminUrl } : {}), T3CODE_HUB_TENANT_ID: TENANT, T3CODE_HUB_SECRET_KEY: HUB_TEST_SECRET_KEY, T3CODE_HUB_MACHINES_URL: "http://127.0.0.1:9", From da84515381d00f50a8b5b0d62f66cd8ffb77fc17 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 02:11:41 +0000 Subject: [PATCH 53/58] feat(server): open the platform's sign-in intent before running a provider login Before the auth connector runs the login command on the sign-in machine, the hub now calls POST {T3CODE_HUB_MACHINES_URL}/provider-homes/{provider}/sign-in, which opens a 30-minute intent: only a login written after it is stored. A refused intent fails the session with a visible message and the login never runs. The docs describe the intent, t3 hub migrate with the host's exact environment, tenant startup that only verifies the schema, and that machines sync only each provider's login file, so user-level provider configuration comes from the hub's settings. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/hub/HubLayers.ts | 9 +-- apps/server/src/hub/HubProviderSignIn.test.ts | 34 ++++++++++- apps/server/src/hub/HubProviderSignIn.ts | 17 +++++- apps/server/src/hub/MachineDirectory.test.ts | 8 +++ apps/server/src/hub/MachineDirectory.ts | 38 ++++++++++++ docs/internals/thread-machines.md | 61 ++++++++++++++++--- packages/contracts/src/runner.ts | 13 ++++ 7 files changed, 161 insertions(+), 19 deletions(-) diff --git a/apps/server/src/hub/HubLayers.ts b/apps/server/src/hub/HubLayers.ts index 646f5ddd726f..29d99c237ae3 100644 --- a/apps/server/src/hub/HubLayers.ts +++ b/apps/server/src/hub/HubLayers.ts @@ -56,10 +56,11 @@ import * as ThreadMachineControls from "./ThreadMachineControls.ts"; import * as ThreadMachineStates from "./ThreadMachineStates.ts"; /** - * Hub thread-machine state (runner cursors, per-turn diffs, git status): the - * hub's tenant Postgres database when it has one (migration 050, applied with - * the other hub migrations), otherwise the server's SQLite database (tests and - * local development). + * Hub thread-machine state (runner cursors, per-turn diffs, git status, + * machine states, provider snapshots, MCP credentials): the hub's tenant + * Postgres database when it has one (migrations 050 and 051, applied by + * `t3 hub migrate` with the other hub migrations), otherwise the server's + * SQLite database (tests and local development). */ export const makeHubStateStoresLayer = ( persistence: Layer.Layer, diff --git a/apps/server/src/hub/HubProviderSignIn.test.ts b/apps/server/src/hub/HubProviderSignIn.test.ts index 8b404a62d655..565f1b3a08b3 100644 --- a/apps/server/src/hub/HubProviderSignIn.test.ts +++ b/apps/server/src/hub/HubProviderSignIn.test.ts @@ -68,10 +68,16 @@ const setup = ( options: { readonly storeTimeout?: Duration.Input; readonly succeedWithoutInput?: boolean; + readonly signInIntentFailure?: { readonly status: number; readonly code: string }; } = {}, ) => Effect.gen(function* () { const calls: Array = []; + // The platform's calls so far, read when the login command starts. + const platform: { calls: Effect.Effect> } = { + calls: Effect.succeed([]), + }; + const intentsAtStart: Array> = []; let submitted = options.succeedWithoutInput === true; const succeeded = { ...waiting, status: "succeeded" as const, stage: "complete" as const }; const runner = yield* serveRunner( @@ -88,8 +94,11 @@ const setup = ( "runner.provider.getCapabilities": () => Effect.succeed({ snapshot: authenticated, sessionModelSwitch: "in-session" as const }), "runner.auth.start": (input) => - Effect.sync(() => { + Effect.gen(function* () { calls.push(`start ${input.connector}`); + intentsAtStart.push( + (yield* platform.calls).filter((call) => call.startsWith("beginProviderSignIn")), + ); return waiting; }), "runner.auth.get": () => Effect.sync(() => (submitted ? succeeded : waiting)), @@ -108,7 +117,9 @@ const setup = ( ); const fake = yield* makeFakeMachineDirectory({ onWake: () => ({ state: "running", runnerUrl: runner.url }), + ...(options.signInIntentFailure ? { signInIntentFailure: options.signInIntentFailure } : {}), }); + platform.calls = fake.platformCalls; const snapshots = yield* HubProviderSnapshots.make; const context = yield* Layer.build( Layer.effect( @@ -133,13 +144,15 @@ const setup = ( const signIn = (yield* ProviderSignInControls.pipe(Effect.provide(context)))!; const waitFor = (id: string, predicate: (session: AuthConnectorSession) => boolean) => Effect.repeat(signIn.get(id), { until: predicate }).pipe(Effect.timeout("5 seconds")); - return { signIn, fake, calls, snapshots, waitFor }; + return { signIn, fake, calls, snapshots, waitFor, intentsAtStart }; }); describe("hub provider sign-in", () => { it.live("runs the login on the sign-in machine and completes once the sign-in is stored", () => Effect.gen(function* () { - const { signIn, fake, calls, snapshots, waitFor } = yield* setup(remoteSession({})); + const { signIn, fake, calls, snapshots, waitFor, intentsAtStart } = yield* setup( + remoteSession({}), + ); const started = yield* signIn.start({ connector: "claude", method: "account" }); expect(started).toMatchObject({ status: "starting", stage: "preparing", flow: "code" }); @@ -162,6 +175,8 @@ describe("hub provider sign-in", () => { const done = yield* waitFor(started.id, (session) => session.status !== "starting"); expect(done).toMatchObject({ status: "succeeded", stage: "complete", message: "Signed in." }); + // The sign-in intent was open before the login command ran. + expect(intentsAtStart).toEqual([["beginProviderSignIn claude"]]); expect(calls).toContain("configure claudeAgent"); expect(calls).toContain("start claude"); expect(calls).toContain("submit runner-session-1 callback"); @@ -217,4 +232,17 @@ describe("hub provider sign-in", () => { expect(github._tag).toBe("AuthConnectorError"); }).pipe(Effect.scoped, Effect.provide(StoresLive)), ); + + it.live("shows a sign-in the platform refused to start and never runs the login", () => + Effect.gen(function* () { + const { signIn, calls, fake, waitFor } = yield* setup(remoteSession({}), { + signInIntentFailure: { status: 409, code: "NOT_HUB_HOSTED" }, + }); + const started = yield* signIn.start({ connector: "claude", method: "account" }); + const failed = yield* waitFor(started.id, (session) => session.status === "failed"); + expect(failed.message).toContain("NOT_HUB_HOSTED"); + expect(calls.some((call) => call.startsWith("start "))).toBe(false); + expect((yield* fake.calls).map((call) => call.method).at(-1)).toBe("idle"); + }).pipe(Effect.scoped, Effect.provide(StoresLive)), + ); }); diff --git a/apps/server/src/hub/HubProviderSignIn.ts b/apps/server/src/hub/HubProviderSignIn.ts index 8703d62f0e16..cd085aa01b9b 100644 --- a/apps/server/src/hub/HubProviderSignIn.ts +++ b/apps/server/src/hub/HubProviderSignIn.ts @@ -8,8 +8,10 @@ * * 1. `start` answers at once with a session that is starting a machine; a * background fiber records the provider's stored sign-in version, ensures - * and wakes the sign-in machine, pushes the instance's settings, and - * starts the connector there (`runner.auth.start`). The session mirrors the + * and wakes the sign-in machine, pushes the instance's settings, opens the + * platform's sign-in intent (`POST provider-homes/{provider}/sign-in`; only + * a login written after it is stored, and a failure is shown on the + * session), and starts the connector there (`runner.auth.start`). The session mirrors the * runner's session (prompts, URLs, codes, fields) as it progresses; * `submit` and `cancel` are forwarded. Flows that finish in a browser on the * machine carry `workspaceBrowserUrl`. @@ -239,6 +241,17 @@ export const make = (options: HubProviderSignInOptions = {}) => instances: { [instanceId]: settings }, }).pipe(Effect.ignore); } + // Only a login written after the intent opens is stored. + yield* directory + .beginProviderSignIn(provider.home) + .pipe( + Effect.mapError((error) => + failure( + "start", + `Sign-in could not be started: ${error.code ?? error.detail}. Try again.`, + ), + ), + ); return yield* connection.client["runner.auth.start"](input); }), ); diff --git a/apps/server/src/hub/MachineDirectory.test.ts b/apps/server/src/hub/MachineDirectory.test.ts index 3c8431e561bd..1d0a27b7a914 100644 --- a/apps/server/src/hub/MachineDirectory.test.ts +++ b/apps/server/src/hub/MachineDirectory.test.ts @@ -154,6 +154,12 @@ describe("HTTP machine directory", () => { if (request.method === "DELETE") { return { status: 200, body: { provider: "claude", deleted: true } }; } + if (request.url.endsWith("/sign-in")) { + return { + status: 201, + body: { provider: "claude", epoch: 7, expiresAt: "2026-09-26T01:30:00.000Z" }, + }; + } return { status: 200, body: { providers: [{ provider: "claude", version: 3, updatedAt: "t" }] }, @@ -168,6 +174,7 @@ describe("HTTP machine directory", () => { .pipe(Effect.flip); expect(locked).toMatchObject({ status: 403, code: "REPOSITORY_ACCESS_REQUIRED" }); expect((yield* directory.providerHomes).providers[0]?.version).toBe(3); + expect((yield* directory.beginProviderSignIn("claude")).epoch).toBe(7); expect(yield* directory.deleteProviderHome("claude")).toEqual({ provider: "claude", deleted: true, @@ -177,6 +184,7 @@ describe("HTTP machine directory", () => { ["GET", "/api/repositories/refs?url=https%3A%2F%2Fgithub.com%2Facme%2Fapp"], ["GET", "/api/repositories/refs?url=https%3A%2F%2Fgithub.com%2Facme%2Flocked"], ["GET", "/api/provider-homes"], + ["POST", "/api/provider-homes/claude/sign-in"], ["DELETE", "/api/provider-homes/claude"], ]); }).pipe(Effect.provide(FetchHttpClient.layer)), diff --git a/apps/server/src/hub/MachineDirectory.ts b/apps/server/src/hub/MachineDirectory.ts index 9ea13e240d29..b216a6048c51 100644 --- a/apps/server/src/hub/MachineDirectory.ts +++ b/apps/server/src/hub/MachineDirectory.ts @@ -14,6 +14,7 @@ * * GET /repositories/refs?url= branches (GitHub App repositories) * GET /provider-homes stored provider sign-ins (metadata) + * POST /provider-homes/{provider}/sign-in open a sign-in intent (its epoch) * DELETE /provider-homes/{provider} sign a provider out everywhere * * Implementations: @@ -29,6 +30,7 @@ import { ProviderHomeDeleteResponse, type ProviderHomeId, ProviderHomesResponse, + ProviderSignInIntent, RepositoryRefsResponse, type ThreadMachineEnsureRequest, type ThreadMachineState, @@ -103,6 +105,13 @@ export interface MachineDirectoryShape { ) => Effect.Effect; /** Stored provider sign-ins (versions only). */ readonly providerHomes: Effect.Effect; + /** + * Opens a sign-in intent: only a login the sign-in machine writes after + * this is stored. Call it right before running the login command. + */ + readonly beginProviderSignIn: ( + provider: ProviderHomeId, + ) => Effect.Effect; /** Deletes a provider's stored sign-in; running machines sign out within seconds. */ readonly deleteProviderHome: ( provider: ProviderHomeId, @@ -225,6 +234,15 @@ export const makeHttpMachineDirectory = (options: { Effect.succeed(HttpClientRequest.get(`${base}/provider-homes`)), HttpClientResponse.schemaBodyJson(ProviderHomesResponse), ), + beginProviderSignIn: (provider) => + send( + "beginProviderSignIn", + "", + HttpClientRequest.post( + `${base}/provider-homes/${encodeURIComponent(provider)}/sign-in`, + ).pipe(HttpClientRequest.bodyJson({})), + HttpClientResponse.schemaBodyJson(ProviderSignInIntent), + ), deleteProviderHome: (provider) => send( "deleteProviderHome", @@ -260,6 +278,7 @@ export const makeStaticMachineDirectory = (options: { }), ), providerHomes: Effect.succeed({ providers: [] }), + beginProviderSignIn: (provider) => Effect.succeed({ provider, epoch: 0, expiresAt: null }), deleteProviderHome: (provider) => Effect.succeed({ provider, deleted: false }), }); @@ -304,12 +323,15 @@ export const makeFakeMachineDirectory = (options?: { readonly initial?: ReadonlyArray; readonly onWake?: (threadId: ThreadId, machine: FakeMachine | undefined) => FakeMachine; readonly repositories?: Readonly>; + /** Fails every sign-in intent with this platform error. */ + readonly signInIntentFailure?: { readonly status: number; readonly code: string }; }) => Effect.gen(function* () { const machines = yield* Ref.make(new Map(options?.initial ?? [])); const calls = yield* Ref.make>([]); const platformCalls = yield* Ref.make>([]); const providerHomes = yield* Ref.make(new Map()); + let signInEpoch = 0; const recordPlatform = (call: string) => Ref.update(platformCalls, (current) => [...current, call]); const record = (call: FakeMachineDirectoryCall) => @@ -388,6 +410,22 @@ export const makeFakeMachineDirectory = (options?: { })), })), ), + beginProviderSignIn: (provider) => + Effect.gen(function* () { + yield* recordPlatform(`beginProviderSignIn ${provider}`); + const failure = options?.signInIntentFailure; + if (failure !== undefined) { + return yield* new MachineDirectoryError({ + operation: "beginProviderSignIn", + threadId: "", + status: failure.status, + code: failure.code, + detail: `{"error":"${failure.code}"}`, + }); + } + signInEpoch += 1; + return { provider, epoch: signInEpoch, expiresAt: null }; + }), deleteProviderHome: (provider) => recordPlatform(`deleteProviderHome ${provider}`).pipe( Effect.andThen( diff --git a/docs/internals/thread-machines.md b/docs/internals/thread-machines.md index bec7222af560..3c5852cfbfe8 100644 --- a/docs/internals/thread-machines.md +++ b/docs/internals/thread-machines.md @@ -30,7 +30,7 @@ checkout path. A runner never persists orchestration state. | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `T3CODE_SERVER_MODE=hub` | Enables hub mode | | `T3CODE_HUB_DATABASE_URL` | Postgres URL of the hub schema, for a role without `BYPASSRLS`. libpq's `sslrootcert=system` is accepted. Without it the hub persists to SQLite (tests, development) | -| `T3CODE_HUB_DATABASE_ADMIN_URL` | Optional; the schema owner, used only to apply hub migrations and grant the runtime role. Defaults to the runtime URL | +| `T3CODE_HUB_DATABASE_ADMIN_URL` | Optional; the schema owner, used only by `t3 hub migrate` (see Operations). Tenant processes never get it; a hub given it (development, tests) migrates at startup | | `T3CODE_HUB_TENANT_ID` | The Aldo user ID whose rows this process owns; one hub process serves one user. Required with a database URL | | `T3CODE_HUB_SECRET_KEY` | Base64 32-byte key encrypting the tenant's secrets at rest. Required with a database URL | | `T3CODE_HUB_MACHINES_URL` | Base URL of the machine directory (served by the platform host process) | @@ -115,6 +115,7 @@ the control plane; see `docs/hub-host.md` in the remote-dev repository): ``` GET /repositories/refs?url= → { defaultBranch, refs: [{ name, sha }], truncated } GET /provider-homes → { providers: [{ provider, version, updatedAt }] } +POST /provider-homes/{provider}/sign-in → 201 { provider, epoch, expiresAt } open a sign-in intent DELETE /provider-homes/{provider} → { provider, deleted } ``` @@ -365,8 +366,12 @@ auth connector on the machine of the reserved thread `aldo-provider-sign-in` 1. `start` answers at once with `status: "starting"`, `stage: "preparing"`, message "Starting a machine for sign-in…" and the flow the method will use. In the background the hub reads the provider's stored sign-in version, ensures and wakes the sign-in - machine, pushes the instance's settings, and starts the connector there - (`runner.auth.start`). The session then mirrors the runner's (prompts, verification URLs, + machine, pushes the instance's settings, opens the platform's sign-in intent + (`POST {T3CODE_HUB_MACHINES_URL}/provider-homes/{provider}/sign-in` → + `{ provider, epoch, expiresAt }`, open for 30 minutes; only a login the sign-in machine + writes after it is stored, because the supervisor presents the epoch when it creates the + archive), and only then starts the connector there (`runner.auth.start`). A refused + intent fails the session with a visible message and never runs the login. The session then mirrors the runner's (prompts, verification URLs, codes, fields) under the hub's session id; `submit` and `cancel` are forwarded. Flows that finish in a browser on the machine (`flow: "browser"`) carry `workspaceBrowserUrl` (`T3CODE_HUB_THREAD_BROWSER_URL_TEMPLATE` with the sign-in thread). @@ -379,6 +384,14 @@ auth connector on the machine of the reserved thread `aldo-provider-sign-in` driver refuses agent sessions on that thread, and runners serve `runner.auth.*` only when bound to it. +Machines sync only each provider's login file (`claude` `.claude/.credentials.json`, +`codex` `.codex/auth.json`, `opencode` `.local/share/opencode/auth.json`, `grok` +`.grok/auth.json`, `cursor` `.config/cursor/auth.json`, `prime` `.prime/agent/auth.json`, +`muse` `.config/muse/auth.json`). Nothing else a CLI keeps in its home (settings, hooks, +skills, MCP configuration) reaches thread machines, so T3 never relies on it: user-level +provider configuration comes from the hub's provider instance settings, which the hub +pushes to each machine (`runner.provider.configure`). + Connectors map to provider homes as `claude`→`claude`, `codex`→`codex`, `opencode`→`opencode`, `grok`→`grok`, `cursor`→`cursor`, `prime-agent`→`prime`, `muse`→`muse`. Source-control connectors (GitHub, GitLab, Azure DevOps, Bitbucket) answer `AuthConnectorError` on a hub. @@ -457,12 +470,36 @@ runner, so a hub on a fresh base directory still sends them. **Migrations.** Hub migrations are numbered entries in `apps/server/src/persistence/Postgres/migrations/index.ts`: 001–049 for hub persistence, -050–099 for thread-machine state (050 and 051 today). A hub applies pending ones at startup with -`T3CODE_HUB_DATABASE_ADMIN_URL` (or the runtime URL), each in its own transaction under a -transaction-scoped advisory lock, recorded in `hub_schema_migrations`. They are applied by id, -not list position, so separately owned ranges can land in any order. Every new tenant table -leads its keys and indexes with `user_id` and enables `hubTenantPolicyStatements` (001). -Keep migrations expand/contract compatible: an older hub may still be running. +050–099 for thread-machine state (050 and 051 today). They are applied by +`t3 hub migrate`, each in its own transaction under a transaction-scoped advisory lock, +recorded in `hub_schema_migrations`, by id rather than list position, so separately owned +ranges can land in any order. Every new tenant table leads its keys and indexes with +`user_id` and enables `hubTenantPolicyStatements` (001). Keep migrations expand/contract +compatible: an older hub may still be running. + +The hub host runs the command once before activating any tenant (`docs/hub-host.md` in the +remote-dev repository): + +```bash +node /bin.mjs hub migrate --base-dir # cwd , a fresh temporary directory +``` + +with exactly `PATH`, `HOME=/home`, `LANG`, `NODE_ENV=production`, `T3CODE_HOME=`, +`T3CODE_SERVER_MODE=hub`, `T3CODE_HUB_DATABASE_URL` (runtime role), +`T3CODE_HUB_DATABASE_ADMIN_URL` (when configured; otherwise the runtime role migrates), +`T3CODE_HUB_TENANT_ID=00000000-0000-4000-8000-000000000000` (ignored), +`T3CODE_HUB_SECRET_KEY` (ignored) and `T3CODE_TELEMETRY_ENABLED=false`: no machines URL or +token, gateway token or port. It applies every pending migration, grants the runtime role +DML on the schema's tables (read-only on the history) when the admin role differs, starts +no server, touches no tenant data, prints no credentials, and is idempotent; exit 0 means +the schema is current and the grants are in place, anything else is a failure (the host +kills it after five minutes). + +Tenant hub processes start with the runtime URL only. They never attempt DDL: at startup +they check that `hub_schema_migrations` records every migration the build knows and +otherwise exit with `HubSchemaNotMigratedError` naming the missing ones ("Run `t3 hub +migrate` …"). A hub given an admin URL (development, tests, the loopback harness) still +migrates at startup, as the command would. **Roles and row-level security.** Every tenant table has a forced policy comparing `user_id` with the transaction-local `hub.user_id`. The runtime client sets it with `SET LOCAL` at the @@ -548,6 +585,9 @@ Not built yet: - Standalone behavior: `src/environment/ServerEnvironment.test.ts` (capabilities) and `src/server.test.ts` (hub-only RPCs answer `unsupported` / `AuthConnectorError`), plus the rest of the server suite. +- `t3 hub migrate` with the host's exact environment (`src/cli/hub.test.ts`) and tenant + startup that only verifies the schema (`persistence/Postgres/HubDatabase.test.ts`), both + Postgres-gated. - Postgres: set `T3_HUB_TEST_DATABASE_URL` to a disposable database the tests may create schemas in (for example a private loopback cluster as `postgres`). Each test gets its own schema and, when the connecting role bypasses RLS on loopback, a runtime role without @@ -622,7 +662,8 @@ including API keys, stay hub settings; the hub delivers them to machines. `submitAuthConnector` / `cancelAuthConnector` flow works unchanged on a hub for provider connectors (not source control). The first session answers `status: "starting"`, `stage: "preparing"`, message "Starting a machine for sign-in…" while the sign-in machine -starts, then mirrors the provider's prompt; after the CLI finishes it stays `verifying` +starts (a refused sign-in intent fails it with a visible message), then mirrors the +provider's prompt; after the CLI finishes it stays `verifying` ("Saving your sign-in…") until the platform stored it, then `succeeded`. Sessions with `flow: "browser"` carry `AuthConnectorSession.workspaceBrowserUrl` (the machine's browser page, same origin, frameable). `server.listProviderSignIns({})` → diff --git a/packages/contracts/src/runner.ts b/packages/contracts/src/runner.ts index 7640b1ba7752..796a8ff24692 100644 --- a/packages/contracts/src/runner.ts +++ b/packages/contracts/src/runner.ts @@ -283,6 +283,19 @@ export const ProviderHomesResponse = Schema.Struct({ }); export type ProviderHomesResponse = typeof ProviderHomesResponse.Type; +/** + * `POST {machines}/provider-homes/{provider}/sign-in` — opens a 30-minute + * sign-in intent. Only a login the sign-in machine writes after it is stored + * (the machine presents the epoch when it creates the archive); starting + * again supersedes it and sign-out closes it. + */ +export const ProviderSignInIntent = Schema.Struct({ + provider: Schema.String, + epoch: Schema.Number, + expiresAt: Schema.optional(Schema.NullOr(Schema.String)), +}); +export type ProviderSignInIntent = typeof ProviderSignInIntent.Type; + /** `DELETE {machines}/provider-homes/{provider}` — sign out everywhere. */ export const ProviderHomeDeleteResponse = Schema.Struct({ provider: Schema.String, From 1e50e9fd30a740a05cb074d26cb1dfbda0a705a3 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 02:30:58 +0000 Subject: [PATCH 54/58] feat(web): wake and pause thread machines and manage hub sign-ins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that the hub answers threadMachines.wake/pause, the client offers them instead of hiding explicit wake: "Wake machine" on the asleep composer notice, the files panel, working-tree and branch diffs while the machine sleeps, and the thread menu; "Pause machine" in the thread menu of a running machine. A busy machine warns "Machine is busy"; other failures show an error. Provider sign-in reads the session's decoded workspaceBrowserUrl (the reserved-thread fallback is gone) and shows the hub's staged progress ("Starting a machine for sign-in…", "Saving your sign-in…"). Settings → Providers marks providers signed in on machines and offers a confirmed sign-out. All of it is gated on the threadMachines capability. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/web/src/components/ChatView.tsx | 6 +- apps/web/src/components/DiffPanel.tsx | 12 +- apps/web/src/components/Sidebar.tsx | 17 ++- .../src/components/ThreadMachineStatus.tsx | 42 ++++++- .../components/hub/HubProviderSignIns.test.ts | 35 ++++++ .../src/components/hub/HubProviderSignIns.tsx | 119 ++++++++++++++++++ .../components/hub/hubAuthConnector.test.ts | 73 ++++++++--- .../src/components/hub/hubAuthConnector.ts | 36 +++--- .../hub/threadMachineActions.test.ts | 67 ++++++++++ .../components/hub/threadMachineActions.ts | 118 ++++++++++++++--- .../src/components/hub/useHubDiffDefaults.ts | 12 +- .../src/components/hub/useHubThreadContext.ts | 31 +++-- .../settings/AuthConnectorDialog.tsx | 20 +-- .../settings/ProviderSettingsPanel.tsx | 45 +++++-- .../components/threadActionMenu.logic.test.ts | 11 ++ .../src/components/threadActionMenu.logic.ts | 9 ++ apps/web/src/contextMenuFallback.ts | 12 ++ apps/web/src/hooks/useThreadActionMenu.ts | 20 ++- apps/web/src/landingDemo/runtime.ts | 7 ++ docs/internals/thread-machines.md | 26 ++-- .../client-runtime/src/state/sourceControl.ts | 14 +++ .../src/state/threadCommands.ts | 13 ++ 22 files changed, 644 insertions(+), 101 deletions(-) create mode 100644 apps/web/src/components/hub/HubProviderSignIns.test.ts create mode 100644 apps/web/src/components/hub/HubProviderSignIns.tsx create mode 100644 apps/web/src/components/hub/threadMachineActions.test.ts diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 7b5e77bc6767..102cd63d770b 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -7323,7 +7323,11 @@ function ChatViewContent(props: ChatViewProps) { ) : (activeRightPanelSurface?.kind === "files" || activeRightPanelSurface?.kind === "file") && hubThread.machineUnavailable && hubThread.machineView ? ( - + ) : (activeRightPanelSurface?.kind === "files" || activeRightPanelSurface?.kind === "file") && activeProject && activeWorkspaceRoot ? ( diff --git a/apps/web/src/components/DiffPanel.tsx b/apps/web/src/components/DiffPanel.tsx index 461455b29cad..3c86a68a7a2c 100644 --- a/apps/web/src/components/DiffPanel.tsx +++ b/apps/web/src/components/DiffPanel.tsx @@ -77,6 +77,7 @@ import { vcsEnvironment } from "../state/vcs"; import { buildBaseRefChoices, filterBaseRefChoices } from "../lib/baseRefChoices"; import { createGitDiffFileContentsLoader } from "../lib/diffFileContents"; import { useDiffPanelTurnDefaultWhileMachineAsleep } from "./hub/useHubDiffDefaults"; +import { ThreadMachineAsleepPanel } from "./ThreadMachineStatus"; type DiffThemeType = "light" | "dark"; const AUTOMATIC_BASE_REF = "__automatic_base_ref__"; @@ -201,7 +202,10 @@ export default function DiffPanel({ selectedTurn && (selectedTurn.checkpointTurnCount ?? inferredCheckpointTurnCountByTurnId[selectedTurn.turnId]); const latestTurn = orderedTurnDiffSummaries[0]; - useDiffPanelTurnDefaultWhileMachineAsleep(routeThreadRef, latestTurn?.turnId ?? null); + const unavailableMachine = useDiffPanelTurnDefaultWhileMachineAsleep( + routeThreadRef, + latestTurn?.turnId ?? null, + ); const selectedScopeLabel = selectedTurnId === null ? selectedGitScope === "unstaged" @@ -829,6 +833,12 @@ export default function DiffPanel({
Turn diffs are unavailable because this project is not a git repository.
+ ) : selectedTurnId === null && unavailableMachine ? ( + ) : selectedTurnId !== null && orderedTurnDiffSummaries.length === 0 ? (
No completed turns yet. diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx index de2a4deecab0..b3e99d681225 100644 --- a/apps/web/src/components/Sidebar.tsx +++ b/apps/web/src/components/Sidebar.tsx @@ -97,6 +97,8 @@ import { } from "../sidebarProjectGrouping"; import { legacyProjectCwdPreferenceKey, useUiStateStore } from "../uiStateStore"; import { readIsHubEnvironment, usePrimaryIsHub } from "../hubMode"; +import { deriveThreadMachineView } from "../threadMachine"; +import { threadMachineControlFor, useThreadMachineControls } from "./hub/threadMachineActions"; import { useThreadSelectionStore } from "../threadSelectionStore"; import { useThreadActions } from "../hooks/useThreadActions"; import { useHandleNewThread } from "../hooks/useHandleNewThread"; @@ -1755,6 +1757,7 @@ export default function Sidebar() { archiveThread, deleteThread, } = useThreadActions(); + const { wake: wakeMachine, pause: pauseMachine } = useThreadMachineControls(); const updateThreadMetadata = useAtomCommand(threadEnvironment.updateMetadata, { reportFailure: false, }); @@ -3105,6 +3108,7 @@ export default function Sidebar() { const isSettled = settledThreadKeysRef.current.has(threadKey); const isSnoozed = snoozedThreadKeysRef.current.has(threadKey); const isPinned = thread.pinnedAt != null; + const hub = readIsHubEnvironment(thread.environmentId); // Presets resolve at menu-open time (same as the popover). const snoozePresets = resolveSnoozePresets(new Date(), timestampFormat); const clicked = await settlePromise(() => @@ -3123,8 +3127,11 @@ export default function Sidebar() { snooze: supportsSnooze, pinning: supportsPinning, titleRegeneration: supportsTitleRegeneration, - workspacePath: !readIsHubEnvironment(thread.environmentId), + workspacePath: !hub, }, + machineControl: hub + ? threadMachineControlFor(deriveThreadMachineView(thread.machine)) + : null, snoozePresets, }), position, @@ -3233,6 +3240,12 @@ export default function Sidebar() { case "copy-thread-id": copyThreadIdToClipboard(thread.id, { threadId: thread.id }); return; + case "wake-machine": + await wakeMachine(threadRef); + return; + case "pause-machine": + await pauseMachine(threadRef); + return; case "archive": { if (confirmThreadArchive) { const confirmed = await settlePromise(() => @@ -3310,11 +3323,13 @@ export default function Sidebar() { handleMultiSelectContextMenu, markThreadUnread, openProjectSettings, + pauseMachine, projectCwdByKey, serverConfigs, startThreadRename, updateThreadMetadata, timestampFormat, + wakeMachine, ], ); diff --git a/apps/web/src/components/ThreadMachineStatus.tsx b/apps/web/src/components/ThreadMachineStatus.tsx index 7d6b1ea8cc58..a9755be0d99e 100644 --- a/apps/web/src/components/ThreadMachineStatus.tsx +++ b/apps/web/src/components/ThreadMachineStatus.tsx @@ -1,4 +1,4 @@ -import type { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import type { EnvironmentId, ScopedThreadRef, ThreadId } from "@t3tools/contracts"; import { scopeThreadRef } from "@t3tools/client-runtime/environment"; import { CloudIcon, MoonIcon, TriangleAlertIcon } from "lucide-react"; import { useMemo } from "react"; @@ -11,6 +11,7 @@ import { type ThreadMachineView, } from "../threadMachine"; import type { ComposerBannerStackItem } from "./chat/ComposerBannerStack"; +import { useThreadMachineControls } from "./hub/threadMachineActions"; import { StatusDot } from "./ManagedDevPcStatus"; import { Button } from "./ui/button"; import { Tooltip, TooltipPopup, TooltipTrigger } from "./ui/tooltip"; @@ -65,6 +66,8 @@ export function buildThreadMachineBannerItem(input: { readonly view: ThreadMachineView | null; readonly onRetry: (() => void) | null; readonly retrying: boolean; + readonly onWake: (() => void) | null; + readonly waking: boolean; readonly onDismiss: () => void; }): ComposerBannerStackItem | null { const { view } = input; @@ -76,6 +79,15 @@ export function buildThreadMachineBannerItem(input: { priority: "notice", icon: , title: view.description, + ...(input.onWake + ? { + actions: ( + + ), + } + : {}), dismissLabel: "Dismiss machine status", onDismiss: input.onDismiss, }; @@ -101,8 +113,19 @@ export function buildThreadMachineBannerItem(input: { return null; } -/** Stands in for machine-backed panels (files) while the machine sleeps. */ -export function ThreadMachineAsleepPanel({ view }: { readonly view: ThreadMachineView }) { +/** + * Stands in for machine-backed panels (files, working-tree diffs) while the + * machine sleeps or failed, with an explicit wake. + */ +export function ThreadMachineAsleepPanel(props: { + readonly threadRef: ScopedThreadRef | null; + readonly view: ThreadMachineView; + /** What lives on the machine, as the start of a sentence ("Files"). */ + readonly subject: string; +}) { + const { view } = props; + const { wake, pending } = useThreadMachineControls(); + const threadRef = props.threadRef; return (
@@ -112,8 +135,19 @@ export function ThreadMachineAsleepPanel({ view }: { readonly view: ThreadMachin

{view.phase === "failed" ? (view.detail ?? "This thread's machine could not start.") - : "Files live on this thread's machine. Send a message to wake it."} + : `${props.subject} live on this thread's machine. Wake it, or send a message.`}

+ {threadRef ? ( + + ) : null}
); } diff --git a/apps/web/src/components/hub/HubProviderSignIns.test.ts b/apps/web/src/components/hub/HubProviderSignIns.test.ts new file mode 100644 index 000000000000..d9d28de7d6b7 --- /dev/null +++ b/apps/web/src/components/hub/HubProviderSignIns.test.ts @@ -0,0 +1,35 @@ +import type { ProviderSignInList } from "@t3tools/contracts"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { describeProviderSignIn, providerSignInFor } from "./HubProviderSignIns"; + +const list: ProviderSignInList = { + signIns: [ + { connector: "claude", version: 3, updatedAt: "2026-09-26T09:00:00.000Z" }, + { connector: "codex", version: 1, updatedAt: null }, + ], +}; + +describe("providerSignInFor", () => { + it("finds the stored sign-in for a provider's connector", () => { + expect(providerSignInFor(list, "claude")?.version).toBe(3); + expect(providerSignInFor(list, "cursor")).toBeNull(); + expect(providerSignInFor(null, "claude")).toBeNull(); + }); +}); + +describe("describeProviderSignIn", () => { + it("says when the sign-in was saved when the platform knows", () => { + vi.useFakeTimers({ now: new Date("2026-09-26T11:00:00.000Z") }); + try { + expect(describeProviderSignIn("Claude", list.signIns[0]!)).toBe( + "Your Claude sign-in is saved (2h ago). Every thread machine starts signed in.", + ); + expect(describeProviderSignIn("Codex", list.signIns[1]!)).toBe( + "Your Codex sign-in is saved. Every thread machine starts signed in.", + ); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/apps/web/src/components/hub/HubProviderSignIns.tsx b/apps/web/src/components/hub/HubProviderSignIns.tsx new file mode 100644 index 000000000000..3e5610490d69 --- /dev/null +++ b/apps/web/src/components/hub/HubProviderSignIns.tsx @@ -0,0 +1,119 @@ +import type { + AuthConnectorKind, + EnvironmentId, + ProviderSignIn, + ProviderSignInList, +} from "@t3tools/contracts"; +import { + isAtomCommandInterrupted, + squashAtomCommandFailure, +} from "@t3tools/client-runtime/state/runtime"; +import { LogOutIcon } from "lucide-react"; +import { useState } from "react"; + +import { readLocalApi } from "../../localApi"; +import { useEnvironmentQuery } from "../../state/query"; +import { sourceControlEnvironment } from "../../state/sourceControl"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { formatRelativeTimeLabel } from "../../timestampFormat"; +import { Badge } from "../ui/badge"; +import { Button } from "../ui/button"; +import { toastManager } from "../ui/toast"; +import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; + +/** + * Hub mode: the provider sign-ins stored for thread machines. Pass `null` off + * a hub; standalone servers do not answer the call. + */ +export function useHubProviderSignIns(environmentId: EnvironmentId | null) { + return useEnvironmentQuery( + environmentId ? sourceControlEnvironment.providerSignIns({ environmentId, input: {} }) : null, + ); +} + +export function providerSignInFor( + list: ProviderSignInList | null, + connector: AuthConnectorKind, +): ProviderSignIn | null { + return list?.signIns.find((signIn) => signIn.connector === connector) ?? null; +} + +/** Tooltip for a stored sign-in. Stored is not proof of a valid login; the provider's status is. */ +export function describeProviderSignIn(serviceName: string, signIn: ProviderSignIn): string { + const saved = signIn.updatedAt ? formatRelativeTimeLabel(signIn.updatedAt) : ""; + return [ + `Your ${serviceName} sign-in is saved${saved ? ` (${saved})` : ""}.`, + "Every thread machine starts signed in.", + ].join(" "); +} + +/** + * Shows that a provider is signed in on thread machines, with a confirmed + * sign-out. Renders nothing when no sign-in is stored. + */ +export function HubProviderSignInStatus(props: { + readonly environmentId: EnvironmentId; + readonly connector: AuthConnectorKind; + readonly serviceName: string; + readonly signIn: ProviderSignIn | null; + readonly onSignedOut: () => void; +}) { + const { environmentId, connector, serviceName, signIn, onSignedOut } = props; + const signOut = useAtomCommand(sourceControlEnvironment.signOutProvider, { + reportFailure: false, + }); + const [signingOut, setSigningOut] = useState(false); + if (signIn === null) return null; + + const confirmAndSignOut = async () => { + const api = readLocalApi(); + const message = [ + `Sign out of ${serviceName} on thread machines?`, + "Running machines drop the sign-in within seconds; sign in again to use it.", + ].join("\n"); + const confirmed = api + ? await api.dialogs.confirm(message, { variant: "destructive" }).catch(() => false) + : window.confirm(message); + if (!confirmed) return; + setSigningOut(true); + const result = await signOut({ environmentId, input: { connector } }); + setSigningOut(false); + if (result._tag === "Success") { + toastManager.add({ type: "success", title: `Signed out of ${serviceName}` }); + onSignedOut(); + return; + } + if (!isAtomCommandInterrupted(result)) { + const error = squashAtomCommandFailure(result); + toastManager.add({ + type: "error", + title: `Could not sign out of ${serviceName}`, + description: error instanceof Error ? error.message : "An error occurred.", + }); + } + }; + + return ( + + + }> + Signed in on machines + + + {describeProviderSignIn(serviceName, signIn)} + + + + + ); +} diff --git a/apps/web/src/components/hub/hubAuthConnector.test.ts b/apps/web/src/components/hub/hubAuthConnector.test.ts index c1871962c61a..9bf50c430ae9 100644 --- a/apps/web/src/components/hub/hubAuthConnector.test.ts +++ b/apps/web/src/components/hub/hubAuthConnector.test.ts @@ -1,3 +1,4 @@ +import type { AuthConnectorSession } from "@t3tools/contracts"; import { afterEach, describe, expect, it, vi } from "vite-plus/test"; afterEach(() => { @@ -16,28 +17,42 @@ async function load(bootstrap: Record) { return import("./hubAuthConnector"); } +const HUB_BOOTSTRAP = { + serverMode: "hub", + threadBrowserUrlTemplate: "/_devpc/threads/{threadId}/browser", +}; + +function session(patch: Partial = {}): AuthConnectorSession { + return { + id: "session", + connector: "cursor", + method: "account", + status: "starting", + flow: "browser", + stage: "preparing", + message: "Starting a machine for sign-in…", + verificationUrl: null, + userCode: null, + fields: [], + expiresAt: null, + ...patch, + }; +} + describe("resolveAuthWorkspaceBrowserUrl", () => { - it("prefers the browser the sign-in session names", async () => { - const { resolveAuthWorkspaceBrowserUrl } = await load({ - serverMode: "hub", - threadBrowserUrlTemplate: "/_devpc/threads/{threadId}/browser", - }); + it("opens the browser the hub's sign-in session names", async () => { + const { resolveAuthWorkspaceBrowserUrl } = await load(HUB_BOOTSTRAP); expect( - resolveAuthWorkspaceBrowserUrl({ id: "session", workspaceBrowserUrl: "/_aldo/sign-in" }), - ).toBe("/_aldo/sign-in"); + resolveAuthWorkspaceBrowserUrl( + session({ workspaceBrowserUrl: "/_devpc/threads/aldo-provider-sign-in/browser" }), + ), + ).toBe("/_devpc/threads/aldo-provider-sign-in/browser"); }); - it("opens the hub's sign-in machine browser when the session names none", async () => { - const { resolveAuthWorkspaceBrowserUrl } = await load({ - serverMode: "hub", - threadBrowserUrlTemplate: "/_devpc/threads/{threadId}/browser", - }); - expect(resolveAuthWorkspaceBrowserUrl({ id: "session" })).toBe( - "/_devpc/threads/aldo-provider-sign-in/browser", - ); - expect(resolveAuthWorkspaceBrowserUrl(null)).toBe( - "/_devpc/threads/aldo-provider-sign-in/browser", - ); + it("has no browser on a hub until the session names one", async () => { + const { resolveAuthWorkspaceBrowserUrl } = await load(HUB_BOOTSTRAP); + expect(resolveAuthWorkspaceBrowserUrl(session({ workspaceBrowserUrl: null }))).toBeNull(); + expect(resolveAuthWorkspaceBrowserUrl(null)).toBeNull(); }); it("keeps the shared workspace browser on a persistent workspace", async () => { @@ -45,6 +60,26 @@ describe("resolveAuthWorkspaceBrowserUrl", () => { previewUrlTemplate: "https://{port}.preview.example.test/", previewUrls: { "6080": "https://6080.preview.example.test/vnc.html" }, }); - expect(resolveAuthWorkspaceBrowserUrl(null)).toBe("https://6080.preview.example.test/vnc.html"); + expect(resolveAuthWorkspaceBrowserUrl(session())).toBe( + "https://6080.preview.example.test/vnc.html", + ); + }); +}); + +describe("hubAuthConnectorProgress", () => { + it("shows the hub's staged messages while it starts the machine and saves the sign-in", async () => { + const { hubAuthConnectorProgress } = await load(HUB_BOOTSTRAP); + expect(hubAuthConnectorProgress(session())).toBe("Starting a machine for sign-in…"); + expect( + hubAuthConnectorProgress(session({ stage: "verifying", message: "Saving your sign-in…" })), + ).toBe("Saving your sign-in…"); + }); + + it("stays quiet once the provider asks the user for something", async () => { + const { hubAuthConnectorProgress } = await load(HUB_BOOTSTRAP); + expect( + hubAuthConnectorProgress(session({ status: "waiting", stage: "authorize", message: "Go" })), + ).toBeNull(); + expect(hubAuthConnectorProgress(null)).toBeNull(); }); }); diff --git a/apps/web/src/components/hub/hubAuthConnector.ts b/apps/web/src/components/hub/hubAuthConnector.ts index c2fa7eb08ef1..8b0e19da4519 100644 --- a/apps/web/src/components/hub/hubAuthConnector.ts +++ b/apps/web/src/components/hub/hubAuthConnector.ts @@ -1,27 +1,25 @@ +import type { AuthConnectorSession } from "@t3tools/contracts"; + import { managedWorkspaceBrowserUrl } from "../../managedDevPc"; /** - * The machine a hub starts for provider sign-in has its own browser, reached - * like a thread's under this reserved id. + * Where a sign-in that needs the workspace browser should open it: the page + * the session names (a hub's sign-in machine), else the persistent + * workspace's browser. A hub names it once the provider's flow runs in a + * browser on its sign-in machine; until then there is none. */ -export const HUB_SIGN_IN_BROWSER_THREAD_ID = "aldo-provider-sign-in"; +export function resolveAuthWorkspaceBrowserUrl( + session: AuthConnectorSession | null, +): string | null { + return session?.workspaceBrowserUrl ?? managedWorkspaceBrowserUrl(); +} /** - * Where a sign-in that needs the workspace browser should open it: the URL - * the session names (a hub's sign-in machine), else the deployment's browser. - * - * TODO(thread-machines): read `session.workspaceBrowserUrl` directly once the - * `AuthConnectorSession` contract carries it; until then decoding drops it - * and the reserved sign-in id stands in. + * A hub reports its own progress while it starts the sign-in machine and + * while it saves the finished sign-in; show that message as the status line. */ -export function resolveAuthWorkspaceBrowserUrl(session: object | null): string | null { - if ( - session !== null && - "workspaceBrowserUrl" in session && - typeof session.workspaceBrowserUrl === "string" && - session.workspaceBrowserUrl.length > 0 - ) { - return session.workspaceBrowserUrl; - } - return managedWorkspaceBrowserUrl(HUB_SIGN_IN_BROWSER_THREAD_ID); +export function hubAuthConnectorProgress( + session: Pick | null, +): string | null { + return session?.stage === "preparing" || session?.stage === "verifying" ? session.message : null; } diff --git a/apps/web/src/components/hub/threadMachineActions.test.ts b/apps/web/src/components/hub/threadMachineActions.test.ts new file mode 100644 index 000000000000..71caef76b51e --- /dev/null +++ b/apps/web/src/components/hub/threadMachineActions.test.ts @@ -0,0 +1,67 @@ +import { ThreadId, ThreadMachineControlError } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { deriveThreadMachineView } from "../../threadMachine"; +import { + describeThreadMachineControlFailure, + threadMachineControlFor, +} from "./threadMachineActions"; + +const view = (state: "none" | "starting" | "running" | "paused" | "saved" | "failed") => + deriveThreadMachineView({ state, detail: null, updatedAt: "2026-09-26T10:00:00.000Z" }); + +describe("threadMachineControlFor", () => { + it("wakes sleeping and failed machines and pauses running ones", () => { + expect(threadMachineControlFor(view("paused"))).toBe("wake"); + expect(threadMachineControlFor(view("saved"))).toBe("wake"); + expect(threadMachineControlFor(view("none"))).toBe("wake"); + expect(threadMachineControlFor(view("failed"))).toBe("wake"); + expect(threadMachineControlFor(view("running"))).toBe("pause"); + }); + + it("offers nothing while the machine is in motion or unknown", () => { + expect(threadMachineControlFor(view("starting"))).toBeNull(); + expect(threadMachineControlFor(null)).toBeNull(); + }); +}); + +describe("describeThreadMachineControlFailure", () => { + const error = (reason: ThreadMachineControlError["reason"], detail: string) => + new ThreadMachineControlError({ operation: "threadMachines.pause", reason, detail }); + + it("reports a busy machine as a warning with the server's reason", () => { + expect( + describeThreadMachineControlFailure("pause", error("busy", "A turn is running.")), + ).toEqual({ type: "warning", title: "Machine is busy", description: "A turn is running." }); + }); + + it("reports an unavailable machine as an error", () => { + expect( + describeThreadMachineControlFailure("wake", error("unavailable", "The directory failed.")), + ).toEqual({ + type: "error", + title: "Could not wake machine", + description: "The directory failed.", + }); + expect( + describeThreadMachineControlFailure( + "pause", + new ThreadMachineControlError({ + operation: "threadMachines.pause", + reason: "not-found", + detail: `Thread ${ThreadId.make("t-1")} does not exist or is archived.`, + }), + ), + ).toMatchObject({ type: "error", title: "Could not pause machine" }); + }); + + it("falls back to a generic message for transport failures", () => { + expect(describeThreadMachineControlFailure("wake", new Error("Socket closed"))).toMatchObject({ + type: "error", + description: "Socket closed", + }); + expect(describeThreadMachineControlFailure("wake", "boom")).toMatchObject({ + description: "An error occurred.", + }); + }); +}); diff --git a/apps/web/src/components/hub/threadMachineActions.ts b/apps/web/src/components/hub/threadMachineActions.ts index 8b3fd6239407..90c6ef9a7b2b 100644 --- a/apps/web/src/components/hub/threadMachineActions.ts +++ b/apps/web/src/components/hub/threadMachineActions.ts @@ -1,20 +1,108 @@ -import type { ScopedThreadRef } from "@t3tools/contracts"; +import { type ScopedThreadRef, ThreadMachineControlError } from "@t3tools/contracts"; +import { + isAtomCommandInterrupted, + squashAtomCommandFailure, +} from "@t3tools/client-runtime/state/runtime"; +import * as Schema from "effect/Schema"; +import { useCallback, useState } from "react"; + +import { threadEnvironment } from "../../state/threads"; +import { useAtomCommand } from "../../state/use-atom-command"; +import type { ThreadMachineView } from "../../threadMachine"; +import { stackedThreadToast, toastManager } from "../ui/toast"; + +/** + * Explicit machine lifecycle requests for a hub thread + * (`threadMachines.wake` / `threadMachines.pause`). Turns, terminals, file + * writes and git actions still wake a machine on their own. + */ +export type ThreadMachineControl = "wake" | "pause"; + +/** The control a hub thread offers: wake a sleeping or failed machine, pause a running one. */ +export function threadMachineControlFor( + view: ThreadMachineView | null | undefined, +): ThreadMachineControl | null { + switch (view?.phase) { + case "asleep": + case "failed": + return "wake"; + case "running": + return "pause"; + default: + return null; + } +} + +const isThreadMachineControlError = Schema.is(ThreadMachineControlError); + +export interface ThreadMachineControlNotice { + readonly type: "warning" | "error"; + readonly title: string; + readonly description: string; +} + +/** What to tell the user when a control fails. */ +export function describeThreadMachineControlFailure( + control: ThreadMachineControl, + error: unknown, +): ThreadMachineControlNotice { + if (isThreadMachineControlError(error) && error.reason === "busy") { + return { type: "warning", title: "Machine is busy", description: error.detail }; + } + return { + type: "error", + title: control === "wake" ? "Could not wake machine" : "Could not pause machine", + description: + error instanceof Error && error.message.trim().length > 0 + ? error.message + : "An error occurred.", + }; +} /** - * Explicit machine lifecycle requests for a hub thread. - * - * The hub's `threadMachines.wake` / `threadMachines.pause` RPCs are not in - * the client contracts yet, so explicit wake is reported as unsupported and - * every caller hides its affordance. A turn (send, retry) and a terminal - * still wake the machine on their own. - * - * TODO(thread-machines): call the `threadMachines.wake({ threadId })` RPC - * here once it is part of `WsRpcGroup`, and flip the flag. + * Wake and pause for hub threads, reporting failures as toasts. `pending` + * names the control in flight from this caller, for disabling its button. */ -export const THREAD_MACHINE_WAKE_SUPPORTED = false; +export function useThreadMachineControls() { + const wakeMachine = useAtomCommand(threadEnvironment.wakeMachine, { reportFailure: false }); + const pauseMachine = useAtomCommand(threadEnvironment.pauseMachine, { reportFailure: false }); + const [pending, setPending] = useState(null); + + const run = useCallback( + async (control: ThreadMachineControl, threadRef: ScopedThreadRef): Promise => { + setPending(control); + const target = { + environmentId: threadRef.environmentId, + input: { threadId: threadRef.threadId }, + }; + const result = await (control === "wake" ? wakeMachine(target) : pauseMachine(target)); + setPending(null); + if (result._tag === "Success") { + if (control === "pause") { + toastManager.add( + stackedThreadToast({ + type: "info", + title: "Machine released", + description: "It pauses shortly. Sending a message wakes it again.", + timeout: 5_000, + }), + ); + } + return true; + } + if (!isAtomCommandInterrupted(result)) { + toastManager.add( + stackedThreadToast( + describeThreadMachineControlFailure(control, squashAtomCommandFailure(result)), + ), + ); + } + return false; + }, + [pauseMachine, wakeMachine], + ); -export async function requestThreadMachineWake( - _threadRef: ScopedThreadRef, -): Promise<"requested" | "unsupported"> { - return "unsupported"; + const wake = useCallback((threadRef: ScopedThreadRef) => run("wake", threadRef), [run]); + const pause = useCallback((threadRef: ScopedThreadRef) => run("pause", threadRef), [run]); + return { wake, pause, pending }; } diff --git a/apps/web/src/components/hub/useHubDiffDefaults.ts b/apps/web/src/components/hub/useHubDiffDefaults.ts index 5d1bdba01253..c5a2cc7b5486 100644 --- a/apps/web/src/components/hub/useHubDiffDefaults.ts +++ b/apps/web/src/components/hub/useHubDiffDefaults.ts @@ -5,22 +5,23 @@ import { useEffect } from "react"; import { useDiffPanelStore } from "../../diffPanelStore"; import { useIsHubEnvironment } from "../../hubMode"; import { useThreadShell } from "../../state/entities"; -import { deriveThreadMachineView } from "../../threadMachine"; +import { deriveThreadMachineView, type ThreadMachineView } from "../../threadMachine"; /** * A sleeping hub machine cannot answer working-tree or branch diffs, but the * hub keeps every captured turn diff. Until the user picks a scope, open the * diff panel on the latest turn instead of a view that would wake nothing and - * show an error. + * show an error. Returns the machine while it cannot serve reads (asleep or + * failed), `null` otherwise and off a hub. */ export function useDiffPanelTurnDefaultWhileMachineAsleep( threadRef: ScopedThreadRef | null | undefined, latestTurnId: TurnId | null, -): void { +): ThreadMachineView | null { const hub = useIsHubEnvironment(threadRef?.environmentId); const machine = useThreadShell(hub && threadRef ? threadRef : null)?.machine; - const phase = deriveThreadMachineView(machine)?.phase; - const machineUnavailable = phase === "asleep" || phase === "failed"; + const view = hub ? deriveThreadMachineView(machine) : null; + const machineUnavailable = view?.phase === "asleep" || view?.phase === "failed"; const hasStoredSelection = useDiffPanelStore((state) => threadRef ? state.byThreadKey[scopedThreadKey(threadRef)] !== undefined : true, ); @@ -28,4 +29,5 @@ export function useDiffPanelTurnDefaultWhileMachineAsleep( if (!threadRef || !machineUnavailable || hasStoredSelection || latestTurnId === null) return; useDiffPanelStore.getState().selectTurn(threadRef, latestTurnId); }, [hasStoredSelection, latestTurnId, machineUnavailable, threadRef]); + return machineUnavailable ? view : null; } diff --git a/apps/web/src/components/hub/useHubThreadContext.ts b/apps/web/src/components/hub/useHubThreadContext.ts index 98e00790c3b7..f67bce02105b 100644 --- a/apps/web/src/components/hub/useHubThreadContext.ts +++ b/apps/web/src/components/hub/useHubThreadContext.ts @@ -19,7 +19,7 @@ import { deriveThreadMachineView, type ThreadMachineView } from "../../threadMac import type { ComposerBannerStackItem } from "../chat/ComposerBannerStack"; import { buildThreadMachineBannerItem } from "../ThreadMachineStatus"; import { stackedThreadToast, toastManager } from "../ui/toast"; -import { requestThreadMachineWake, THREAD_MACHINE_WAKE_SUPPORTED } from "./threadMachineActions"; +import { useThreadMachineControls } from "./threadMachineActions"; export interface HubThreadContext { /** The thread's environment is a hub: every thread runs on its own machine. */ @@ -102,13 +102,15 @@ export function useHubThreadContext(input: HubThreadContextInput): HubThreadCont }); const retryTurn = useAtomCommand(threadEnvironment.retryTurn, { reportFailure: false }); + const machineControls = useThreadMachineControls(); const [retrying, setRetrying] = useState(false); const [, setDismissTick] = useState(0); const failedTurnId = input.latestTurn?.state === "error" ? input.latestTurn.turnId : null; - const canRetry = - base.machineView?.phase === "failed" && - threadRef !== null && - (failedTurnId !== null || THREAD_MACHINE_WAKE_SUPPORTED); + const canRetry = base.machineView?.phase === "failed" && threadRef !== null; + const { wake } = machineControls; + const onWake = useCallback(() => { + if (threadRef) void wake(threadRef); + }, [threadRef, wake]); const onRetry = useCallback(() => { if (!threadRef) return; setRetrying(true); @@ -130,11 +132,11 @@ export function useHubThreadContext(input: HubThreadContextInput): HubThreadCont ); } } else { - await requestThreadMachineWake(threadRef); + await wake(threadRef); } setRetrying(false); })(); - }, [failedTurnId, retryTurn, threadRef]); + }, [failedTurnId, retryTurn, threadRef, wake]); const threadKey = threadRef ? scopedThreadKey(threadRef) : null; const bannerView = base.machineView; @@ -148,12 +150,25 @@ export function useHubThreadContext(input: HubThreadContextInput): HubThreadCont view: bannerView, onRetry: canRetry ? onRetry : null, retrying, + onWake: threadRef ? onWake : null, + waking: machineControls.pending === "wake", onDismiss: () => { dismissedMachineBanners.add(bannerId); setDismissTick((tick) => tick + 1); }, }); - }, [bannerDismissed, bannerId, bannerView, canRetry, onRetry, retrying, threadKey]); + }, [ + bannerDismissed, + bannerId, + bannerView, + canRetry, + machineControls.pending, + onRetry, + onWake, + retrying, + threadKey, + threadRef, + ]); const machineWakes = base.machineView !== null && base.machineView.phase !== "running"; const announceMachineWake = useCallback(() => { diff --git a/apps/web/src/components/settings/AuthConnectorDialog.tsx b/apps/web/src/components/settings/AuthConnectorDialog.tsx index f24e060f9beb..cd244200dad9 100644 --- a/apps/web/src/components/settings/AuthConnectorDialog.tsx +++ b/apps/web/src/components/settings/AuthConnectorDialog.tsx @@ -42,7 +42,7 @@ import { Textarea } from "../ui/textarea"; import { toastManager } from "../ui/toast"; import { APP_BASE_NAME } from "../../branding"; import { useIsHubEnvironment } from "../../hubMode"; -import { resolveAuthWorkspaceBrowserUrl } from "../hub/hubAuthConnector"; +import { hubAuthConnectorProgress, resolveAuthWorkspaceBrowserUrl } from "../hub/hubAuthConnector"; export type AuthConnectorMethodOption = { readonly method: AuthConnectorMethod; @@ -168,6 +168,7 @@ export function AuthConnectorDialog(props: { const selectedMethod = methods.find((method) => method.method === session?.method) ?? null; const hub = useIsHubEnvironment(environmentId); const workspaceBrowserUrl = resolveAuthWorkspaceBrowserUrl(session); + const hubProgress = hub ? hubAuthConnectorProgress(session) : null; useEffect(() => { if (!open || !sessionEnvironmentId || !session) return; @@ -383,9 +384,7 @@ export function AuthConnectorDialog(props: { ? `Return to ${APP_BASE_NAME}` : presentationStage === "verifying" ? "Checking your account" - : hub && selectedMethod?.workspaceBrowser === true - ? "Starting a machine for sign-in…" - : "Preparing secure sign-in"; + : "Preparing secure sign-in"; const stageDescription = presentationStage === "authorize" ? (authorizeInstruction ?? @@ -395,7 +394,9 @@ export function AuthConnectorDialog(props: { `Finish approving access in ${browserName}, then paste what it gives you below.`) : session?.stage === "verifying" ? "Keep this window open while the provider confirms your account." - : session?.message; + : hubProgress + ? undefined + : session?.message; return ( @@ -678,15 +679,16 @@ export function AuthConnectorDialog(props: { ) : null} {(session.stage === "preparing" || session.stage === "verifying") && - !session.verificationUrl ? ( + (hubProgress || !session.verificationUrl) ? (
- {session.stage === "verifying" - ? "Confirming your account…" - : "Preparing the provider’s secure sign-in…"} + {hubProgress ?? + (session.stage === "verifying" + ? "Confirming your account…" + : "Preparing the provider’s secure sign-in…")}
) : null}
diff --git a/apps/web/src/components/settings/ProviderSettingsPanel.tsx b/apps/web/src/components/settings/ProviderSettingsPanel.tsx index de9e6934a5f2..c6a6fd282a9f 100644 --- a/apps/web/src/components/settings/ProviderSettingsPanel.tsx +++ b/apps/web/src/components/settings/ProviderSettingsPanel.tsx @@ -76,6 +76,12 @@ import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; import { stackedThreadToast, toastManager } from "../ui/toast"; import { AddProviderInstanceDialog } from "./AddProviderInstanceDialog"; import { AuthConnectorDialog } from "./AuthConnectorDialog"; +import { + HubProviderSignInStatus, + providerSignInFor, + useHubProviderSignIns, +} from "../hub/HubProviderSignIns"; +import { useIsHubEnvironment } from "../../hubMode"; import { resolveAgentAuthMethods } from "./authConnectorMethods"; import { ProviderInstanceCard } from "./ProviderInstanceCard"; import { @@ -458,6 +464,8 @@ export function EnvironmentProviderSettings({ const serverProviders = useAtomValue(serverEnvironment.providersValueAtom(environmentId)) ?? EMPTY_SERVER_PROVIDERS; const serverConfig = useAtomValue(serverEnvironment.configValueAtom(environmentId)); + const hub = useIsHubEnvironment(environmentId); + const providerSignIns = useHubProviderSignIns(hub ? environmentId : null); const refreshServerProviders = useAtomCommand(serverEnvironment.refreshProviders, { reportFailure: false, }); @@ -537,6 +545,14 @@ export function EnvironmentProviderSettings({ })(); }, [environmentId, refreshServerProviders]); + // Hub mode: a sign-in or sign-out changes the stored sign-ins and every + // machine's provider status. + const refreshProviderSignIns = providerSignIns.refresh; + const refreshSignIns = useCallback(() => { + refreshProviderSignIns(); + refreshProviders(); + }, [refreshProviderSignIns, refreshProviders]); + const runProviderUpdate = useCallback( async (candidate: ProviderUpdateCandidate) => { // Ref-based re-entry guard, mirroring refreshProviders: a state updater @@ -840,15 +856,26 @@ export function EnvironmentProviderSettings({ } authConnectorAction={ mode === "editor" && authConnector && !readOnly ? ( - + <> + {hub ? ( + + ) : null} + + ) : undefined } hiddenModels={modelPreferences.hiddenModels} diff --git a/apps/web/src/components/threadActionMenu.logic.test.ts b/apps/web/src/components/threadActionMenu.logic.test.ts index ea23ed0a0070..fd6a5447d593 100644 --- a/apps/web/src/components/threadActionMenu.logic.test.ts +++ b/apps/web/src/components/threadActionMenu.logic.test.ts @@ -46,6 +46,17 @@ describe("buildThreadActionMenuItems", () => { expect(hubIds).toContain("copy-thread-id"); }); + it("offers the hub thread's machine control after the lifecycle actions", () => { + expect(ids(baseState)).not.toContain("wake-machine"); + expect(ids(baseState)).not.toContain("pause-machine"); + const wake = ids({ ...baseState, machineControl: "wake" }); + expect(wake).not.toContain("pause-machine"); + expect(wake.indexOf("wake-machine")).toBe(wake.indexOf("rename") - 1); + const pause = ids({ ...baseState, machineControl: "pause" }); + expect(pause).not.toContain("wake-machine"); + expect(pause.indexOf("pause-machine")).toBe(pause.indexOf("rename") - 1); + }); + it("groups project settings with utility actions before archive", () => { const items = buildThreadActionMenuItems(baseState); const copyIndex = items.findIndex((item) => item.id === "copy"); diff --git a/apps/web/src/components/threadActionMenu.logic.ts b/apps/web/src/components/threadActionMenu.logic.ts index d54f54f03a46..a3762c8da4e1 100644 --- a/apps/web/src/components/threadActionMenu.logic.ts +++ b/apps/web/src/components/threadActionMenu.logic.ts @@ -23,6 +23,8 @@ export type ThreadActionMenuId = | "copy-path" | "copy-branch" | "copy-thread-id" + | "wake-machine" + | "pause-machine" | "archive" | "delete"; @@ -43,6 +45,8 @@ export interface ThreadActionMenuState { /** Absent = true. Hub threads live on their own machines: no local path to copy. */ readonly workspacePath?: boolean; }; + /** Hub threads: wake a sleeping or failed machine, or pause a running one. */ + readonly machineControl?: "wake" | "pause" | null; readonly snoozePresets: ReadonlyArray; } @@ -97,6 +101,11 @@ export function buildThreadActionMenuItems( }, ] : []), + ...(state.machineControl === "wake" + ? [{ id: "wake-machine" as const, label: "Wake machine", icon: "power" }] + : state.machineControl === "pause" + ? [{ id: "pause-machine" as const, label: "Pause machine", icon: "moon" }] + : []), { id: "rename", label: "Rename thread", icon: "pencil", separatorBefore: true }, ...(state.supports.titleRegeneration ? [ diff --git a/apps/web/src/contextMenuFallback.ts b/apps/web/src/contextMenuFallback.ts index 2c43641b2b56..72ae0ee060a0 100644 --- a/apps/web/src/contextMenuFallback.ts +++ b/apps/web/src/contextMenuFallback.ts @@ -70,6 +70,14 @@ const ICON_PATHS: Record s.markThreadUnread); const confirmThreadDelete = useClientSettings((s) => s.confirmThreadDelete); const confirmThreadArchive = useClientSettings((s) => s.confirmThreadArchive); @@ -130,12 +136,13 @@ export function useThreadActionMenu(input: { const thread = readThreadShell(threadRef); if (!thread) return; const now = new Date(); + const hub = readIsHubEnvironment(threadRef.environmentId); const supports = { settlement: readEnvironmentSupportsSettlement(threadRef.environmentId), snooze: readEnvironmentSupportsSnooze(threadRef.environmentId), pinning: readEnvironmentSupportsPinning(threadRef.environmentId), titleRegeneration: readEnvironmentSupportsTitleRegeneration(threadRef.environmentId), - workspacePath: !readIsHubEnvironment(threadRef.environmentId), + workspacePath: !hub, }; const isRegeneratingTitle = thread.titleRegeneration != null; const snoozePresets = resolveSnoozePresets(now, timestampFormat); @@ -148,6 +155,9 @@ export function useThreadActionMenu(input: { isRegeneratingTitle, isRunning: thread.session?.status === "running" && thread.session.activeTurnId != null, supports, + machineControl: hub + ? threadMachineControlFor(deriveThreadMachineView(thread.machine)) + : null, snoozePresets, }); const clicked = await settlePromise(() => api.contextMenu.show(items, position)); @@ -278,6 +288,12 @@ export function useThreadActionMenu(input: { case "copy-thread-id": copyThreadIdToClipboard(thread.id, { threadId: thread.id }); return; + case "wake-machine": + await wakeMachine(threadRef); + return; + case "pause-machine": + await pauseMachine(threadRef); + return; case "archive": { if (confirmThreadArchive) { const confirmed = await settlePromise(() => @@ -343,6 +359,7 @@ export function useThreadActionMenu(input: { logicalProjectKeyByPhysicalKey, markThreadUnread, onStartRename, + pauseMachine, pinThread, projectCwd, projectGroupingSettings, @@ -355,6 +372,7 @@ export function useThreadActionMenu(input: { unsettleThread, unsnoozeThread, updateThreadMetadata, + wakeMachine, ], ); diff --git a/apps/web/src/landingDemo/runtime.ts b/apps/web/src/landingDemo/runtime.ts index 653e23b69e9c..e07d2dafdd2c 100644 --- a/apps/web/src/landingDemo/runtime.ts +++ b/apps/web/src/landingDemo/runtime.ts @@ -566,6 +566,13 @@ const client = new Proxy( ? { [WS_METHODS.serverDiscoverSourceControl]: () => Effect.succeed(hubDemoDiscovery), [WS_METHODS.sourceControlListRepositories]: () => Effect.succeed(hubDemoRepositories), + [WS_METHODS.serverListProviderSignIns]: () => + Effect.succeed({ + signIns: [{ connector: "codex", version: 1, updatedAt: STARTED_AT }], + }), + [WS_METHODS.threadMachinesWake]: () => + Effect.succeed({ state: "starting", detail: null, updatedAt: STARTED_AT }), + [WS_METHODS.threadMachinesPause]: () => Effect.succeed(null), } : {}), [ORCHESTRATION_WS_METHODS.getFullThreadDiff]: () => Effect.succeed({ patch: "", files: [] }), diff --git a/docs/internals/thread-machines.md b/docs/internals/thread-machines.md index 213e9c23df50..a8487f8613de 100644 --- a/docs/internals/thread-machines.md +++ b/docs/internals/thread-machines.md @@ -557,18 +557,26 @@ descriptor, so a hub never flashes local-checkout controls. On a hub: projects: `project.create` with `repositoryIdentity` and the virtual root, deduplicated by remote. Nothing is cloned by the client. - The thread shell's `machine` drives a header status pill, the timeline's waking row - (with the platform's detail), the composer notice (asleep, or failed with a retry that - re-drives the failed turn), and the sidebar (starting reads as working, failed as failed). - `thread-machine.*` activities stay out of the work log unless they are errors. + (with the platform's detail), the composer notice (asleep with "Wake machine", or failed + with a retry that re-drives the failed turn, or wakes the machine when no turn failed), and + the sidebar (starting reads as working, failed as failed). `thread-machine.*` activities + stay out of the work log unless they are errors. +- Explicit controls (`components/hub/threadMachineActions.ts`) call `threadMachines.wake` and + `threadMachines.pause`: "Wake machine" in the composer notice, the files panel, working-tree + and branch diffs while the machine sleeps, and the thread menu (sidebar and header) of a + sleeping or failed machine; "Pause machine" in the thread menu of a running one. `busy` + shows a "Machine is busy" warning, every other failure an error toast. - Surfaces that need a checkout adapt: no Open In, Copy path, t3.json reads or new-thread workspace settings; files show "Machine asleep" while the machine sleeps; the diff panel opens on the latest captured turn; previews and the machine browser use the bootstrap's - per-thread templates (`threadPreviewUrlTemplate`, `threadBrowserUrlTemplate`); provider - sign-in opens the session's `workspaceBrowserUrl` when present. The managed workspace - status, `/settings/workspace` and the durable wake queue are skipped. - -An explicit wake (`threadMachines.wake`) is not wired yet: the client hides that affordance -(`components/hub/threadMachineActions.ts`). + per-thread templates (`threadPreviewUrlTemplate`, `threadBrowserUrlTemplate`). The managed + workspace status, `/settings/workspace` and the durable wake queue are skipped. +- Provider sign-in opens the session's `workspaceBrowserUrl` (a hub names none until the + flow runs in the sign-in machine's browser) and shows the hub's staged messages + ("Starting a machine for sign-in…", "Saving your sign-in…") as the dialog's status line. + Settings → Providers marks providers whose sign-in is stored for machines + (`server.listProviderSignIns`) and offers a confirmed "Sign out" + (`server.signOutProvider`). ## Gaps diff --git a/packages/client-runtime/src/state/sourceControl.ts b/packages/client-runtime/src/state/sourceControl.ts index f425d123ac76..f783dda3b7d9 100644 --- a/packages/client-runtime/src/state/sourceControl.ts +++ b/packages/client-runtime/src/state/sourceControl.ts @@ -53,6 +53,20 @@ export function createSourceControlEnvironmentAtoms( mode: "parallel", }, }), + // Hub mode only: provider sign-ins stored for thread machines. + providerSignIns: createEnvironmentRpcQueryAtomFamily(runtime, { + label: "environment-data:server:provider-sign-ins", + tag: WS_METHODS.serverListProviderSignIns, + }), + signOutProvider: createEnvironmentRpcCommand(runtime, { + label: "environment-data:server:sign-out-provider", + tag: WS_METHODS.serverSignOutProvider, + scheduler: commandScheduler, + concurrency: { + mode: "serial", + key: ({ environmentId }) => environmentId, + }, + }), repository: createEnvironmentRpcQueryAtomFamily(runtime, { label: "environment-data:source-control:repository", tag: WS_METHODS.sourceControlLookupRepository, diff --git a/packages/client-runtime/src/state/threadCommands.ts b/packages/client-runtime/src/state/threadCommands.ts index d4179170ceac..be0cdbfc9bb0 100644 --- a/packages/client-runtime/src/state/threadCommands.ts +++ b/packages/client-runtime/src/state/threadCommands.ts @@ -219,5 +219,18 @@ export function createThreadEnvironmentAtoms( scheduler, concurrency, }), + // Hub mode only (the `threadMachines` capability): explicit machine controls. + wakeMachine: createEnvironmentRpcCommand(runtime, { + label: "environment-data:commands:thread:wake-machine", + tag: WS_METHODS.threadMachinesWake, + scheduler, + concurrency, + }), + pauseMachine: createEnvironmentRpcCommand(runtime, { + label: "environment-data:commands:thread:pause-machine", + tag: WS_METHODS.threadMachinesPause, + scheduler, + concurrency, + }), }; } From 64e54f6395c5b063067e3e54cf95e3907d4a77e5 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 02:38:07 +0000 Subject: [PATCH 55/58] test(server): provide the settings-push test's layers in one call tsgo's effect plugin reports chained Effect.provide calls as a warning, which fails the server typecheck. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/src/hub/HubProviderSnapshots.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/server/src/hub/HubProviderSnapshots.test.ts b/apps/server/src/hub/HubProviderSnapshots.test.ts index 000d4020746d..c3cb3f7763b8 100644 --- a/apps/server/src/hub/HubProviderSnapshots.test.ts +++ b/apps/server/src/hub/HubProviderSnapshots.test.ts @@ -260,8 +260,13 @@ describe("hub provider snapshots", () => { Effect.provideService(RunnerConnectionPool, pool), Effect.provideService(RunnerEventDelivery.RunnerEventDelivery, delivery), Effect.provideService(RemoteSessionRegistry.RemoteSessionRegistry, registry), - Effect.provide(HubProviderSnapshots.layer), - Effect.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-push-" })), + Effect.provide( + HubProviderSnapshots.layer.pipe( + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-hub-push-" }), + ), + ), + ), ); // Connecting pushes the thread's instance, then reads its status. From 65877f9dcb1c0a53f23dd472f385c081ca6278b5 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 02:38:10 +0000 Subject: [PATCH 56/58] test(server): run the thread-machines e2e against a built package --dist runs the runner, the hub and every t3 command from an extracted release artifact (or a bin.mjs), so the artifact that ships is the one verified. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/server/scripts/thread-machines-e2e.mjs | 16 ++++++++++------ docs/internals/thread-machines.md | 5 +++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/apps/server/scripts/thread-machines-e2e.mjs b/apps/server/scripts/thread-machines-e2e.mjs index 262b13c5e65e..60ac6236bb37 100644 --- a/apps/server/scripts/thread-machines-e2e.mjs +++ b/apps/server/scripts/thread-machines-e2e.mjs @@ -35,8 +35,12 @@ * instance. Nothing outside the temporary directory (and the hub database) * is written; processes started here are stopped by PID on exit. * + * With `--dist ` every process runs from a built server package instead + * of the source: an extracted release artifact (`/dist/bin.mjs`) or the + * `bin.mjs` itself. + * * node apps/server/scripts/thread-machines-e2e.mjs [--keep] [--model ] - * [--hub-database-url [--hub-database-admin-url ]] + * [--hub-database-url [--hub-database-admin-url ]] [--dist ] */ import * as NodeChildProcess from "node:child_process"; import * as NodeCrypto from "node:crypto"; @@ -48,16 +52,16 @@ import * as NodeURL from "node:url"; const HUB_PORT = 4421; const RUNNER_PORT = 4422; const HUB = `http://127.0.0.1:${HUB_PORT}`; -const BIN = NodePath.resolve( - NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), - "../src/bin.ts", -); const args = process.argv.slice(2); const flagValue = (name, fallback) => { const index = args.indexOf(name); return index >= 0 ? args[index + 1] : fallback; }; +const DIST = flagValue("--dist", undefined); +const BIN = DIST + ? NodePath.resolve(DIST.endsWith(".mjs") ? DIST : NodePath.join(DIST, "dist", "bin.mjs")) + : NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), "../src/bin.ts"); const KEEP = args.includes("--keep"); const MODEL = { instanceId: "claudeAgent", model: flagValue("--model", "claude-haiku-4-5") }; const HUB_DATABASE_URL = flagValue("--hub-database-url", undefined); @@ -369,7 +373,7 @@ async function main() { NodeFS.writeFileSync(NodePath.join(checkout, "notes.txt"), "hello from the runner\n"); git("add", "."); git("commit", "-m", "initial"); - log(`workspace ${root}; hub persistence: ${HUB_DATABASE_URL ? "Postgres" : "SQLite"}`); + log(`workspace ${root}; hub persistence: ${HUB_DATABASE_URL ? "Postgres" : "SQLite"}; ${BIN}`); if (HUB_DATABASE_URL) seedHubDatabase(); await startRunner(); diff --git a/docs/internals/thread-machines.md b/docs/internals/thread-machines.md index a8487f8613de..12e83ea1ecca 100644 --- a/docs/internals/thread-machines.md +++ b/docs/internals/thread-machines.md @@ -639,8 +639,9 @@ Not built yet: `ProviderService` and the reactors on a hub against the real runner handlers. - End to end with a real provider: `node apps/server/scripts/thread-machines-e2e.mjs [--keep] [--model ] -[--hub-database-url [--hub-database-admin-url ]]` starts a runner (4422) and a - hub (4421) on loopback and drives the hub's public API. With a database URL the hub runs +[--hub-database-url [--hub-database-admin-url ]] [--dist ]` starts a runner + (4422) and a hub (4421) on loopback and drives the hub's public API. `--dist` runs every + process from a built package (an extracted release artifact) instead of the source. With a database URL the hub runs on Postgres for a fresh tenant seeded through `t3 hub import`, and restarts on an empty base directory. Beyond the two turns it checks the client contract: the capability, selectable provider snapshots before any runner reported, the settings push, the shell's From 8257d969918ef1bf1266d7044be6651e3cb3c3e9 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 02:51:26 +0000 Subject: [PATCH 57/58] fix(web): load stored sign-ins only inside the hub's provider status The provider settings panel queried stored sign-ins on every server, which standalone panels (and their tests) never need. The hub-only status owns the query now and reloads it after a sign-in or sign-out. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/components/hub/HubProviderSignIns.tsx | 30 +++++++++---------- .../settings/ProviderSettingsPanel.tsx | 24 ++++++--------- 2 files changed, 24 insertions(+), 30 deletions(-) diff --git a/apps/web/src/components/hub/HubProviderSignIns.tsx b/apps/web/src/components/hub/HubProviderSignIns.tsx index 3e5610490d69..46fc9292f508 100644 --- a/apps/web/src/components/hub/HubProviderSignIns.tsx +++ b/apps/web/src/components/hub/HubProviderSignIns.tsx @@ -9,7 +9,7 @@ import { squashAtomCommandFailure, } from "@t3tools/client-runtime/state/runtime"; import { LogOutIcon } from "lucide-react"; -import { useState } from "react"; +import { useEffect, useState } from "react"; import { readLocalApi } from "../../localApi"; import { useEnvironmentQuery } from "../../state/query"; @@ -21,16 +21,6 @@ import { Button } from "../ui/button"; import { toastManager } from "../ui/toast"; import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; -/** - * Hub mode: the provider sign-ins stored for thread machines. Pass `null` off - * a hub; standalone servers do not answer the call. - */ -export function useHubProviderSignIns(environmentId: EnvironmentId | null) { - return useEnvironmentQuery( - environmentId ? sourceControlEnvironment.providerSignIns({ environmentId, input: {} }) : null, - ); -} - export function providerSignInFor( list: ProviderSignInList | null, connector: AuthConnectorKind, @@ -48,21 +38,30 @@ export function describeProviderSignIn(serviceName: string, signIn: ProviderSign } /** - * Shows that a provider is signed in on thread machines, with a confirmed - * sign-out. Renders nothing when no sign-in is stored. + * Hub mode only: shows that a provider is signed in on thread machines + * (`server.listProviderSignIns`), with a confirmed sign-out. Renders nothing + * when no sign-in is stored. Bump `revision` to reload after a sign-in. */ export function HubProviderSignInStatus(props: { readonly environmentId: EnvironmentId; readonly connector: AuthConnectorKind; readonly serviceName: string; - readonly signIn: ProviderSignIn | null; + readonly revision: number; readonly onSignedOut: () => void; }) { - const { environmentId, connector, serviceName, signIn, onSignedOut } = props; + const { environmentId, connector, serviceName, revision, onSignedOut } = props; + const signIns = useEnvironmentQuery( + sourceControlEnvironment.providerSignIns({ environmentId, input: {} }), + ); + const refreshSignIns = signIns.refresh; const signOut = useAtomCommand(sourceControlEnvironment.signOutProvider, { reportFailure: false, }); const [signingOut, setSigningOut] = useState(false); + useEffect(() => { + if (revision > 0) refreshSignIns(); + }, [refreshSignIns, revision]); + const signIn = providerSignInFor(signIns.data, connector); if (signIn === null) return null; const confirmAndSignOut = async () => { @@ -80,6 +79,7 @@ export function HubProviderSignInStatus(props: { setSigningOut(false); if (result._tag === "Success") { toastManager.add({ type: "success", title: `Signed out of ${serviceName}` }); + refreshSignIns(); onSignedOut(); return; } diff --git a/apps/web/src/components/settings/ProviderSettingsPanel.tsx b/apps/web/src/components/settings/ProviderSettingsPanel.tsx index c6a6fd282a9f..aab363ada655 100644 --- a/apps/web/src/components/settings/ProviderSettingsPanel.tsx +++ b/apps/web/src/components/settings/ProviderSettingsPanel.tsx @@ -76,11 +76,7 @@ import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; import { stackedThreadToast, toastManager } from "../ui/toast"; import { AddProviderInstanceDialog } from "./AddProviderInstanceDialog"; import { AuthConnectorDialog } from "./AuthConnectorDialog"; -import { - HubProviderSignInStatus, - providerSignInFor, - useHubProviderSignIns, -} from "../hub/HubProviderSignIns"; +import { HubProviderSignInStatus } from "../hub/HubProviderSignIns"; import { useIsHubEnvironment } from "../../hubMode"; import { resolveAgentAuthMethods } from "./authConnectorMethods"; import { ProviderInstanceCard } from "./ProviderInstanceCard"; @@ -465,7 +461,8 @@ export function EnvironmentProviderSettings({ useAtomValue(serverEnvironment.providersValueAtom(environmentId)) ?? EMPTY_SERVER_PROVIDERS; const serverConfig = useAtomValue(serverEnvironment.configValueAtom(environmentId)); const hub = useIsHubEnvironment(environmentId); - const providerSignIns = useHubProviderSignIns(hub ? environmentId : null); + // Hub mode: bumped when a sign-in finishes, so the stored sign-ins reload. + const [signInRevision, setSignInRevision] = useState(0); const refreshServerProviders = useAtomCommand(serverEnvironment.refreshProviders, { reportFailure: false, }); @@ -545,13 +542,10 @@ export function EnvironmentProviderSettings({ })(); }, [environmentId, refreshServerProviders]); - // Hub mode: a sign-in or sign-out changes the stored sign-ins and every - // machine's provider status. - const refreshProviderSignIns = providerSignIns.refresh; - const refreshSignIns = useCallback(() => { - refreshProviderSignIns(); + const handleHubSignedIn = useCallback(() => { + setSignInRevision((revision) => revision + 1); refreshProviders(); - }, [refreshProviderSignIns, refreshProviders]); + }, [refreshProviders]); const runProviderUpdate = useCallback( async (candidate: ProviderUpdateCandidate) => { @@ -862,8 +856,8 @@ export function EnvironmentProviderSettings({ environmentId={environmentId} connector={authConnector.connector} serviceName={authConnector.serviceName} - signIn={providerSignInFor(providerSignIns.data, authConnector.connector)} - onSignedOut={refreshSignIns} + revision={signInRevision} + onSignedOut={refreshProviders} /> ) : null} From 0dfd2176fcc40e16ce575031b1015c5fb4016139 Mon Sep 17 00:00:00 2001 From: Logan Sack Date: Sat, 26 Sep 2026 02:59:15 +0000 Subject: [PATCH 58/58] chore(release): publish devpc 0.0.38-devpc.19 Merging this publishes devpc-v0.0.38.19 with devpc-t3-linux-x64.tar.gz, the first managed artifact with hub and runner modes. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/devpc-release-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/devpc-release-version b/.github/devpc-release-version index 240d05bc6d25..03cd7673ed7c 100644 --- a/.github/devpc-release-version +++ b/.github/devpc-release-version @@ -1 +1 @@ -0.0.38-devpc.18 +0.0.38-devpc.19