diff --git a/apps/desktop/scripts/electron-launcher.mjs b/apps/desktop/scripts/electron-launcher.mjs index 07fb87b051f1..20f535d603a5 100644 --- a/apps/desktop/scripts/electron-launcher.mjs +++ b/apps/desktop/scripts/electron-launcher.mjs @@ -228,6 +228,14 @@ function patchMainBundleInfoPlist(appBundlePath, iconPath, executableName) { setPlistString(infoPlistPath, "CFBundleIdentifier", APP_BUNDLE_ID); setPlistString(infoPlistPath, "CFBundleExecutable", executableName); setPlistString(infoPlistPath, "CFBundleIconFile", "icon.icns"); + // Without this key macOS denies every Apple Event with errAEEventNotPermitted + // (-1743) and never shows the Automation prompt, so Codex Computer Use and any + // other MCP server we spawn silently fail to drive other apps. + setPlistString( + infoPlistPath, + "NSAppleEventsUsageDescription", + "This app needs to control other apps to run Computer Use automations you approve.", + ); setPlistJson(infoPlistPath, "CFBundleURLTypes", [ { CFBundleURLName: APP_BUNDLE_ID, diff --git a/apps/desktop/src/app/DesktopApp.ts b/apps/desktop/src/app/DesktopApp.ts index 4101840530f6..e47eea0f1e9f 100644 --- a/apps/desktop/src/app/DesktopApp.ts +++ b/apps/desktop/src/app/DesktopApp.ts @@ -1,8 +1,10 @@ import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; +import { DEFAULT_SERVER_SETTINGS, ServerSettings } from "@t3tools/contracts/settings"; import * as NetService from "@t3tools/shared/Net"; import * as Crypto from "effect/Crypto"; @@ -11,6 +13,7 @@ import * as ElectronDialog from "../electron/ElectronDialog.ts"; import * as ElectronProtocol from "../electron/ElectronProtocol.ts"; import * as ElectronSafeStorage from "../electron/ElectronSafeStorage.ts"; import { installDesktopIpcHandlers } from "../ipc/DesktopIpcHandlers.ts"; +import * as ComputerHistoryManager from "../computerHistory/ComputerHistoryManager.ts"; import * as DesktopAppIdentity from "./DesktopAppIdentity.ts"; import * as DesktopClerk from "./DesktopClerk.ts"; import * as DesktopApplicationMenu from "../window/DesktopApplicationMenu.ts"; @@ -200,6 +203,32 @@ const bootstrap = Effect.gen(function* () { yield* installDesktopIpcHandlers(); yield* logBootstrapInfo("bootstrap ipc handlers registered"); + // Start the Computer History recorder when already enabled so capture resumes + // after relaunch without requiring a visit to Settings. + yield* Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const raw = yield* fileSystem + .readFileString(environment.serverSettingsPath) + .pipe(Effect.orElseSucceed(() => "{}")); + const decoded = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(ServerSettings))( + raw, + ).pipe(Effect.orElseSucceed(() => DEFAULT_SERVER_SETTINGS)); + // ensureDaemon keeps ComputerHistoryOperationError in the error channel so + // bootstrap can Effect.catch without turning spawn/layout failures into defects. + const manager = yield* ComputerHistoryManager.ComputerHistoryManager; + yield* manager.ensureDaemon(environment.stateDir, decoded.computerHistory).pipe( + Effect.catchTags({ + ComputerHistoryOperationError: (cause) => + Effect.logWarning("Computer History daemon bootstrap skipped", { cause }), + }), + ); + }).pipe( + Effect.catch((cause) => + Effect.logWarning("Computer History daemon bootstrap skipped", { cause }), + ), + ); + if (!(yield* Ref.get(state.quitting))) { // In wsl-only mode the renderer is served by the WSL backend, which can be // slow to cold-boot — show a "Connecting to WSL" splash immediately so the @@ -300,6 +329,11 @@ const scopedProgram = Effect.scoped( yield* Effect.addFinalizer(() => Effect.gen(function* () { + // electronApp.quit() can race ahead of the layer-scope cascade — stop + // Computer History here the same way we stop backends, so the recorder + // receives SIGTERM instead of surviving or being hard-killed. + const history = yield* ComputerHistoryManager.ComputerHistoryManager; + yield* history.stopDaemon().pipe(Effect.ignore); const pool = yield* DesktopBackendPool.DesktopBackendPool; // Stop every backend in the pool, not just the primary. The // electronApp.quit() path can race ahead of the layer-scope diff --git a/apps/desktop/src/computerHistory/ComputerHistoryManager.ts b/apps/desktop/src/computerHistory/ComputerHistoryManager.ts new file mode 100644 index 000000000000..7e6ba8d67c57 --- /dev/null +++ b/apps/desktop/src/computerHistory/ComputerHistoryManager.ts @@ -0,0 +1,494 @@ +// @effect-diagnostics nodeBuiltinImport:off +// @effect-diagnostics globalDate:off +import { spawn, type ChildProcess } from "node:child_process"; +import * as NodeFs from "node:fs"; +import * as NodePath from "node:path"; +import { shell } from "electron"; + +import type { + ComputerHistoryClearScope, + ComputerHistoryStatus, + ComputerHistoryTimeline, + ComputerHistorySettings, +} from "@t3tools/contracts"; +import { + clearHistory, + computerHistoryResourcesDir, + defaultCodexHome, + deleteMemory, + ensureComputerHistoryLayout, + listTimeline, + readStatusFile, + resolveComputerHistoryRoot, + writeControlFile, +} from "@t3tools/shared/computerHistory"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import { resolveDesktopMcpBinaryPathSync } from "./resolveBinary.ts"; + +type DaemonState = { + child: ChildProcess | null; + generation: number; + rootPath: string | null; + stopping: Promise | null; +}; + +export class ComputerHistoryOperationError extends Schema.TaggedErrorClass()( + "ComputerHistoryOperationError", + { + operation: Schema.Literals([ + "mergePatchSettings", + "ensureDaemon", + "stopDaemon", + "getStatus", + "getTimeline", + "clear", + "removeMemory", + "revealMemory", + ]), + root: Schema.String, + path: Schema.optionalKey(Schema.String), + cause: Schema.Defect(), + }, +) { + override get message(): string { + return `Computer History ${this.operation} failed`; + } +} + +async function writeUnavailableStatus(root: string, lastError: string): Promise { + await ensureComputerHistoryLayout(root); + const payload = { + phase: "unavailable", + accessibilityGranted: false, + eventCount: 0, + platform: process.platform, + updatedAt: new Date().toISOString(), + lastError, + }; + await NodeFs.promises.writeFile( + NodePath.join(root, "status.json"), + `${JSON.stringify(payload, null, 2)}\n`, + "utf8", + ); +} + +async function writeStoppedStatus(root: string): Promise { + await ensureComputerHistoryLayout(root); + const payload = { + phase: "stopped", + accessibilityGranted: false, + eventCount: 0, + platform: process.platform, + updatedAt: new Date().toISOString(), + }; + await NodeFs.promises.writeFile( + NodePath.join(root, "status.json"), + `${JSON.stringify(payload, null, 2)}\n`, + "utf8", + ); +} + +// Errors stay in the Effect error channel so callers can catch/ignore. +// Never convert these to defects — bootstrap and quit recover with Effect.catch. +export class ComputerHistoryManager extends Context.Service< + ComputerHistoryManager, + { + readonly mergePatchSettings: ( + stateDir: string, + persisted: ComputerHistorySettings, + patch: Partial, + ) => Effect.Effect; + readonly ensureDaemon: ( + stateDir: string, + settings: ComputerHistorySettings, + ) => Effect.Effect; + readonly stopDaemon: () => Effect.Effect; + readonly getStatus: ( + stateDir: string, + settings: ComputerHistorySettings, + ) => Effect.Effect; + readonly getTimeline: ( + stateDir: string, + ) => Effect.Effect; + readonly clear: ( + stateDir: string, + scope: ComputerHistoryClearScope, + settings: ComputerHistorySettings, + ) => Effect.Effect; + readonly removeMemory: ( + stateDir: string, + path: string, + settings: ComputerHistorySettings, + ) => Effect.Effect; + readonly revealMemory: ( + stateDir: string, + path: string, + ) => Effect.Effect; + readonly currentRoot: () => Effect.Effect; + } +>()("ComputerHistoryManager") {} + +const runDaemonOp = ( + operation: "ensureDaemon" | "stopDaemon", + root: string, + run: () => Promise, +): Effect.Effect => + Effect.tryPromise({ + try: run, + catch: (cause) => new ComputerHistoryOperationError({ operation, root, cause }), + }); + +export const make = Effect.gen(function* () { + // Plain mutable state — avoids Effect.runPromise on every Ref touch inside + // imperative daemon helpers (Macroscope Effect Service Conventions). + const state: DaemonState = { + child: null, + generation: 0, + rootPath: null, + stopping: null, + }; + + // Serialize ensure/stop so a stale status poll cannot respawn after disable. + let ensureChain: Promise = Promise.resolve(); + // Serialize settings patches so rapid IPC updates merge instead of clobbering. + let patchChain: Promise = Promise.resolve(null); + let lastMergedSettings: ComputerHistorySettings | null = null; + const enqueueEnsure = (task: () => Promise): Promise => { + const run = ensureChain.then(task, task); + ensureChain = run.then( + () => undefined, + () => undefined, + ); + return run; + }; + + const stopDaemonImpl = async (): Promise => { + if (state.stopping) { + await state.stopping; + return; + } + const child = state.child; + if (!child) return; + + let stopping!: Promise; + stopping = new Promise((resolve) => { + let settled = false; + let timer: ReturnType | undefined; + + const finish = () => { + if (settled) return; + settled = true; + if (timer !== undefined) clearTimeout(timer); + child.off("exit", onExit); + if (state.child === child) { + state.child = null; + } + // Only clear if this stop still owns the slot — a later stop must keep its promise. + if (state.stopping === stopping) { + state.stopping = null; + } + resolve(); + }; + + const onExit = () => { + finish(); + }; + child.once("exit", onExit); + try { + child.kill("SIGTERM"); + } catch { + finish(); + return; + } + timer = setTimeout(() => { + let signaled = false; + try { + // `kill` returns false when the process is already gone (ESRCH). + signaled = child.kill("SIGKILL"); + } catch { + finish(); + return; + } + if (!signaled) { + finish(); + return; + } + // Only clear after confirmed exit — do not treat signal delivery as stop. + timer = setTimeout(() => { + // Failsafe if the OS never emits `exit` after SIGKILL. + finish(); + }, 3_000); + }, 2_000); + }); + + state.stopping = stopping; + await stopping; + }; + + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + try { + await stopDaemonImpl(); + } catch { + // Best-effort teardown on layer shutdown. + } + }), + ); + + const ensureDaemonImpl = async ( + stateDir: string, + settings: ComputerHistorySettings, + ): Promise => { + const root = resolveComputerHistoryRoot(stateDir); + await ensureComputerHistoryLayout(root); + await writeControlFile(root, { + enabled: settings.enabled, + paused: settings.paused, + appFilterMode: settings.appFilterMode, + apps: [...settings.apps], + websiteFilterMode: settings.websiteFilterMode, + websites: [...settings.websites], + }); + state.rootPath = root; + + if (!settings.enabled) { + await stopDaemonImpl(); + // Clear stale running/paused status so polls do not treat an intentional + // stop as a crash while ServerSettings persistence is still catching up. + await writeStoppedStatus(root); + return; + } + + if (state.stopping) { + await state.stopping; + } + if (state.child && !state.child.killed) { + return; + } + + const binary = resolveDesktopMcpBinaryPathSync(); + if (!binary) { + await writeUnavailableStatus(root, "t3-desktop-mcp binary not found"); + return; + } + + const generation = state.generation + 1; + const child = spawn(binary, ["computer-history", "--root", root], { + stdio: ["ignore", "ignore", "pipe"], + detached: false, + }); + state.child = child; + state.generation = generation; + state.rootPath = root; + state.stopping = null; + child.stderr?.on("data", (chunk: Buffer) => { + process.stderr.write(chunk); + }); + child.on("error", (error) => { + if (state.child === child && state.generation === generation) { + state.child = null; + } + void writeUnavailableStatus( + root, + `failed to start computer-history daemon: ${error.message}`, + ).catch(() => { + // Best-effort status write — never crash the main process. + }); + }); + child.on("exit", (code, signal) => { + if (state.generation !== generation) { + return; + } + if (state.child === child) { + state.child = null; + } + // Intentional stop (disable / layer shutdown) — do not mark unavailable. + if (state.stopping !== null) { + return; + } + const detail = + code === null + ? `computer-history daemon exited (${signal ?? "signal"})` + : `computer-history daemon exited (code ${code})`; + void writeUnavailableStatus(root, detail).catch(() => { + // Best-effort status write — never crash the main process. + }); + }); + }; + + const mergePatchSettingsImpl = async ( + _stateDir: string, + persisted: ComputerHistorySettings, + patch: Partial, + ): Promise => { + const base = lastMergedSettings ?? persisted; + // Never resurrect `enabled` from control.json — the server may rewrite that + // file from stale ServerSettings while a disable patch is in flight. + const enabled = patch.enabled ?? base.enabled; + const merged = { + ...base, + ...patch, + enabled, + ...(patch.apps === undefined ? {} : { apps: [...patch.apps] }), + ...(patch.websites === undefined ? {} : { websites: [...patch.websites] }), + }; + lastMergedSettings = merged; + return merged; + }; + + const applyPatchSettings = ( + stateDir: string, + persisted: ComputerHistorySettings, + patch: Partial, + ): Promise => { + const run = patchChain.then(() => mergePatchSettingsImpl(stateDir, persisted, patch)); + patchChain = run.then( + (settings) => settings, + () => lastMergedSettings, + ); + return run; + }; + + return ComputerHistoryManager.of({ + mergePatchSettings: (stateDir, persisted, patch) => + Effect.tryPromise({ + try: () => applyPatchSettings(stateDir, persisted, patch), + catch: (cause) => { + const root = resolveComputerHistoryRoot(stateDir); + return new ComputerHistoryOperationError({ + operation: "mergePatchSettings", + root, + cause, + }); + }, + }), + ensureDaemon: (stateDir, settings) => + runDaemonOp("ensureDaemon", resolveComputerHistoryRoot(stateDir), () => + enqueueEnsure(() => ensureDaemonImpl(stateDir, settings)), + ), + stopDaemon: () => + runDaemonOp("stopDaemon", state.rootPath ?? "", () => enqueueEnsure(() => stopDaemonImpl())), + getStatus: (stateDir, settings) => + Effect.tryPromise({ + try: async () => { + const root = resolveComputerHistoryRoot(stateDir); + await ensureComputerHistoryLayout(root); + const file = await readStatusFile(root); + const daemonRunning = Boolean(state.child && !state.child.killed); + // Prefer in-memory merge over disk settings: disable can land in the + // manager before `updateSettings` finishes persisting enabled:false. + const effectiveSettings = lastMergedSettings ?? settings; + const effectiveEnabled = effectiveSettings.enabled; + const effectivePaused = effectiveSettings.paused; + const staleRunning = + effectiveEnabled && + !daemonRunning && + (file?.phase === "running" || file?.phase === "paused"); + if (staleRunning) { + await writeUnavailableStatus(root, "computer-history daemon is not running"); + } + const liveFile = staleRunning ? await readStatusFile(root) : file; + const memoriesPath = NodePath.join(root, "memories", "resources"); + const codexMirrorPath = effectiveSettings.mirrorToCodex + ? NodePath.join(defaultCodexHome(), "memories", "extensions", "skysight", "resources") + : undefined; + return { + enabled: effectiveEnabled, + paused: effectivePaused, + phase: !effectiveEnabled + ? "stopped" + : daemonRunning + ? (liveFile?.phase ?? "starting") + : (liveFile?.phase ?? "stopped"), + accessibilityGranted: liveFile?.accessibilityGranted ?? false, + rootPath: root, + memoriesPath, + ...(codexMirrorPath ? { codexMirrorPath } : {}), + ...(liveFile?.activeSegmentId ? { activeSegmentId: liveFile.activeSegmentId } : {}), + eventCount: liveFile?.eventCount ?? 0, + ...(liveFile?.lastError ? { lastError: liveFile.lastError } : {}), + platform: liveFile?.platform ?? process.platform, + } satisfies ComputerHistoryStatus; + }, + catch: (cause) => + new ComputerHistoryOperationError({ + operation: "getStatus", + root: resolveComputerHistoryRoot(stateDir), + cause, + }), + }), + getTimeline: (stateDir) => + Effect.tryPromise({ + try: () => listTimeline(resolveComputerHistoryRoot(stateDir)), + catch: (cause) => + new ComputerHistoryOperationError({ + operation: "getTimeline", + root: resolveComputerHistoryRoot(stateDir), + cause, + }), + }), + clear: (stateDir, scope, settings) => + Effect.tryPromise({ + try: () => { + const effectiveSettings = lastMergedSettings ?? settings; + return clearHistory(resolveComputerHistoryRoot(stateDir), scope, { + ...(effectiveSettings.mirrorToCodex ? { codexHome: defaultCodexHome() } : {}), + }); + }, + catch: (cause) => + new ComputerHistoryOperationError({ + operation: "clear", + root: resolveComputerHistoryRoot(stateDir), + cause, + }), + }), + removeMemory: (stateDir, path, settings) => + Effect.tryPromise({ + try: () => { + const effectiveSettings = lastMergedSettings ?? settings; + return deleteMemory(resolveComputerHistoryRoot(stateDir), path, { + ...(effectiveSettings.mirrorToCodex ? { codexHome: defaultCodexHome() } : {}), + }); + }, + catch: (cause) => + new ComputerHistoryOperationError({ + operation: "removeMemory", + root: resolveComputerHistoryRoot(stateDir), + path, + cause, + }), + }), + revealMemory: (stateDir, path) => + Effect.tryPromise({ + try: async () => { + const root = resolveComputerHistoryRoot(stateDir); + const resourcesDir = computerHistoryResourcesDir(root); + const resolved = NodePath.resolve(path); + const resourcesRoot = NodePath.resolve(resourcesDir); + if (resolved !== resourcesRoot && !resolved.startsWith(resourcesRoot + NodePath.sep)) { + return false; + } + if (!NodeFs.existsSync(resolved)) return false; + shell.showItemInFolder(resolved); + return true; + }, + catch: (cause) => + new ComputerHistoryOperationError({ + operation: "revealMemory", + root: resolveComputerHistoryRoot(stateDir), + path, + cause, + }), + }), + currentRoot: () => Effect.sync(() => state.rootPath), + }); +}); + +export const layer: Layer.Layer = Layer.effect( + ComputerHistoryManager, + make, +); diff --git a/apps/desktop/src/computerHistory/resolveBinary.ts b/apps/desktop/src/computerHistory/resolveBinary.ts new file mode 100644 index 000000000000..195f3e198a5a --- /dev/null +++ b/apps/desktop/src/computerHistory/resolveBinary.ts @@ -0,0 +1,47 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFs from "node:fs"; +import * as NodePath from "node:path"; +import { fileURLToPath } from "node:url"; + +const DESKTOP_MCP = process.platform === "win32" ? "t3-desktop-mcp.exe" : "t3-desktop-mcp"; + +/** + * Locate the bundled/dev desktop MCP binary for Computer History daemon spawn. + * Mirrors server resolution but stays sync for Electron main. + */ +export function resolveDesktopMcpBinaryPathSync(): string | undefined { + const override = process.env.T3CODE_DESKTOP_MCP_PATH; + if (override && NodeFs.existsSync(override)) return override; + + const here = NodePath.dirname(fileURLToPath(import.meta.url)); + const candidates = + process.platform === "darwin" + ? [ + NodePath.resolve( + here, + "../../../../native/t3-desktop-mcp/.build/apple/Products/Release", + DESKTOP_MCP, + ), + NodePath.resolve(here, "../../../../native/t3-desktop-mcp/.build/release", DESKTOP_MCP), + NodePath.resolve( + here, + "../../../native/t3-desktop-mcp/.build/apple/Products/Release", + DESKTOP_MCP, + ), + NodePath.resolve(process.resourcesPath ?? "", "t3-desktop-mcp", DESKTOP_MCP), + ] + : [ + NodePath.resolve( + here, + "../../../../native/t3-desktop-mcp-rs/target/release", + DESKTOP_MCP, + ), + NodePath.resolve(here, "../../../native/t3-desktop-mcp-rs/target/release", DESKTOP_MCP), + NodePath.resolve(process.resourcesPath ?? "", "t3-desktop-mcp", DESKTOP_MCP), + ]; + + for (const candidate of candidates) { + if (NodeFs.existsSync(candidate)) return candidate; + } + return undefined; +} diff --git a/apps/desktop/src/computerUse/permissions.test.ts b/apps/desktop/src/computerUse/permissions.test.ts new file mode 100644 index 000000000000..1e11e11be135 --- /dev/null +++ b/apps/desktop/src/computerUse/permissions.test.ts @@ -0,0 +1,99 @@ +// @effect-diagnostics nodeBuiltinImport:off - Fixture reads of Chrome Secure Preferences stay synchronous. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { assert, describe, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; + +vi.mock("electron", () => ({ + systemPreferences: { + isTrustedAccessibilityClient: vi.fn(() => false), + getMediaAccessStatus: vi.fn(() => "denied"), + }, + shell: { + openExternal: vi.fn(async () => undefined), + }, +})); + +import * as Electron from "electron"; + +import { openComputerUsePrivacySettings, readComputerUsePermissions } from "./permissions.ts"; + +describe("computerUse permissions", () => { + it("reports accessibility and screen recording on darwin", () => { + const previous = process.platform; + Object.defineProperty(process, "platform", { value: "darwin" }); + try { + const state = readComputerUsePermissions(); + assert.equal(state.platform, "darwin"); + assert.deepEqual( + state.permissions.map((permission) => permission.kind), + ["accessibility", "screenRecording"], + ); + assert.equal(state.permissions[0]?.status, "denied"); + assert.equal(state.permissions[1]?.status, "denied"); + } finally { + Object.defineProperty(process, "platform", { value: previous }); + } + }); + + it("marks privacy permissions not required off macOS", () => { + const previous = process.platform; + Object.defineProperty(process, "platform", { value: "linux" }); + try { + const state = readComputerUsePermissions(); + assert.equal(state.platform, "linux"); + assert.isTrue(state.permissions.every((permission) => permission.status === "notRequired")); + } finally { + Object.defineProperty(process, "platform", { value: previous }); + } + }); + + it("opens the Accessibility privacy pane and prompts trust", async () => { + const previous = process.platform; + Object.defineProperty(process, "platform", { value: "darwin" }); + try { + const opened = await openComputerUsePrivacySettings("accessibility"); + assert.isTrue(opened); + assert.equal( + vi.mocked(Electron.systemPreferences.isTrustedAccessibilityClient).mock.calls.at(-1)?.[0], + true, + ); + assert.equal( + vi.mocked(Electron.shell.openExternal).mock.calls.at(-1)?.[0], + "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility", + ); + } finally { + Object.defineProperty(process, "platform", { value: previous }); + } + }); + + it("detects an unpacked extension from Secure Preferences without state", () => { + const previous = process.platform; + Object.defineProperty(process, "platform", { value: "darwin" }); + const home = NodeOS.homedir(); + const securePath = NodePath.join( + home, + "Library/Application Support/Google/Chrome/Default/Secure Preferences", + ); + try { + if (!NodeFS.existsSync(securePath)) return; + const raw = NodeFS.readFileSync(securePath, "utf8"); + const parsed = JSON.parse(raw) as { + extensions?: { settings?: Record }; + }; + const hasExtension = Boolean(parsed.extensions?.settings?.kgdolgnijopbghhomnblabjkmjhnoage); + const state = readComputerUsePermissions(); + if (hasExtension) { + assert.deepEqual(state.chromeExtension, { + status: "installed", + detail: "Browser extension installed", + }); + } else { + assert.isTrue(["missing", "unknown"].includes(state.chromeExtension.status)); + } + } finally { + Object.defineProperty(process, "platform", { value: previous }); + } + }); +}); diff --git a/apps/desktop/src/computerUse/permissions.ts b/apps/desktop/src/computerUse/permissions.ts new file mode 100644 index 000000000000..dff57048464e --- /dev/null +++ b/apps/desktop/src/computerUse/permissions.ts @@ -0,0 +1,288 @@ +// @effect-diagnostics nodeBuiltinImport:off - Sync Chrome preference reads for IPC must stay off the Effect runtime. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { + DesktopChromeExtensionStatus, + DesktopComputerUsePermission, + DesktopComputerUsePermissionsState, + DesktopComputerUsePrivacyPane, +} from "@t3tools/contracts"; +import * as Electron from "electron"; + +const CHROME_EXTENSION_ID = "kgdolgnijopbghhomnblabjkmjhnoage"; + +function platformTag(): DesktopComputerUsePermissionsState["platform"] { + switch (process.platform) { + case "darwin": + return "darwin"; + case "win32": + return "win32"; + case "linux": + return "linux"; + default: + return "other"; + } +} + +function screenRecordingStatus(): DesktopComputerUsePermission["status"] { + try { + const status = Electron.systemPreferences.getMediaAccessStatus("screen"); + switch (status) { + case "granted": + return "granted"; + case "denied": + case "restricted": + return "denied"; + case "not-determined": + return "notDetermined"; + default: + return "unknown"; + } + } catch { + return "unknown"; + } +} + +function accessibilityStatus(): DesktopComputerUsePermission["status"] { + try { + return Electron.systemPreferences.isTrustedAccessibilityClient(false) ? "granted" : "denied"; + } catch { + return "unknown"; + } +} + +function chromeProfileRoots(): string[] { + const home = NodeOS.homedir(); + switch (process.platform) { + case "darwin": + return [ + NodePath.join(home, "Library/Application Support/Google/Chrome"), + NodePath.join(home, "Library/Application Support/Google/Chrome Beta"), + NodePath.join(home, "Library/Application Support/Google/Chrome Canary"), + NodePath.join(home, "Library/Application Support/Chromium"), + ]; + case "win32": { + const local = process.env.LOCALAPPDATA ?? NodePath.join(home, "AppData", "Local"); + return [ + NodePath.join(local, "Google", "Chrome", "User Data"), + NodePath.join(local, "Google", "Chrome Beta", "User Data"), + NodePath.join(local, "Chromium", "User Data"), + ]; + } + default: { + const config = process.env.XDG_CONFIG_HOME ?? NodePath.join(home, ".config"); + return [ + NodePath.join(config, "google-chrome"), + NodePath.join(config, "google-chrome-beta"), + NodePath.join(config, "chromium"), + ]; + } + } +} + +function profileDirectories(root: string): string[] { + try { + return NodeFS.readdirSync(root, { withFileTypes: true }) + .filter((entry) => { + if (!entry.isDirectory()) { + return false; + } + // Chrome uses "Default" plus "Profile N"; also pick up Guest/System if present. + return ( + entry.name === "Default" || + entry.name.startsWith("Profile ") || + entry.name === "Guest Profile" || + entry.name === "System Profile" + ); + }) + .map((entry) => NodePath.join(root, entry.name)); + } catch { + return []; + } +} + +function chromeExtensionInstalledInPreferences(preferencesPath: string): boolean { + try { + const raw = NodeFS.readFileSync(preferencesPath, "utf8"); + const parsed = JSON.parse(raw) as { + extensions?: { + settings?: Record< + string, + { + state?: number; + location?: number; + path?: string; + disable_reasons?: unknown; + was_installed_by_default?: boolean; + } + >; + }; + }; + const entry = parsed.extensions?.settings?.[CHROME_EXTENSION_ID]; + if (!entry) return false; + + // Explicit disable reasons mean it is present but turned off. + if (entry.disable_reasons !== undefined && entry.disable_reasons !== null) { + const reasons = entry.disable_reasons; + if (typeof reasons === "number" && reasons !== 0) return false; + if (typeof reasons === "object" && Object.keys(reasons as object).length > 0) return false; + } + + // Chromium: 0 = disabled, 1 = enabled. Unpacked installs often omit `state` + // in Secure Preferences while still being active (location 4 = UNPACKED). + if (typeof entry.state === "number") return entry.state === 1; + if (typeof entry.path === "string" && entry.path.length > 0) return true; + if (entry.location === 4) return true; + return true; + } catch { + return false; + } +} + +function nativeHostRegistered(root: string): boolean { + const hostPath = NodePath.join(root, "NativeMessagingHosts", "com.t3tools.t3code.desktop.json"); + try { + return NodeFS.statSync(hostPath).isFile(); + } catch { + return false; + } +} + +/** Windows registers the host via the registry + a support-dir manifest. */ +function nativeHostRegisteredWindows(): boolean { + const local = process.env.LOCALAPPDATA; + if (!local) return false; + const hostPath = NodePath.join(local, "t3-desktop-mcp", "com.t3tools.t3code.desktop.json"); + try { + return NodeFS.statSync(hostPath).isFile(); + } catch { + return false; + } +} + +function resolveChromeExtensionStatus(): { + status: DesktopChromeExtensionStatus; + detail: string; +} { + let sawChrome = false; + let hostRegistered = process.platform === "win32" && nativeHostRegisteredWindows(); + + for (const root of chromeProfileRoots()) { + try { + if (!NodeFS.statSync(root).isDirectory()) continue; + } catch { + continue; + } + sawChrome = true; + if (nativeHostRegistered(root)) hostRegistered = true; + + for (const profile of profileDirectories(root)) { + const preferenceFiles = [ + NodePath.join(profile, "Secure Preferences"), + NodePath.join(profile, "Preferences"), + ]; + for (const preferencesPath of preferenceFiles) { + if (chromeExtensionInstalledInPreferences(preferencesPath)) { + return { + status: "installed", + detail: "Browser extension installed", + }; + } + } + } + } + + if (!sawChrome) { + return { + status: "unknown", + detail: "Chrome profile not found on this machine", + }; + } + if (hostRegistered) { + return { + status: "missing", + detail: "Native host registered — load the unpacked extension in chrome://extensions", + }; + } + return { + status: "missing", + detail: "Browser extension not installed", + }; +} + +export function readComputerUsePermissions(): DesktopComputerUsePermissionsState { + const platform = platformTag(); + const chromeExtension = resolveChromeExtensionStatus(); + + if (platform !== "darwin") { + return { + platform, + permissions: [ + { + kind: "accessibility", + status: "notRequired", + label: "Accessibility", + }, + { + kind: "screenRecording", + status: "notRequired", + label: "Screen Recording", + }, + ], + chromeExtension, + }; + } + + return { + platform, + permissions: [ + { + kind: "accessibility", + status: accessibilityStatus(), + label: "Accessibility", + }, + { + kind: "screenRecording", + status: screenRecordingStatus(), + label: "Screen Recording", + }, + ], + chromeExtension, + }; +} + +function privacySettingsUrl(pane: DesktopComputerUsePrivacyPane): string { + // Legacy Security preference pane anchors still open the right list on + // modern macOS and are more reliable than the Settings app deep links. + switch (pane) { + case "accessibility": + return "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"; + case "screenRecording": + return "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture"; + } +} + +export async function openComputerUsePrivacySettings( + pane: DesktopComputerUsePrivacyPane, +): Promise { + if (process.platform !== "darwin") { + return false; + } + + // Prompting trust adds this app to the Accessibility list when missing. + if (pane === "accessibility") { + try { + Electron.systemPreferences.isTrustedAccessibilityClient(true); + } catch { + // Still open Settings even if the prompt API fails. + } + } + + try { + await Electron.shell.openExternal(privacySettingsUrl(pane)); + return true; + } catch { + return false; + } +} diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index 37fd873a1b03..2cdd5ad54ab2 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -13,6 +13,18 @@ import { setServerExposureMode, setTailscaleServeEnabled, } from "./methods/serverExposure.ts"; +import { + getComputerUsePermissions, + openComputerUsePrivacySettings, +} from "./methods/computerUse.ts"; +import { + clearComputerHistory, + deleteComputerHistoryMemory, + getComputerHistoryStatus, + getComputerHistoryTimeline, + patchComputerHistorySettings, + revealComputerHistoryMemory, +} from "./methods/computerHistory.ts"; import { bootstrapSshBearerSession, disconnectSshEnvironment, @@ -87,6 +99,14 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(showContextMenu); yield* ipc.handle(openExternal); yield* ipc.handle(probeRemoteEditors); + yield* ipc.handle(getComputerUsePermissions); + yield* ipc.handle(openComputerUsePrivacySettings); + yield* ipc.handle(getComputerHistoryStatus); + yield* ipc.handle(getComputerHistoryTimeline); + yield* ipc.handle(patchComputerHistorySettings); + yield* ipc.handle(clearComputerHistory); + yield* ipc.handle(revealComputerHistoryMemory); + yield* ipc.handle(deleteComputerHistoryMemory); yield* ipc.handle(getUpdateState); yield* ipc.handle(setUpdateChannel); yield* ipc.handle(downloadUpdate); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 02f9ad0df36e..a11ab28f224d 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -4,6 +4,15 @@ export const SET_THEME_CHANNEL = "desktop:set-theme"; export const CONTEXT_MENU_CHANNEL = "desktop:context-menu"; export const OPEN_EXTERNAL_CHANNEL = "desktop:open-external"; export const PROBE_REMOTE_EDITORS_CHANNEL = "desktop:probe-remote-editors"; +export const GET_COMPUTER_USE_PERMISSIONS_CHANNEL = "desktop:get-computer-use-permissions"; +export const OPEN_COMPUTER_USE_PRIVACY_SETTINGS_CHANNEL = + "desktop:open-computer-use-privacy-settings"; +export const GET_COMPUTER_HISTORY_STATUS_CHANNEL = "desktop:get-computer-history-status"; +export const GET_COMPUTER_HISTORY_TIMELINE_CHANNEL = "desktop:get-computer-history-timeline"; +export const PATCH_COMPUTER_HISTORY_SETTINGS_CHANNEL = "desktop:patch-computer-history-settings"; +export const CLEAR_COMPUTER_HISTORY_CHANNEL = "desktop:clear-computer-history"; +export const REVEAL_COMPUTER_HISTORY_MEMORY_CHANNEL = "desktop:reveal-computer-history-memory"; +export const DELETE_COMPUTER_HISTORY_MEMORY_CHANNEL = "desktop:delete-computer-history-memory"; export const MENU_ACTION_CHANNEL = "desktop:menu-action"; export const QUIT_SHORTCUT_CHANNEL = "desktop:quit-shortcut"; export const GET_WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:get-window-fullscreen-state"; diff --git a/apps/desktop/src/ipc/methods/computerHistory.ts b/apps/desktop/src/ipc/methods/computerHistory.ts new file mode 100644 index 000000000000..5d19cabed665 --- /dev/null +++ b/apps/desktop/src/ipc/methods/computerHistory.ts @@ -0,0 +1,114 @@ +// @effect-diagnostics preferSchemaOverJson:off +// @effect-diagnostics tryCatchInEffectGen:off +import { + ComputerHistoryClearScope, + ComputerHistoryStatusSchema, + ComputerHistoryTimelineSchema, +} from "@t3tools/contracts"; +import { DEFAULT_SERVER_SETTINGS, ServerSettings } from "@t3tools/contracts/settings"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Schema from "effect/Schema"; + +import * as DesktopEnvironment from "../../app/DesktopEnvironment.ts"; +import * as ComputerHistoryManager from "../../computerHistory/ComputerHistoryManager.ts"; +import * as IpcChannels from "../channels.ts"; +import * as DesktopIpc from "../DesktopIpc.ts"; + +const readHistorySettings = Effect.fn("desktop.computerHistory.readSettings")(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const raw = yield* fileSystem + .readFileString(environment.serverSettingsPath) + .pipe(Effect.orElseSucceed(() => "{}")); + const decoded = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(ServerSettings))( + raw, + ).pipe(Effect.orElseSucceed(() => DEFAULT_SERVER_SETTINGS)); + return decoded.computerHistory; +}); + +export const getComputerHistoryStatus = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.GET_COMPUTER_HISTORY_STATUS_CHANNEL, + payload: Schema.Undefined, + result: ComputerHistoryStatusSchema, + handler: Effect.fn("desktop.ipc.computerHistory.getStatus")(function* () { + // Status is observational — do not ensureDaemon here. A poll that captured + // enabled:true must not respawn the recorder after the user disables it. + // Lifecycle belongs to bootstrap + patchComputerHistorySettings. + const settings = yield* readHistorySettings(); + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const manager = yield* ComputerHistoryManager.ComputerHistoryManager; + return yield* manager.getStatus(environment.stateDir, settings); + }), +}); + +export const getComputerHistoryTimeline = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.GET_COMPUTER_HISTORY_TIMELINE_CHANNEL, + payload: Schema.Undefined, + result: ComputerHistoryTimelineSchema, + handler: Effect.fn("desktop.ipc.computerHistory.getTimeline")(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const manager = yield* ComputerHistoryManager.ComputerHistoryManager; + return yield* manager.getTimeline(environment.stateDir); + }), +}); + +export const patchComputerHistorySettings = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.PATCH_COMPUTER_HISTORY_SETTINGS_CHANNEL, + payload: Schema.Struct({ + enabled: Schema.optionalKey(Schema.Boolean), + paused: Schema.optionalKey(Schema.Boolean), + mirrorToCodex: Schema.optionalKey(Schema.Boolean), + appFilterMode: Schema.optionalKey(Schema.Literals(["exclude", "includeOnly"])), + apps: Schema.optionalKey(Schema.Array(Schema.String)), + websiteFilterMode: Schema.optionalKey(Schema.Literals(["exclude", "includeOnly"])), + websites: Schema.optionalKey(Schema.Array(Schema.String)), + }), + result: ComputerHistoryStatusSchema, + handler: Effect.fn("desktop.ipc.computerHistory.patchSettings")(function* (patch) { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const manager = yield* ComputerHistoryManager.ComputerHistoryManager; + // Persistence is owned by the server settings RPC (`updateSettings` in the + // renderer). This IPC only reconciles the recorder daemon — never rewrite + // the full settings document (that races the server's atomic writer). + const current = yield* readHistorySettings(); + const settings = yield* manager.mergePatchSettings(environment.stateDir, current, patch); + yield* manager.ensureDaemon(environment.stateDir, settings); + return yield* manager.getStatus(environment.stateDir, settings); + }), +}); + +export const clearComputerHistory = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.CLEAR_COMPUTER_HISTORY_CHANNEL, + payload: ComputerHistoryClearScope, + result: ComputerHistoryTimelineSchema, + handler: Effect.fn("desktop.ipc.computerHistory.clear")(function* (scope) { + const settings = yield* readHistorySettings(); + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const manager = yield* ComputerHistoryManager.ComputerHistoryManager; + return yield* manager.clear(environment.stateDir, scope, settings); + }), +}); + +export const revealComputerHistoryMemory = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.REVEAL_COMPUTER_HISTORY_MEMORY_CHANNEL, + payload: Schema.String, + result: Schema.Boolean, + handler: Effect.fn("desktop.ipc.computerHistory.reveal")(function* (path) { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const manager = yield* ComputerHistoryManager.ComputerHistoryManager; + return yield* manager.revealMemory(environment.stateDir, path); + }), +}); + +export const deleteComputerHistoryMemory = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.DELETE_COMPUTER_HISTORY_MEMORY_CHANNEL, + payload: Schema.String, + result: ComputerHistoryTimelineSchema, + handler: Effect.fn("desktop.ipc.computerHistory.delete")(function* (path) { + const settings = yield* readHistorySettings(); + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const manager = yield* ComputerHistoryManager.ComputerHistoryManager; + return yield* manager.removeMemory(environment.stateDir, path, settings); + }), +}); diff --git a/apps/desktop/src/ipc/methods/computerUse.ts b/apps/desktop/src/ipc/methods/computerUse.ts new file mode 100644 index 000000000000..64fa5eb32284 --- /dev/null +++ b/apps/desktop/src/ipc/methods/computerUse.ts @@ -0,0 +1,31 @@ +import { + DesktopComputerUsePermissionsStateSchema, + DesktopComputerUsePrivacyPaneSchema, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import { + openComputerUsePrivacySettings as openPrivacySettings, + readComputerUsePermissions, +} from "../../computerUse/permissions.ts"; +import * as IpcChannels from "../channels.ts"; +import * as DesktopIpc from "../DesktopIpc.ts"; + +export const getComputerUsePermissions = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.GET_COMPUTER_USE_PERMISSIONS_CHANNEL, + payload: Schema.Undefined, + result: DesktopComputerUsePermissionsStateSchema, + handler: Effect.fn("desktop.ipc.computerUse.getComputerUsePermissions")(function* () { + return readComputerUsePermissions(); + }), +}); + +export const openComputerUsePrivacySettings = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.OPEN_COMPUTER_USE_PRIVACY_SETTINGS_CHANNEL, + payload: DesktopComputerUsePrivacyPaneSchema, + result: Schema.Boolean, + handler: Effect.fn("desktop.ipc.computerUse.openComputerUsePrivacySettings")(function* (pane) { + return yield* Effect.promise(() => openPrivacySettings(pane)); + }), +}); diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 14caeed8a9a1..95e69793421b 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -63,6 +63,7 @@ import * as DesktopWindow from "./window/DesktopWindow.ts"; import * as DesktopWslBackend from "./wsl/DesktopWslBackend.ts"; import * as DesktopWslEnvironment from "./wsl/DesktopWslEnvironment.ts"; import * as DesktopWslServerTree from "./wsl/DesktopWslServerTree.ts"; +import * as ComputerHistoryManager from "./computerHistory/ComputerHistoryManager.ts"; const desktopEnvironmentLayer = Layer.unwrap( Effect.gen(function* () { @@ -136,6 +137,7 @@ const desktopFoundationLayer = Layer.mergeAll( DesktopConnectionCatalogStore.layer.pipe(Layer.provideMerge(DesktopSavedEnvironments.layer)), DesktopAssets.layer, DesktopObservability.layer, + ComputerHistoryManager.layer, ).pipe(Layer.provideMerge(desktopEnvironmentLayer)); const desktopSshLayer = desktopSshEnvironmentLayer.pipe( diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 6741e3922391..33e88f11b3c6 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -110,6 +110,22 @@ contextBridge.exposeInMainWorld("desktopBridge", { }), openExternal: (url: string) => ipcRenderer.invoke(IpcChannels.OPEN_EXTERNAL_CHANNEL, url), probeRemoteEditors: () => ipcRenderer.invoke(IpcChannels.PROBE_REMOTE_EDITORS_CHANNEL, undefined), + getComputerUsePermissions: () => + ipcRenderer.invoke(IpcChannels.GET_COMPUTER_USE_PERMISSIONS_CHANNEL, undefined), + openComputerUsePrivacySettings: (pane) => + ipcRenderer.invoke(IpcChannels.OPEN_COMPUTER_USE_PRIVACY_SETTINGS_CHANNEL, pane), + getComputerHistoryStatus: () => + ipcRenderer.invoke(IpcChannels.GET_COMPUTER_HISTORY_STATUS_CHANNEL, undefined), + getComputerHistoryTimeline: () => + ipcRenderer.invoke(IpcChannels.GET_COMPUTER_HISTORY_TIMELINE_CHANNEL, undefined), + patchComputerHistorySettings: (patch) => + ipcRenderer.invoke(IpcChannels.PATCH_COMPUTER_HISTORY_SETTINGS_CHANNEL, patch), + clearComputerHistory: (scope) => + ipcRenderer.invoke(IpcChannels.CLEAR_COMPUTER_HISTORY_CHANNEL, scope), + revealComputerHistoryMemory: (path) => + ipcRenderer.invoke(IpcChannels.REVEAL_COMPUTER_HISTORY_MEMORY_CHANNEL, path), + deleteComputerHistoryMemory: (path) => + ipcRenderer.invoke(IpcChannels.DELETE_COMPUTER_HISTORY_MEMORY_CHANNEL, path), onMenuAction: (listener) => { const wrappedListener = (_event: Electron.IpcRendererEvent, action: unknown) => { if (typeof action !== "string") return; diff --git a/apps/server/src/computerHistory/service.ts b/apps/server/src/computerHistory/service.ts new file mode 100644 index 000000000000..be5b75aa82ca --- /dev/null +++ b/apps/server/src/computerHistory/service.ts @@ -0,0 +1,88 @@ +/** + * Computer History server helpers: context injection for all providers and a + * background summarization loop when running in desktop mode. + */ +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schedule from "effect/Schedule"; +import * as Duration from "effect/Duration"; +import { + defaultCodexHome, + buildComputerHistoryContextBlock, + loadRecentContextMarkdown, + resolveComputerHistoryRoot, + runSummarizationPass, +} from "@t3tools/shared/computerHistory"; + +import * as ServerConfig from "../config.ts"; +import * as ServerSettings from "../serverSettings.ts"; + +export const loadComputerHistoryContext = Effect.fn("computerHistory.loadContext")(function* () { + const config = yield* ServerConfig.ServerConfig; + const settings = yield* ServerSettings.ServerSettingsService; + const snapshot = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => undefined)); + if (!snapshot?.computerHistory.enabled || snapshot.computerHistory.paused) { + return undefined; + } + const root = resolveComputerHistoryRoot(config.stateDir); + const markdown = yield* Effect.tryPromise(() => loadRecentContextMarkdown(root)).pipe( + Effect.orElseSucceed(() => undefined), + ); + if (!markdown) return undefined; + return buildComputerHistoryContextBlock(markdown); +}); + +/** + * Adapter-local load: capture `ServerConfig` / `ServerSettingsService` at + * construction so `sendTurn` keeps `R = never` while every provider can inject + * the same Computer History block. + */ +export const loadComputerHistoryContextProvided = ( + serverConfig: ServerConfig.ServerConfig["Service"], + serverSettings: ServerSettings.ServerSettingsService["Service"], +) => + loadComputerHistoryContext().pipe( + Effect.provideService(ServerConfig.ServerConfig, serverConfig), + Effect.provideService(ServerSettings.ServerSettingsService, serverSettings), + Effect.orElseSucceed((): string | undefined => undefined), + ); + +export const runComputerHistorySummarization = Effect.fn("computerHistory.summarize")(function* () { + const config = yield* ServerConfig.ServerConfig; + const settings = yield* ServerSettings.ServerSettingsService; + const snapshot = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => undefined)); + if (!snapshot?.computerHistory.enabled) { + return { created: 0 }; + } + const root = resolveComputerHistoryRoot(config.stateDir); + const codexHome = defaultCodexHome(); + return yield* Effect.tryPromise(() => + runSummarizationPass(root, { + mirrorToCodex: snapshot.computerHistory.mirrorToCodex, + codexHome, + }), + ).pipe(Effect.orElseSucceed(() => ({ created: 0 }))); +}); + +/** + * Fork a lightweight loop that summarizes segments. + * Safe to install in any runtime; no-ops when Computer History is disabled. + * + * Desktop IPC owns `control.json` (enable/disable/filters). The server must not + * rewrite that file from ServerSettings — Settings can patch the daemon before + * `updateSettings` lands, and a stale sync would undo enable/disable. + * + * Canonical export is `layer` (same convention as BackgroundPolicy / UsageService). + * Callers acquire state via `ServerConfig` / `ServerSettingsService` — never via + * process.env globals. + */ +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + yield* runComputerHistorySummarization().pipe(Effect.ignore); + + yield* Effect.repeat( + runComputerHistorySummarization().pipe(Effect.ignore), + Schedule.spaced(Duration.minutes(1)), + ).pipe(Effect.forkScoped); + }), +); diff --git a/apps/server/src/desktopControl/desktopMcpBinary.test.ts b/apps/server/src/desktopControl/desktopMcpBinary.test.ts new file mode 100644 index 000000000000..826a137e63a9 --- /dev/null +++ b/apps/server/src/desktopControl/desktopMcpBinary.test.ts @@ -0,0 +1,103 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { assert, describe, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; + +import { resolveDesktopMcpPath } from "./desktopMcpBinary.ts"; + +describe("desktopMcpBinary", () => { + it.effect("resolves the override path on macOS", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-binary-", + }); + const binaryPath = `${baseDir}/t3-desktop-mcp`; + yield* fileSystem.writeFileString(binaryPath, "binary"); + yield* fileSystem.chmod(binaryPath, 0o755); + + const resolved = yield* resolveDesktopMcpPath().pipe( + Effect.provideService(HostProcessPlatform, "darwin"), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: binaryPath, + }), + ); + + assert.equal(resolved, binaryPath); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("resolves the override on Linux and Windows too", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-binary-", + }); + + // Windows ships the .exe; the other platforms do not. + for (const [platform, name] of [ + ["linux", "t3-desktop-mcp"], + ["win32", "t3-desktop-mcp.exe"], + ] as const) { + const binaryPath = `${baseDir}/${name}`; + yield* fileSystem.writeFileString(binaryPath, "binary"); + if (platform !== "win32") { + yield* fileSystem.chmod(binaryPath, 0o755); + } + + const resolved = yield* resolveDesktopMcpPath().pipe( + Effect.provideService(HostProcessPlatform, platform), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: binaryPath, + }), + ); + assert.equal(resolved, binaryPath); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("returns undefined on platforms with no desktop backend", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-binary-", + }); + const binaryPath = `${baseDir}/t3-desktop-mcp`; + yield* fileSystem.writeFileString(binaryPath, "binary"); + + // Neither backend covers these, so the tools must not be offered even + // when someone points the override at a binary. + for (const platform of ["freebsd", "aix"] as const) { + const resolved = yield* resolveDesktopMcpPath().pipe( + Effect.provideService(HostProcessPlatform, platform), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: binaryPath, + }), + ); + assert.equal(resolved, undefined); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("returns undefined when nothing is built or overridden", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-binary-", + }); + + const resolved = yield* resolveDesktopMcpPath().pipe( + Effect.provideService(HostProcessPlatform, "darwin"), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: `${baseDir}/does-not-exist`, + }), + ); + + // A dev checkout that has built the binary will resolve a bundled + // candidate; otherwise nothing matches. Both are valid — the contract is + // that a missing override never throws and never returns the bad path. + assert.notEqual(resolved, `${baseDir}/does-not-exist`); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/desktopControl/desktopMcpBinary.ts b/apps/server/src/desktopControl/desktopMcpBinary.ts new file mode 100644 index 000000000000..4284aea20b17 --- /dev/null +++ b/apps/server/src/desktopControl/desktopMcpBinary.ts @@ -0,0 +1,96 @@ +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; + +export const DESKTOP_MCP_EXECUTABLE_NAME = "t3-desktop-mcp"; + +/** + * Locate the bundled desktop-control MCP server. + * + * macOS ships the Swift package in `native/t3-desktop-mcp`. Windows and Linux + * ship the Rust crate in `native/t3-desktop-mcp-rs`. Candidate lists are + * platform-specific so a stray Rust binary on macOS (or Swift on Windows) is + * never preferred over the functional backend. Resolves to undefined when the + * binary is absent — callers treat that as "do not offer the tools". + */ +export const resolveDesktopMcpPath = Effect.fn("desktopControl.resolveDesktopMcpPath")( + function* () { + const platform = yield* HostProcessPlatform; + if (platform !== "darwin" && platform !== "win32" && platform !== "linux") { + return undefined; + } + + const environment = yield* HostProcessEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + + const override = environment.T3CODE_DESKTOP_MCP_PATH; + // Windows keeps the extension; the staged directory name does not. + const executableName = + platform === "win32" ? `${DESKTOP_MCP_EXECUTABLE_NAME}.exe` : DESKTOP_MCP_EXECUTABLE_NAME; + + const packaged = [ + // Packaged: staged into app Resources beside the server bundle. + path.resolve(import.meta.dirname, DESKTOP_MCP_EXECUTABLE_NAME, executableName), + path.resolve(import.meta.dirname, "..", DESKTOP_MCP_EXECUTABLE_NAME, executableName), + ]; + + const rustDev = [ + path.resolve( + import.meta.dirname, + "../../../../native/t3-desktop-mcp-rs/target/release", + executableName, + ), + path.resolve( + import.meta.dirname, + "../../../native/t3-desktop-mcp-rs/target/release", + executableName, + ), + ]; + + const swiftDev = [ + path.resolve( + import.meta.dirname, + "../../../../native/t3-desktop-mcp/.build/apple/Products/Release", + DESKTOP_MCP_EXECUTABLE_NAME, + ), + path.resolve( + import.meta.dirname, + "../../../../native/t3-desktop-mcp/.build/release", + DESKTOP_MCP_EXECUTABLE_NAME, + ), + path.resolve( + import.meta.dirname, + "../../../native/t3-desktop-mcp/.build/apple/Products/Release", + DESKTOP_MCP_EXECUTABLE_NAME, + ), + ]; + + const candidates = [ + ...(override ? [override] : []), + ...packaged, + ...(platform === "darwin" ? swiftDev : rustDev), + ]; + + for (const candidate of candidates) { + const exists = yield* fileSystem.exists(candidate).pipe(Effect.orElseSucceed(() => false)); + if (!exists) { + continue; + } + const stat = yield* fileSystem.stat(candidate).pipe(Effect.option); + if (Option.isNone(stat) || stat.value.type !== "File") { + continue; + } + // Windows does not use POSIX execute bits the same way; existence of a + // regular file is enough. On POSIX, skip non-executable paths so a bad + // override does not block a valid packaged binary. + if (platform !== "win32" && (stat.value.mode & 0o111) === 0) { + continue; + } + return candidate; + } + return undefined; + }, +); diff --git a/apps/server/src/desktopControl/desktopMcpLaunch.test.ts b/apps/server/src/desktopControl/desktopMcpLaunch.test.ts new file mode 100644 index 000000000000..efdf38ab5830 --- /dev/null +++ b/apps/server/src/desktopControl/desktopMcpLaunch.test.ts @@ -0,0 +1,105 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { ServerSettingsError } from "@t3tools/contracts"; +import { assert, describe, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Stream from "effect/Stream"; + +import * as ServerSettings from "../serverSettings.ts"; +import { resolveEnabledDesktopMcp } from "./desktopMcpLaunch.ts"; + +describe("resolveEnabledDesktopMcp", () => { + it.effect("omits the MCP when Computer Use is disabled", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-enabled-", + }); + const binaryPath = `${baseDir}/t3-desktop-mcp`; + yield* fileSystem.writeFileString(binaryPath, "binary"); + yield* fileSystem.chmod(binaryPath, 0o755); + + const resolved = yield* resolveEnabledDesktopMcp().pipe( + Effect.provideService(HostProcessPlatform, "darwin"), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: binaryPath, + }), + Effect.provide(ServerSettings.layerTest({ desktopControl: { enabled: false } })), + ); + + assert.equal(resolved, undefined); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("passes env when agent cursor or browser control is off", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-enabled-", + }); + const binaryPath = `${baseDir}/t3-desktop-mcp`; + yield* fileSystem.writeFileString(binaryPath, "binary"); + yield* fileSystem.chmod(binaryPath, 0o755); + + const resolved = yield* resolveEnabledDesktopMcp().pipe( + Effect.provideService(HostProcessPlatform, "darwin"), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: binaryPath, + }), + Effect.provide( + ServerSettings.layerTest({ + desktopControl: { + enabled: true, + agentCursorEnabled: false, + browserControlEnabled: false, + }, + }), + ), + ); + + assert.isDefined(resolved); + assert.equal(resolved?.path, binaryPath); + assert.deepEqual(resolved?.env, [ + { name: "T3_DESKTOP_AGENT_CURSOR", value: "0" }, + { name: "T3_DESKTOP_BROWSER", value: "0" }, + ]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("omits the MCP when settings cannot be read", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-enabled-", + }); + const binaryPath = `${baseDir}/t3-desktop-mcp`; + yield* fileSystem.writeFileString(binaryPath, "binary"); + yield* fileSystem.chmod(binaryPath, 0o755); + + const settingsError = new ServerSettingsError({ + settingsPath: `${baseDir}/settings.json`, + operation: "read-file", + cause: "boom", + }); + const failingService = { + start: Effect.void, + ready: Effect.void, + getSettings: Effect.fail(settingsError), + updateSettings: () => Effect.fail(settingsError), + streamChanges: Stream.empty, + subscribeChanges: Effect.succeed(Stream.empty), + }; + + const resolved = yield* resolveEnabledDesktopMcp().pipe( + Effect.provideService(HostProcessPlatform, "darwin"), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: binaryPath, + }), + Effect.provideService(ServerSettings.ServerSettingsService, failingService as never), + ); + + assert.equal(resolved, undefined); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/desktopControl/desktopMcpLaunch.ts b/apps/server/src/desktopControl/desktopMcpLaunch.ts new file mode 100644 index 000000000000..dba5627b8138 --- /dev/null +++ b/apps/server/src/desktopControl/desktopMcpLaunch.ts @@ -0,0 +1,90 @@ +/** + * Resolves the desktop MCP binary only when Computer Use is enabled in + * server settings. Settings lookup failures fail closed (tools omitted). + */ +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import * as ServerSettings from "../serverSettings.ts"; +import { resolveDesktopMcpPath } from "./desktopMcpBinary.ts"; + +export type DesktopMcpLaunch = { + readonly path: string; + readonly env: ReadonlyArray<{ readonly name: string; readonly value: string }>; +}; + +type DesktopControlFlags = { + readonly enabled: boolean; + readonly agentCursorEnabled: boolean; + readonly browserControlEnabled: boolean; +}; + +const disabledDesktopControl = { + enabled: false, + agentCursorEnabled: false, + browserControlEnabled: false, +} as const satisfies DesktopControlFlags; + +/** + * Always acquire `ServerSettings.ServerSettingsService` from the Effect + * environment. Callers that need R=never session methods should use + * `makeResolveEnabledDesktopMcp` instead of yielding this effect directly. + */ +const readDesktopControlFlags = Effect.fn("desktopControl.readDesktopControlFlags")(function* () { + const settings = yield* ServerSettings.ServerSettingsService; + return yield* settings.getSettings.pipe( + Effect.map((snapshot): DesktopControlFlags => snapshot.desktopControl), + // Fail closed: never inject desktop MCP when we cannot confirm enablement. + Effect.orElseSucceed((): DesktopControlFlags => disabledDesktopControl), + ); +}); + +export const resolveEnabledDesktopMcp = Effect.fn("desktopControl.resolveEnabledDesktopMcp")( + function* () { + const path = yield* resolveDesktopMcpPath(); + if (path === undefined) { + return undefined; + } + + const desktopControl = yield* readDesktopControlFlags(); + if (!desktopControl.enabled) { + return undefined; + } + + const env: Array<{ name: string; value: string }> = []; + if (!desktopControl.agentCursorEnabled) { + env.push({ name: "T3_DESKTOP_AGENT_CURSOR", value: "0" }); + } + if (!desktopControl.browserControlEnabled) { + env.push({ name: "T3_DESKTOP_BROWSER", value: "0" }); + } + + return { path, env } satisfies DesktopMcpLaunch; + }, +); + +/** + * Capture desktop-MCP resolution dependencies once at adapter construction so + * each session can re-read settings without widening `startSession`'s Effect + * context (adapters require `R = never` on session methods). + */ +export const makeResolveEnabledDesktopMcp = Effect.fn( + "desktopControl.makeResolveEnabledDesktopMcp", +)(function* () { + const settings = yield* ServerSettings.ServerSettingsService; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const platform = yield* HostProcessPlatform; + const environment = yield* HostProcessEnvironment; + + return () => + resolveEnabledDesktopMcp().pipe( + Effect.provideService(ServerSettings.ServerSettingsService, settings), + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + Effect.provideService(HostProcessPlatform, platform), + Effect.provideService(HostProcessEnvironment, environment), + ); +}); diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.approval.test.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.approval.test.ts index 05370781c0d0..6e9e2fccebad 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.approval.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.approval.test.ts @@ -30,4 +30,52 @@ describe("runtimeEventToActivities approval details", () => { expect(activity?.kind).toBe("approval.requested"); expect((activity?.payload as Record | undefined)?.detail).toBe(detail); }); + + it("preserves permission approvals for the client", () => { + const event = { + type: "request.opened", + eventId: EventId.make("evt-permissions-opened"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-08-10T00:00:00.000Z", + threadId: ThreadId.make("thread-1"), + requestId: RuntimeRequestId.make("permissions-1"), + payload: { + requestType: "permissions_approval", + detail: "Control the desktop", + }, + } satisfies ProviderRuntimeEvent; + + const [activity] = runtimeEventToActivities(event); + + expect(activity?.summary).toBe("Permission requested"); + expect((activity?.payload as Record | undefined)?.requestKind).toBe( + "permissions", + ); + expect((activity?.payload as Record | undefined)?.requestType).toBe( + "permissions_approval", + ); + }); + + it("preserves generic MCP tool approvals for the client", () => { + const event = { + type: "request.opened", + eventId: EventId.make("evt-tool-opened"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-08-10T00:00:00.000Z", + threadId: ThreadId.make("thread-1"), + requestId: RuntimeRequestId.make("tool-1"), + payload: { + requestType: "tool_approval", + detail: "Allow node_repl to run this tool call?", + }, + } satisfies ProviderRuntimeEvent; + + const [activity] = runtimeEventToActivities(event); + + expect(activity?.summary).toBe("Tool approval requested"); + expect((activity?.payload as Record | undefined)?.requestKind).toBe("tool"); + expect((activity?.payload as Record | undefined)?.requestType).toBe( + "tool_approval", + ); + }); }); diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index e24825d3d1fc..973471a726dc 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -298,7 +298,7 @@ function sessionStatusAllowsActiveTurn( function requestKindFromCanonicalRequestType( requestType: string | undefined, -): "command" | "file-read" | "file-change" | undefined { +): "command" | "file-read" | "file-change" | "tool" | "permissions" | undefined { switch (requestType) { case "command_execution_approval": case "exec_command_approval": @@ -308,6 +308,10 @@ function requestKindFromCanonicalRequestType( case "file_change_approval": case "apply_patch_approval": return "file-change"; + case "permissions_approval": + return "permissions"; + case "tool_approval": + return "tool"; default: return undefined; } @@ -388,7 +392,11 @@ export function runtimeEventToActivities( ? "File-read approval requested" : requestKind === "file-change" ? "File-change approval requested" - : "Approval requested", + : requestKind === "tool" + ? "Tool approval requested" + : requestKind === "permissions" + ? "Permission requested" + : "Approval requested", payload: { requestId: toApprovalRequestId(event.requestId), ...(requestKind ? { requestKind } : {}), diff --git a/apps/server/src/provider/CodexDeveloperInstructions.ts b/apps/server/src/provider/CodexDeveloperInstructions.ts index aa7d106e102c..f2e927484054 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.ts @@ -169,12 +169,14 @@ function toSingleLine(value: string): string { export function buildCodexDeveloperInstructions( interactionMode: ProviderInteractionMode, runtime: CodexRuntimeInfo, + options?: { readonly computerHistoryContext?: string }, ): string { const base = interactionMode === "plan" ? CODEX_PLAN_MODE_DEVELOPER_INSTRUCTIONS : CODEX_DEFAULT_MODE_DEVELOPER_INSTRUCTIONS; + const history = options?.computerHistoryContext ? `\n\n${options.computerHistoryContext}` : ""; return `${base} -In case you're asked: you are running in T3 Code through the Codex harness, as ${toSingleLine(runtime.model)} with ${toSingleLine(runtime.reasoningEffort)} reasoning effort. No need to mention this otherwise.`; +In case you're asked: you are running in T3 Code through the Codex harness, as ${toSingleLine(runtime.model)} with ${toSingleLine(runtime.reasoningEffort)} reasoning effort. No need to mention this otherwise.${history}`; } diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 5715b68a1e45..daab44cd3f6a 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -74,7 +74,10 @@ import * as Stream from "effect/Stream"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; +import * as ComputerHistoryService from "../../computerHistory/service.ts"; +import { makeResolveEnabledDesktopMcp } from "../../desktopControl/desktopMcpLaunch.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; +import * as ServerSettings from "../../serverSettings.ts"; import { resolveClaudeSdkExecutablePath } from "../Drivers/ClaudeExecutable.ts"; import { makeClaudeEnvironment } from "../Drivers/ClaudeHome.ts"; import { @@ -1257,11 +1260,16 @@ const buildUserMessageEffect = Effect.fn("buildUserMessageEffect")(function* ( readonly fileSystem: FileSystem.FileSystem; readonly attachmentsDir: string; readonly boundInstanceId: ProviderInstanceId; + readonly computerHistoryContext?: string; }, ) { const text = buildPromptText(input, dependencies.boundInstanceId); const sdkContent: Array> = []; + if (dependencies.computerHistoryContext) { + sdkContent.push({ type: "text", text: dependencies.computerHistoryContext }); + } + if (text.length > 0) { sdkContent.push({ type: "text", text }); } @@ -1659,6 +1667,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const serverConfig = yield* ServerConfig; + const serverSettings = yield* ServerSettings.ServerSettingsService; const crypto = yield* Crypto.Crypto; const claudeEnvironment = yield* makeClaudeEnvironment(claudeSettings, options?.environment).pipe( Effect.provideService(Path.Path, path), @@ -1676,6 +1685,8 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( : undefined); const managedNativeEventLogger = options?.nativeEventLogger === undefined ? nativeEventLogger : undefined; + // Re-resolve per session (settings toggles) without widening startSession's R. + const resolveDesktopMcp = yield* makeResolveEnabledDesktopMcp(); const createQuery = options?.createQuery ?? @@ -4151,6 +4162,38 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( ...(input.cwd ? [input.cwd] : []), serverConfig.attachmentsDir, ]; + // Desktop MCP is offered when the platform binary resolves and Computer + // Use is enabled. Re-resolve per session so Settings toggles apply without + // restarting the app. + const desktopMcp = yield* resolveDesktopMcp(); + const mcpServers = { + ...(mcpSession + ? { + "t3-code": { + type: "http" as const, + url: mcpSession.endpoint, + headers: { + Authorization: mcpSession.authorizationHeader, + }, + }, + } + : {}), + ...(desktopMcp + ? { + "t3-desktop": { + type: "stdio" as const, + command: desktopMcp.path, + ...(desktopMcp.env.length > 0 + ? { + env: Object.fromEntries( + desktopMcp.env.map((entry) => [entry.name, entry.value]), + ), + } + : {}), + }, + } + : {}), + }; const queryOptions: ClaudeQueryOptions = { ...(input.cwd ? { cwd: input.cwd } : {}), ...(apiModelId ? { model: apiModelId } : {}), @@ -4176,19 +4219,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( env: claudeEnvironment, additionalDirectories, ...(Object.keys(extraArgs).length > 0 ? { extraArgs } : {}), - ...(mcpSession - ? { - mcpServers: { - "t3-code": { - type: "http", - url: mcpSession.endpoint, - headers: { - Authorization: mcpSession.authorizationHeader, - }, - }, - }, - } - : {}), + ...(Object.keys(mcpServers).length > 0 ? { mcpServers } : {}), }; yield* Effect.annotateCurrentSpan({ @@ -4443,10 +4474,15 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( }); } + const computerHistoryContext = yield* ComputerHistoryService.loadComputerHistoryContextProvided( + serverConfig, + serverSettings, + ); const message = yield* buildUserMessageEffect(input, { fileSystem, attachmentsDir: serverConfig.attachmentsDir, boundInstanceId, + ...(computerHistoryContext ? { computerHistoryContext } : {}), }); yield* Queue.offer(context.promptQueue, { diff --git a/apps/server/src/provider/Layers/CodexAdapter.test.ts b/apps/server/src/provider/Layers/CodexAdapter.test.ts index 5358716aabe4..87836725d6ab 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.test.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.test.ts @@ -936,6 +936,129 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { }), ); + it.effect("maps Computer Use permission requests and resolutions to canonical approvals", () => + Effect.gen(function* () { + const { adapter, runtime } = yield* startLifecycleRuntime(); + const eventsFiber = yield* Stream.runCollect(Stream.take(adapter.streamEvents, 2)).pipe( + Effect.forkChild, + ); + + yield* runtime.emit({ + id: asEventId("evt-permissions-requested"), + kind: "request", + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + turnId: asTurnId("turn-1"), + itemId: asItemId("permissions-1"), + createdAt: "2026-01-01T00:00:00.000Z", + method: "mcpServer/elicitation/request", + requestKind: "permissions", + requestId: ApprovalRequestId.make("req-permissions-1"), + payload: { + _meta: { + codex_approval_kind: "mcp_tool_call", + connector_id: "computer-use", + }, + message: "Control the desktop", + mode: "form", + requestedSchema: { type: "object", properties: {} }, + serverName: "computer-use", + threadId: "provider-thread-1", + turnId: "turn-1", + }, + } satisfies ProviderEvent); + yield* runtime.emit({ + id: asEventId("evt-permissions-resolved"), + kind: "notification", + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + turnId: asTurnId("turn-1"), + itemId: asItemId("permissions-1"), + createdAt: "2026-01-01T00:00:01.000Z", + method: "item/requestApproval/decision", + requestKind: "permissions", + requestId: ApprovalRequestId.make("req-permissions-1"), + payload: { + requestId: "req-permissions-1", + requestKind: "permissions", + decision: "accept", + }, + } satisfies ProviderEvent); + + const events = Array.from(yield* Fiber.join(eventsFiber)); + NodeAssert.equal(events[0]?.type, "request.opened"); + if (events[0]?.type === "request.opened") { + NodeAssert.equal(events[0].payload.requestType, "permissions_approval"); + NodeAssert.equal(events[0].payload.detail, "Control the desktop"); + } + NodeAssert.equal(events[1]?.type, "request.resolved"); + if (events[1]?.type === "request.resolved") { + NodeAssert.equal(events[1].payload.requestType, "permissions_approval"); + NodeAssert.equal(events[1].payload.decision, "accept"); + } + }), + ); + + it.effect("maps generic MCP tool requests and resolutions to canonical tool approvals", () => + Effect.gen(function* () { + const { adapter, runtime } = yield* startLifecycleRuntime(); + const eventsFiber = yield* Stream.runCollect(Stream.take(adapter.streamEvents, 2)).pipe( + Effect.forkChild, + ); + + yield* runtime.emit({ + id: asEventId("evt-tool-requested"), + kind: "request", + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + turnId: asTurnId("turn-1"), + createdAt: "2026-01-01T00:00:00.000Z", + method: "mcpServer/elicitation/request", + requestKind: "tool", + requestId: ApprovalRequestId.make("req-tool-1"), + payload: { + _meta: { + codex_approval_kind: "mcp_tool_call", + }, + message: "Allow node_repl to run this tool call?", + mode: "form", + requestedSchema: { type: "object", properties: {} }, + serverName: "node_repl", + threadId: "provider-thread-1", + turnId: "turn-1", + }, + } satisfies ProviderEvent); + yield* runtime.emit({ + id: asEventId("evt-tool-resolved"), + kind: "notification", + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + turnId: asTurnId("turn-1"), + createdAt: "2026-01-01T00:00:01.000Z", + method: "item/requestApproval/decision", + requestKind: "tool", + requestId: ApprovalRequestId.make("req-tool-1"), + payload: { + requestId: "req-tool-1", + requestKind: "tool", + decision: "accept", + }, + } satisfies ProviderEvent); + + const events = Array.from(yield* Fiber.join(eventsFiber)); + NodeAssert.equal(events[0]?.type, "request.opened"); + if (events[0]?.type === "request.opened") { + NodeAssert.equal(events[0].payload.requestType, "tool_approval"); + NodeAssert.equal(events[0].payload.detail, "Allow node_repl to run this tool call?"); + } + NodeAssert.equal(events[1]?.type, "request.resolved"); + if (events[1]?.type === "request.resolved") { + NodeAssert.equal(events[1].payload.requestType, "tool_approval"); + NodeAssert.equal(events[1].payload.decision, "accept"); + } + }), + ); + it.effect("preserves explicit empty multi-select user-input answers", () => Effect.gen(function* () { const { adapter, runtime } = yield* startLifecycleRuntime(); diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 065156d36473..b3ca6e9cd1bd 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -41,6 +41,7 @@ import * as EffectCodexSchema from "effect-codex-app-server/schema"; import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import { getCodexServiceTierOptionValue } from "../../codexModelOptions.ts"; +import { makeResolveEnabledDesktopMcp } from "../../desktopControl/desktopMcpLaunch.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; import { @@ -54,6 +55,8 @@ import { import { type CodexAdapterShape } from "../Services/CodexAdapter.ts"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; +import * as ComputerHistoryService from "../../computerHistory/service.ts"; +import * as ServerSettings from "../../serverSettings.ts"; import { CodexResumeCursorSchema, CodexSessionRuntimeThreadIdMissingError, @@ -302,6 +305,8 @@ function toRequestTypeFromMethod(method: string): CanonicalRequestType { return "file_read_approval"; case "item/fileChange/requestApproval": return "file_change_approval"; + case "item/permissions/requestApproval": + return "permissions_approval"; case "applyPatchApproval": return "apply_patch_approval"; case "execCommandApproval": @@ -325,6 +330,10 @@ function toRequestTypeFromKind(kind: ProviderRequestKind | undefined): Canonical return "file_read_approval"; case "file-change": return "file_change_approval"; + case "permissions": + return "permissions_approval"; + case "tool": + return "tool_approval"; default: return "unknown"; } @@ -819,6 +828,20 @@ function mapToRuntimeEvents( ); return payload?.reason ?? undefined; } + case "item/permissions/requestApproval": { + const payload = readPayload( + EffectCodexSchema.ServerRequest__PermissionsRequestApprovalParams, + event.payload, + ); + return payload?.reason ?? undefined; + } + case "mcpServer/elicitation/request": { + const payload = readPayload( + EffectCodexSchema.ServerRequest__McpServerElicitationRequestParams, + event.payload, + ); + return payload?.message ?? undefined; + } case "applyPatchApproval": { const payload = readPayload( EffectCodexSchema.ServerRequest__ApplyPatchApprovalParams, @@ -850,7 +873,9 @@ function mapToRuntimeEvents( ...runtimeEventBase(event, canonicalThreadId), type: "request.opened", payload: { - requestType: toRequestTypeFromMethod(event.method), + requestType: event.requestKind + ? toRequestTypeFromKind(event.requestKind) + : toRequestTypeFromMethod(event.method), ...(detail ? { detail } : {}), ...(event.payload !== undefined ? { args: event.payload } : {}), }, @@ -1626,10 +1651,12 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( options?: CodexAdapterLiveOptions, ) { const boundInstanceId = options?.instanceId ?? ProviderInstanceId.make("codex"); + const resolveDesktopMcp = yield* makeResolveEnabledDesktopMcp(); const fileSystem = yield* FileSystem.FileSystem; const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const crypto = yield* Crypto.Crypto; const serverConfig = yield* Effect.service(ServerConfig); + const serverSettings = yield* ServerSettings.ServerSettingsService; const nativeEventLogger = options?.nativeEventLogger ?? (options?.nativeEventLogPath !== undefined @@ -1663,6 +1690,40 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( ? getCodexServiceTierOptionValue(input.modelSelection) : undefined; const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); + // Offer Computer Use tools whenever the desktop MCP resolves and is + // enabled — same as Claude/Cursor/Grok. Wire via Codex `-c mcp_servers.*`. + const desktopMcp = yield* resolveDesktopMcp(); + const appServerArgs: string[] = []; + if (mcpSession) { + appServerArgs.push( + "-c", + `mcp_servers.t3-code.url=${mcpSession.endpoint}`, + "-c", + 'mcp_servers.t3-code.bearer_token_env_var="T3_MCP_BEARER_TOKEN"', + ); + } + if (desktopMcp) { + // Prefer TOML literal strings so Windows paths keep backslashes; + // fall back to escaped basic strings when a value contains `'`. + const quoteToml = (value: string): string => { + if (!value.includes("'") && !value.includes("\n") && !value.includes("\r")) { + return `'${value}'`; + } + return `"${value + .replace(/\\/g, "\\\\") + .replace(/"/g, '\\"') + .replace(/\n/g, "\\n") + .replace(/\r/g, "\\r") + .replace(/\t/g, "\\t")}"`; + }; + appServerArgs.push("-c", `mcp_servers.t3-desktop.command=${quoteToml(desktopMcp.path)}`); + for (const entry of desktopMcp.env) { + appServerArgs.push( + "-c", + `mcp_servers.t3-desktop.env.${entry.name}=${quoteToml(entry.value)}`, + ); + } + } const runtimeInput: CodexSessionRuntimeOptions = { threadId: input.threadId, providerInstanceId: boundInstanceId, @@ -1679,18 +1740,20 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( ? { model: input.modelSelection.model } : {}), ...(serviceTier ? { serviceTier } : {}), - ...(mcpSession + ...(mcpSession || desktopMcp ? { environment: { ...(options?.environment ?? process.env), - T3_MCP_BEARER_TOKEN: mcpSession.authorizationHeader.replace(/^Bearer\s+/, ""), + ...(mcpSession + ? { + T3_MCP_BEARER_TOKEN: mcpSession.authorizationHeader.replace( + /^Bearer\s+/, + "", + ), + } + : {}), }, - appServerArgs: [ - "-c", - `mcp_servers.t3-code.url=${mcpSession.endpoint}`, - "-c", - 'mcp_servers.t3-code.bearer_token_env_var="T3_MCP_BEARER_TOKEN"', - ], + appServerArgs, } : {}), }; @@ -1816,6 +1879,10 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( input.modelSelection?.instanceId === boundInstanceId ? getCodexServiceTierOptionValue(input.modelSelection) : undefined; + const computerHistoryContext = yield* ComputerHistoryService.loadComputerHistoryContextProvided( + serverConfig, + serverSettings, + ); return yield* session.runtime .sendTurn({ ...(input.input !== undefined ? { input: input.input } : {}), @@ -1830,6 +1897,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( ...(serviceTier ? { serviceTier } : {}), ...(input.interactionMode !== undefined ? { interactionMode: input.interactionMode } : {}), ...(codexAttachments.length > 0 ? { attachments: codexAttachments } : {}), + ...(computerHistoryContext ? { computerHistoryContext } : {}), }) .pipe(Effect.mapError((cause) => mapCodexRuntimeError(input.threadId, "turn/start", cause))); }); diff --git a/apps/server/src/provider/Layers/CodexProvider.test.ts b/apps/server/src/provider/Layers/CodexProvider.test.ts index 26e77f82a79e..3f0843fc7007 100644 --- a/apps/server/src/provider/Layers/CodexProvider.test.ts +++ b/apps/server/src/provider/Layers/CodexProvider.test.ts @@ -2,10 +2,18 @@ import { assert, it } from "@effect/vitest"; import { applyPreferredCodexDefaultModel, + buildCodexInitializeParams, isLegacyCodexModel, mapCodexModelCapabilities, } from "./CodexProvider.ts"; +it("advertises form elicitation support to Codex App Server sessions", () => { + assert.deepStrictEqual(buildCodexInitializeParams().capabilities, { + experimentalApi: true, + mcpServerOpenaiFormElicitation: true, + }); +}); + it("keeps only the GPT-5.6 Codex family out of legacy models", () => { assert.deepStrictEqual( ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-5.6-sol", "gpt-5.4"].map((model) => [ diff --git a/apps/server/src/provider/Layers/CodexProvider.ts b/apps/server/src/provider/Layers/CodexProvider.ts index 5c0f76dff4e3..7d694bf7c0c0 100644 --- a/apps/server/src/provider/Layers/CodexProvider.ts +++ b/apps/server/src/provider/Layers/CodexProvider.ts @@ -315,6 +315,7 @@ export function buildCodexInitializeParams(): CodexSchema.V1InitializeParams { }, capabilities: { experimentalApi: true, + mcpServerOpenaiFormElicitation: true, }, }; } diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts index 2a2de6839209..a15822e4826b 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts @@ -16,10 +16,15 @@ import { } from "../CodexDeveloperInstructions.ts"; import { codexSessionAppServerArgs } from "./codexLaunchArgs.ts"; import { + buildMcpApprovalResponse, + buildPermissionsApprovalResponse, buildTurnStartParams, hasConfiguredMcpServer, + isComputerUseMcpApproval, + isMcpToolApproval, isRecoverableThreadResumeError, makeMemoryConsolidationNotificationFilter, + mcpApprovalRequestKind, openCodexThread, } from "./CodexSessionRuntime.ts"; const isCodexAppServerRequestError = Schema.is(CodexErrors.CodexAppServerRequestError); @@ -248,6 +253,116 @@ describe("buildTurnStartParams", () => { }); }); +describe("buildPermissionsApprovalResponse", () => { + const permissions = { + network: { enabled: true }, + fileSystem: { + entries: [{ access: "write" as const, path: { type: "path" as const, path: "/tmp" } }], + }, + }; + + it("grants the requested execution context for this turn", () => { + NodeAssert.deepStrictEqual(buildPermissionsApprovalResponse(permissions, "accept"), { + permissions, + scope: "turn", + }); + }); + + it("persists an accepted execution context only for acceptForSession", () => { + NodeAssert.deepStrictEqual(buildPermissionsApprovalResponse(permissions, "acceptForSession"), { + permissions, + scope: "session", + }); + }); + + it("denies every requested capability on decline or cancellation", () => { + for (const decision of ["decline", "cancel"] as const) { + NodeAssert.deepStrictEqual(buildPermissionsApprovalResponse(permissions, decision), { + permissions: {}, + scope: "turn", + }); + } + }); +}); + +describe("MCP tool approval", () => { + const request = { + _meta: { + codex_approval_kind: "mcp_tool_call", + connector_id: "computer-use", + persist: ["session", "always"], + }, + message: "Allow Computer Use to control this desktop?", + mode: "form" as const, + requestedSchema: { type: "object" as const, properties: {} }, + serverName: "computer-use", + threadId: "provider-thread-1", + turnId: "turn-1", + }; + + it("recognizes only the Computer Use connector approval", () => { + NodeAssert.equal(isComputerUseMcpApproval(request), true); + NodeAssert.equal( + isComputerUseMcpApproval({ + ...request, + _meta: { ...request._meta, connector_id: "calendar" }, + }), + false, + ); + }); + + it("recognizes generic MCP tool guardian approvals without a connector id", () => { + const genericRequest = { + ...request, + _meta: { codex_approval_kind: "mcp_tool_call" as const }, + message: "Allow node_repl to run this tool call?", + serverName: "node_repl", + }; + + NodeAssert.equal(isMcpToolApproval(genericRequest), true); + NodeAssert.equal(isComputerUseMcpApproval(genericRequest), false); + NodeAssert.equal(mcpApprovalRequestKind(genericRequest), "tool"); + NodeAssert.equal(mcpApprovalRequestKind(request), "permissions"); + }); + + it("does not recognize URL or unrelated form elicitations as MCP tool approvals", () => { + NodeAssert.equal( + isMcpToolApproval({ + ...request, + mode: "url", + url: "https://example.com/approve", + elicitationId: "elicitation-1", + }), + false, + ); + const unrelatedRequest = { + ...request, + _meta: { connector_id: "computer-use" }, + }; + NodeAssert.equal(isMcpToolApproval(unrelatedRequest), false); + NodeAssert.equal(mcpApprovalRequestKind(unrelatedRequest), undefined); + NodeAssert.equal( + mcpApprovalRequestKind({ + ...request, + mode: "url", + url: "https://example.com/approve", + elicitationId: "elicitation-1", + }), + undefined, + ); + }); + + it("maps approval decisions to MCP actions and session persistence", () => { + NodeAssert.deepStrictEqual(buildMcpApprovalResponse("accept"), { action: "accept" }); + NodeAssert.deepStrictEqual(buildMcpApprovalResponse("acceptForSession"), { + action: "accept", + _meta: { persist: "session" }, + }); + NodeAssert.deepStrictEqual(buildMcpApprovalResponse("decline"), { action: "decline" }); + NodeAssert.deepStrictEqual(buildMcpApprovalResponse("cancel"), { action: "cancel" }); + }); +}); + describe("buildCodexDeveloperInstructions", () => { it("appends runtime info after the mode instructions", () => { const instructions = buildCodexDeveloperInstructions("default", { diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.ts index 8f887c1b5929..88516810852d 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.ts @@ -60,6 +60,15 @@ const RECOVERABLE_THREAD_RESUME_ERROR_SNIPPETS = [ "does not exist", "no rollout found", ]; +const ComputerUseMcpApprovalMeta = Schema.Struct({ + codex_approval_kind: Schema.Literal("mcp_tool_call"), + connector_id: Schema.Literal("computer-use"), +}); +const McpToolApprovalMeta = Schema.Struct({ + codex_approval_kind: Schema.Literal("mcp_tool_call"), +}); +const isComputerUseApprovalMeta = Schema.is(ComputerUseMcpApprovalMeta); +const isMcpToolApprovalMeta = Schema.is(McpToolApprovalMeta); export function hasConfiguredMcpServer(appServerArgs: ReadonlyArray | undefined): boolean { return appServerArgs?.some((argument) => argument.includes("mcp_servers.")) === true; @@ -119,6 +128,8 @@ export interface CodexSessionRuntimeSendTurnInput { readonly serviceTier?: CodexServiceTier | undefined; readonly effort?: EffectCodexSchema.V2TurnStartParams__ReasoningEffort | undefined; readonly interactionMode?: ProviderInteractionMode; + /** Preloaded Computer History context from the Effect-owned loader. */ + readonly computerHistoryContext?: string; } export interface CodexThreadTurnSnapshot { @@ -154,6 +165,50 @@ export interface CodexSessionRuntimeShape { readonly close: Effect.Effect; } +export function buildPermissionsApprovalResponse( + permissions: EffectCodexSchema.PermissionsRequestApprovalParams["permissions"], + decision: ProviderApprovalDecision, +): EffectCodexSchema.PermissionsRequestApprovalResponse { + return { + permissions: decision === "accept" || decision === "acceptForSession" ? permissions : {}, + scope: decision === "acceptForSession" ? "session" : "turn", + }; +} + +export function isComputerUseMcpApproval( + payload: EffectCodexSchema.McpServerElicitationRequestParams, +): boolean { + return payload.mode !== "url" && isComputerUseApprovalMeta(payload._meta); +} + +export function isMcpToolApproval( + payload: EffectCodexSchema.McpServerElicitationRequestParams, +): boolean { + return payload.mode !== "url" && isMcpToolApprovalMeta(payload._meta); +} + +export function mcpApprovalRequestKind( + payload: EffectCodexSchema.McpServerElicitationRequestParams, +): ProviderRequestKind | undefined { + if (!isMcpToolApproval(payload)) return undefined; + return isComputerUseMcpApproval(payload) ? "permissions" : "tool"; +} + +export function buildMcpApprovalResponse( + decision: ProviderApprovalDecision, +): EffectCodexSchema.McpServerElicitationRequestResponse { + switch (decision) { + case "accept": + return { action: "accept" }; + case "acceptForSession": + return { action: "accept", _meta: { persist: "session" } }; + case "decline": + return { action: "decline" }; + case "cancel": + return { action: "cancel" }; + } +} + export type CodexSessionRuntimeError = | CodexErrors.CodexAppServerError | CodexSessionRuntimePendingApprovalNotFoundError @@ -340,21 +395,31 @@ function buildCodexCollaborationMode(input: { readonly interactionMode?: ProviderInteractionMode; readonly model?: string; readonly effort?: EffectCodexSchema.V2TurnStartParams__ReasoningEffort; + readonly computerHistoryContext?: string; }): EffectCodexSchema.V2TurnStartParams__CollaborationMode | undefined { - if (input.interactionMode === undefined) { + // Mode-less turns still need Computer History when present — otherwise ordinary + // sendTurn calls silently drop the loaded context. + if (input.interactionMode === undefined && !input.computerHistoryContext) { return undefined; } + const interactionMode = input.interactionMode ?? "default"; const model = normalizeCodexModelSlug(input.model) ?? DEFAULT_MODEL; const reasoningEffort = input.effort ?? "medium"; return { - mode: input.interactionMode, + mode: interactionMode, settings: { model, reasoning_effort: reasoningEffort, - developer_instructions: buildCodexDeveloperInstructions(input.interactionMode, { - model, - reasoningEffort, - }), + developer_instructions: buildCodexDeveloperInstructions( + interactionMode, + { + model, + reasoningEffort, + }, + input.computerHistoryContext + ? { computerHistoryContext: input.computerHistoryContext } + : undefined, + ), }, }; } @@ -371,6 +436,7 @@ export function buildTurnStartParams(input: { readonly serviceTier?: CodexServiceTier; readonly effort?: EffectCodexSchema.V2TurnStartParams__ReasoningEffort; readonly interactionMode?: ProviderInteractionMode; + readonly computerHistoryContext?: string; }): Effect.Effect< CodexTurnStartParamsWithCollaborationMode, CodexErrors.CodexAppServerProtocolParseError @@ -391,6 +457,9 @@ export function buildTurnStartParams(input: { ...(input.interactionMode ? { interactionMode: input.interactionMode } : {}), ...(input.model ? { model: input.model } : {}), ...(input.effort ? { effort: input.effort } : {}), + ...(input.computerHistoryContext + ? { computerHistoryContext: input.computerHistoryContext } + : {}), }); return decodeCodexTurnStartParamsWithCollaborationMode({ @@ -1599,6 +1668,127 @@ export const makeCodexSessionRuntime = ( }), ); + yield* client.handleServerRequest("item/permissions/requestApproval", (payload) => + Effect.gen(function* () { + const requestId = ApprovalRequestId.make( + yield* randomUUIDv4("permissions-approval-request"), + ); + const turnId = TurnId.make(payload.turnId); + const itemId = ProviderItemId.make(payload.itemId); + const decision = yield* Deferred.make(); + + yield* Ref.update(pendingApprovalsRef, (current) => { + const next = new Map(current); + next.set(requestId, { + requestId, + jsonRpcId: payload.itemId, + requestKind: "permissions", + turnId, + itemId, + decision, + }); + return next; + }); + yield* Ref.update(approvalCorrelationsRef, (current) => { + const next = new Map(current); + next.set(payload.itemId, { + requestId, + requestKind: "permissions", + turnId, + itemId, + }); + return next; + }); + + yield* emitEvent({ + kind: "request", + threadId: options.threadId, + method: "item/permissions/requestApproval", + requestId, + requestKind: "permissions", + turnId, + itemId, + payload, + }); + + const resolved = yield* Deferred.await(decision).pipe( + Effect.ensuring( + Ref.update(pendingApprovalsRef, (current) => { + const next = new Map(current); + next.delete(requestId); + return next; + }), + ), + ); + return buildPermissionsApprovalResponse(payload.permissions, resolved); + }), + ); + + yield* client.handleServerRequest("mcpServer/elicitation/request", (payload) => + Effect.gen(function* () { + const requestKind = mcpApprovalRequestKind(payload); + if (!requestKind) { + // We advertise form elicitation support at initialize. Unrecognized + // forms (and URL elicitations) must be answered, not methodNotFound, + // or Codex can stall the turn. + return { action: "decline" } as const; + } + + const requestId = ApprovalRequestId.make(yield* randomUUIDv4("mcp-approval-request")); + const turnId = payload.turnId ? TurnId.make(payload.turnId) : undefined; + const decision = yield* Deferred.make(); + // Concurrent elicitations share a serverName; correlate by the JSON-RPC + // request id (fiber-local) so serverRequest/resolved cannot collide. + const incomingRequestId = yield* CodexClient.CurrentServerRequestId; + const correlationKey = + incomingRequestId !== undefined ? String(incomingRequestId) : String(requestId); + + yield* Ref.update(pendingApprovalsRef, (current) => { + const next = new Map(current); + next.set(requestId, { + requestId, + jsonRpcId: correlationKey, + requestKind, + turnId, + itemId: undefined, + decision, + }); + return next; + }); + yield* Ref.update(approvalCorrelationsRef, (current) => { + const next = new Map(current); + next.set(correlationKey, { + requestId, + requestKind, + turnId, + itemId: undefined, + }); + return next; + }); + + yield* emitEvent({ + kind: "request", + threadId: options.threadId, + method: "mcpServer/elicitation/request", + requestId, + requestKind, + ...(turnId ? { turnId } : {}), + payload, + }); + + const resolved = yield* Deferred.await(decision).pipe( + Effect.ensuring( + Ref.update(pendingApprovalsRef, (current) => { + const next = new Map(current); + next.delete(requestId); + return next; + }), + ), + ); + return buildMcpApprovalResponse(resolved); + }), + ); + yield* client.handleServerRequest("item/tool/requestUserInput", (payload) => Effect.gen(function* () { const requestId = ApprovalRequestId.make(yield* randomUUIDv4("user-input-request")); @@ -1813,6 +2003,7 @@ export const makeCodexSessionRuntime = ( const normalizedModel = normalizeCodexModelSlug( input.model ?? (yield* Ref.get(sessionRef)).model, ); + const computerHistoryContext = input.computerHistoryContext; const params = yield* buildTurnStartParams({ threadId: providerThreadId, runtimeMode: options.runtimeMode, @@ -1822,6 +2013,7 @@ export const makeCodexSessionRuntime = ( ...(input.serviceTier ? { serviceTier: input.serviceTier } : {}), ...(input.effort ? { effort: input.effort } : {}), ...(input.interactionMode ? { interactionMode: input.interactionMode } : {}), + ...(computerHistoryContext ? { computerHistoryContext } : {}), }); const rawResponse = yield* client.raw.request("turn/start", params); const response = yield* decodeV2TurnStartResponse(rawResponse).pipe( diff --git a/apps/server/src/provider/Layers/CursorAdapter.ts b/apps/server/src/provider/Layers/CursorAdapter.ts index 30c173d8fae8..fb3f32aee70d 100644 --- a/apps/server/src/provider/Layers/CursorAdapter.ts +++ b/apps/server/src/provider/Layers/CursorAdapter.ts @@ -42,7 +42,10 @@ import type * as EffectAcpSchema from "effect-acp/schema"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; +import * as ComputerHistoryService from "../../computerHistory/service.ts"; +import { makeResolveEnabledDesktopMcp } from "../../desktopControl/desktopMcpLaunch.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; +import * as ServerSettings from "../../serverSettings.ts"; import { ProviderAdapterProcessError, ProviderAdapterRequestError, @@ -316,10 +319,12 @@ export function makeCursorAdapter( ) { return Effect.gen(function* () { const boundInstanceId = options?.instanceId ?? ProviderInstanceId.make("cursor"); + const resolveDesktopMcp = yield* makeResolveEnabledDesktopMcp(); const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const serverConfig = yield* Effect.service(ServerConfig); + const serverSettings = yield* ServerSettings.ServerSettingsService; const crypto = yield* Crypto.Crypto; const nativeEventLogger = options?.nativeEventLogger ?? @@ -532,6 +537,37 @@ export function makeCursorAdapter( : cursorSettings; const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); + // Desktop MCP whenever the binary resolves and Computer Use is enabled. + // Re-resolve per session so Settings toggles apply without restart. + // Do not gate on the HTTP t3-code MCP session (fresh Cursor threads). + const desktopMcp = yield* resolveDesktopMcp(); + const mcpServers = [ + ...(mcpSession + ? [ + { + type: "http" as const, + name: "t3-code", + url: mcpSession.endpoint, + headers: [ + { + name: "Authorization", + value: mcpSession.authorizationHeader, + }, + ], + }, + ] + : []), + ...(desktopMcp + ? [ + { + name: "t3-desktop", + command: desktopMcp.path, + args: [] as string[], + env: [...desktopMcp.env], + }, + ] + : []), + ]; const acp = yield* makeCursorAcpRuntime({ cursorSettings: effectiveCursorSettings, ...(options?.environment ? { environment: options.environment } : {}), @@ -539,23 +575,7 @@ export function makeCursorAdapter( cwd, ...(resumeSessionId ? { resumeSessionId } : {}), clientInfo: { name: "t3-code", version: "0.0.0" }, - ...(mcpSession - ? { - mcpServers: [ - { - type: "http" as const, - name: "t3-code", - url: mcpSession.endpoint, - headers: [ - { - name: "Authorization", - value: mcpSession.authorizationHeader, - }, - ], - }, - ], - } - : {}), + ...(mcpServers.length > 0 ? { mcpServers } : {}), ...acpNativeLoggers, }).pipe( Effect.provideService(Crypto.Crypto, crypto), @@ -967,6 +987,14 @@ export function makeCursorAdapter( } const promptParts: Array = []; + const computerHistoryContext = + yield* ComputerHistoryService.loadComputerHistoryContextProvided( + serverConfig, + serverSettings, + ); + if (computerHistoryContext) { + promptParts.push({ type: "text", text: computerHistoryContext }); + } if (input.input?.trim()) { promptParts.push({ type: "text", text: input.input.trim() }); } @@ -1002,7 +1030,10 @@ export function makeCursorAdapter( } } - if (promptParts.length === 0) { + if ( + promptParts.length === 0 || + (!input.input?.trim() && (!input.attachments || input.attachments.length === 0)) + ) { return yield* new ProviderAdapterValidationError({ provider: PROVIDER, operation: "sendTurn", diff --git a/apps/server/src/provider/Layers/GrokAdapter.test.ts b/apps/server/src/provider/Layers/GrokAdapter.test.ts index 6cb71660a74c..e5abfa3aa69e 100644 --- a/apps/server/src/provider/Layers/GrokAdapter.test.ts +++ b/apps/server/src/provider/Layers/GrokAdapter.test.ts @@ -26,6 +26,7 @@ import { } from "@t3tools/contracts"; import { ServerConfig } from "../../config.ts"; +import { ServerSettingsService } from "../../serverSettings.ts"; import { grokPromptSettlementBelongsToContext, makeGrokAdapter } from "./GrokAdapter.ts"; const decodeGrokSettings = Schema.decodeSync(GrokSettings); @@ -84,7 +85,10 @@ async function readJsonLines(filePath: string) { const grokAdapterTestLayer = ServerConfig.layerTest(process.cwd(), { prefix: "t3code-grok-adapter-test-", -}).pipe(Layer.provideMerge(NodeServices.layer)); +}).pipe( + Layer.provideMerge(ServerSettingsService.layerTest()), + Layer.provideMerge(NodeServices.layer), +); const makeTestAdapter = (binaryPath: string, options?: Parameters[1]) => makeGrokAdapter(decodeGrokSettings({ binaryPath }), options).pipe(Effect.orDie); diff --git a/apps/server/src/provider/Layers/GrokAdapter.ts b/apps/server/src/provider/Layers/GrokAdapter.ts index 858d862e6d5f..7a71db342474 100644 --- a/apps/server/src/provider/Layers/GrokAdapter.ts +++ b/apps/server/src/provider/Layers/GrokAdapter.ts @@ -34,7 +34,10 @@ import type * as EffectAcpSchema from "effect-acp/schema"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; +import * as ComputerHistoryService from "../../computerHistory/service.ts"; +import { makeResolveEnabledDesktopMcp } from "../../desktopControl/desktopMcpLaunch.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; +import * as ServerSettings from "../../serverSettings.ts"; import { ProviderAdapterProcessError, ProviderAdapterRequestError, @@ -227,10 +230,12 @@ export function grokPromptSettlementBelongsToContext(input: { export function makeGrokAdapter(grokSettings: GrokSettings, options?: GrokAdapterLiveOptions) { return Effect.gen(function* () { const boundInstanceId = options?.instanceId ?? ProviderInstanceId.make("grok"); + const resolveDesktopMcp = yield* makeResolveEnabledDesktopMcp(); const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const serverConfig = yield* Effect.service(ServerConfig); + const serverSettings = yield* ServerSettings.ServerSettingsService; const crypto = yield* Crypto.Crypto; const nativeEventLogger = options?.nativeEventLogger ?? @@ -570,6 +575,37 @@ export function makeGrokAdapter(grokSettings: GrokSettings, options?: GrokAdapte }); const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); + // Desktop MCP whenever the binary resolves and Computer Use is enabled. + // Re-resolve per session so Settings toggles apply without restart. + // Do not gate on the HTTP t3-code MCP session (fresh Grok threads). + const desktopMcp = yield* resolveDesktopMcp(); + const mcpServers = [ + ...(mcpSession + ? [ + { + type: "http" as const, + name: "t3-code", + url: mcpSession.endpoint, + headers: [ + { + name: "Authorization", + value: mcpSession.authorizationHeader, + }, + ], + }, + ] + : []), + ...(desktopMcp + ? [ + { + name: "t3-desktop", + command: desktopMcp.path, + args: [] as string[], + env: [...desktopMcp.env], + }, + ] + : []), + ]; const acp = yield* makeGrokAcpRuntime({ grokSettings, ...(options?.environment ? { environment: options.environment } : {}), @@ -577,23 +613,7 @@ export function makeGrokAdapter(grokSettings: GrokSettings, options?: GrokAdapte cwd, ...(resumeSessionId ? { resumeSessionId } : {}), clientInfo: { name: "t3-code", version: "0.0.0" }, - ...(mcpSession - ? { - mcpServers: [ - { - type: "http" as const, - name: "t3-code", - url: mcpSession.endpoint, - headers: [ - { - name: "Authorization", - value: mcpSession.authorizationHeader, - }, - ], - }, - ], - } - : {}), + ...(mcpServers.length > 0 ? { mcpServers } : {}), ...acpNativeLoggers, }).pipe( Effect.provideService(Crypto.Crypto, crypto), @@ -957,6 +977,11 @@ export function makeGrokAdapter(grokSettings: GrokSettings, options?: GrokAdapte }); const text = input.input?.trim(); + const computerHistoryContext = + yield* ComputerHistoryService.loadComputerHistoryContextProvided( + serverConfig, + serverSettings, + ); const imagePromptParts = yield* Effect.forEach( input.attachments ?? [], (attachment) => @@ -991,11 +1016,14 @@ export function makeGrokAdapter(grokSettings: GrokSettings, options?: GrokAdapte }), ); const promptParts: Array = [ + ...(computerHistoryContext + ? [{ type: "text" as const, text: computerHistoryContext }] + : []), ...(text ? [{ type: "text" as const, text }] : []), ...imagePromptParts, ]; - if (promptParts.length === 0) { + if (promptParts.length === 0 || (!text && imagePromptParts.length === 0)) { return yield* new ProviderAdapterValidationError({ provider: PROVIDER, operation: "sendTurn", diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 42fc93624668..fd29e5c1be67 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -9,6 +9,7 @@ import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; import * as BackgroundPolicy from "./background/BackgroundPolicy.ts"; import * as HostPowerMonitor from "./background/HostPowerMonitor.ts"; +import * as ComputerHistoryService from "./computerHistory/service.ts"; import * as ServerConfig from "./config.ts"; import { otlpTracesProxyRouteLayer, @@ -162,6 +163,9 @@ const HostPowerMonitorLayerLive = HostPowerMonitor.layer.pipe( const BackgroundLayerLive = BackgroundPolicy.layer.pipe( Layer.provide(HostPowerMonitorLayerLive), Layer.provideMerge(ServerSettingsLayerLive), + Layer.provideMerge( + ComputerHistoryService.layer.pipe(Layer.provideMerge(ServerSettingsLayerLive)), + ), ); const UsageLayerLive = UsageService.layer.pipe(Layer.provide(ServerSettingsLayerLive)); diff --git a/apps/web/src/assets/computer-use/agent-cursor-badge.png b/apps/web/src/assets/computer-use/agent-cursor-badge.png new file mode 100644 index 000000000000..e972c52d9b27 Binary files /dev/null and b/apps/web/src/assets/computer-use/agent-cursor-badge.png differ diff --git a/apps/web/src/assets/computer-use/agent-cursor-icon.png b/apps/web/src/assets/computer-use/agent-cursor-icon.png new file mode 100644 index 000000000000..e972c52d9b27 Binary files /dev/null and b/apps/web/src/assets/computer-use/agent-cursor-icon.png differ diff --git a/apps/web/src/assets/computer-use/brave.svg b/apps/web/src/assets/computer-use/brave.svg new file mode 100644 index 000000000000..e192b2d839aa --- /dev/null +++ b/apps/web/src/assets/computer-use/brave.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/apps/web/src/assets/computer-use/edge.svg b/apps/web/src/assets/computer-use/edge.svg new file mode 100644 index 000000000000..82159a6adbcf --- /dev/null +++ b/apps/web/src/assets/computer-use/edge.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/apps/web/src/assets/computer-use/firefox.svg b/apps/web/src/assets/computer-use/firefox.svg new file mode 100644 index 000000000000..6c2333f51482 --- /dev/null +++ b/apps/web/src/assets/computer-use/firefox.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/apps/web/src/assets/computer-use/google-chrome.svg b/apps/web/src/assets/computer-use/google-chrome.svg new file mode 100644 index 000000000000..e053d777e946 --- /dev/null +++ b/apps/web/src/assets/computer-use/google-chrome.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/apps/web/src/components/chat/ComposerPendingApprovalPanel.test.tsx b/apps/web/src/components/chat/ComposerPendingApprovalPanel.test.tsx index de67505a55e4..8cdee28b46ed 100644 --- a/apps/web/src/components/chat/ComposerPendingApprovalPanel.test.tsx +++ b/apps/web/src/components/chat/ComposerPendingApprovalPanel.test.tsx @@ -28,4 +28,39 @@ describe("ComposerPendingApprovalPanel", () => { expect(markup).toContain("max-w-full"); expect(markup).toContain("[overflow-wrap:anywhere]"); }); + + it("labels permission requests clearly", () => { + const markup = renderToStaticMarkup( + , + ); + + expect(markup).toContain("Permission requested"); + expect(markup).toContain('aria-label="Requested permissions"'); + }); + + it("distinguishes generic tool approvals from permission requests", () => { + const markup = renderToStaticMarkup( + , + ); + + expect(markup).toContain("Tool approval requested"); + expect(markup).toContain('aria-label="Tool request"'); + expect(markup).not.toContain("Permission requested"); + }); }); diff --git a/apps/web/src/components/chat/ComposerPendingApprovalPanel.tsx b/apps/web/src/components/chat/ComposerPendingApprovalPanel.tsx index 391546355b2c..9bb57867b12b 100644 --- a/apps/web/src/components/chat/ComposerPendingApprovalPanel.tsx +++ b/apps/web/src/components/chat/ComposerPendingApprovalPanel.tsx @@ -15,13 +15,21 @@ export const ComposerPendingApprovalPanel = memo(function ComposerPendingApprova ? "Command approval requested" : approval.requestKind === "file-read" ? "File-read approval requested" - : "File-change approval requested"; + : approval.requestKind === "file-change" + ? "File-change approval requested" + : approval.requestKind === "tool" + ? "Tool approval requested" + : "Permission requested"; const detailLabel = approval.requestKind === "command" ? "Command" : approval.requestKind === "file-read" ? "File to read" - : "File change"; + : approval.requestKind === "file-change" + ? "File change" + : approval.requestKind === "tool" + ? "Tool request" + : "Requested permissions"; return (
diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index c90aa771f8d1..d9088d045f95 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -58,6 +58,7 @@ import { MousePointerClickIcon, PaintbrushIcon, MinusIcon, + ShieldIcon, SquarePenIcon, TerminalIcon, Undo2Icon, @@ -1947,6 +1948,7 @@ type WorkEntryIconName = | "globe" | "hammer" | "message-circle" + | "shield" | "square-pen" | "terminal" | "wrench" @@ -1969,6 +1971,8 @@ function WorkEntryIconSvg({ name, className }: { name: WorkEntryIconName; classN return ; case "message-circle": return ; + case "shield": + return ; case "square-pen": return ; case "terminal": @@ -2076,6 +2080,8 @@ function workEntryIconName(workEntry: TimelineWorkEntry): WorkEntryIconName { if (workEntry.requestKind === "command") return "terminal"; if (workEntry.requestKind === "file-read") return "eye"; if (workEntry.requestKind === "file-change") return "square-pen"; + if (workEntry.requestKind === "tool") return "wrench"; + if (workEntry.requestKind === "permissions") return "shield"; if (workEntry.itemType === "command_execution" || workEntry.command) { return "terminal"; diff --git a/apps/web/src/components/settings/ComputerHistorySettings.tsx b/apps/web/src/components/settings/ComputerHistorySettings.tsx new file mode 100644 index 000000000000..e198ad3cff00 --- /dev/null +++ b/apps/web/src/components/settings/ComputerHistorySettings.tsx @@ -0,0 +1,596 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { PlusIcon, XIcon } from "lucide-react"; +import type { + ComputerHistoryClearScope, + ComputerHistoryStatus, + ComputerHistoryTimelineItem, +} from "@t3tools/contracts"; +import { DEFAULT_UNIFIED_SETTINGS } from "@t3tools/contracts/settings"; + +import { usePrimarySettings, useUpdatePrimarySettings } from "../../hooks/useSettings"; +import { + AlertDialog, + AlertDialogClose, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogPopup, + AlertDialogTitle, +} from "../ui/alert-dialog"; +import { Button } from "../ui/button"; +import { + Dialog, + DialogClose, + DialogDescription, + DialogFooter, + DialogHeader, + DialogPanel, + DialogPopup, + DialogTitle, +} from "../ui/dialog"; +import { Input } from "../ui/input"; +import { Switch } from "../ui/switch"; +import { + SettingIconAction, + SettingResetButton, + SettingRowTitle, + SettingsPageContainer, + SettingsRow, + SettingsSection, +} from "./settingsLayout"; +import { searchableSetting } from "./settingsSearch"; + +function isDesktopHost(): boolean { + return typeof window !== "undefined" && window.desktopBridge !== undefined; +} + +function normalizeEntry(value: string): string { + return value.trim(); +} + +function PrivacyExclusionList({ + title, + addLabel, + placeholder, + items, + onChange, + disabled, +}: { + title: string; + addLabel: string; + placeholder: string; + items: ReadonlyArray; + onChange: (next: string[]) => void; + disabled?: boolean; +}) { + const [draft, setDraft] = useState(""); + const [adding, setAdding] = useState(false); + + const commit = () => { + const next = normalizeEntry(draft); + if (!next) { + setAdding(false); + setDraft(""); + return; + } + const lowered = next.toLowerCase(); + if (items.some((item) => item.toLowerCase() === lowered)) { + setDraft(""); + setAdding(false); + return; + } + onChange([...items, next]); + setDraft(""); + setAdding(false); + }; + + return ( +
+

{title}

+
+ {adding ? ( +
{ + event.preventDefault(); + commit(); + }} + > + setDraft(value)} + onKeyDown={(event) => { + if (event.key === "Escape") { + event.preventDefault(); + event.stopPropagation(); + setAdding(false); + setDraft(""); + } + }} + /> + +
+ ) : ( + + )} + + {items.length === 0 ? ( +

None excluded

+ ) : ( +
    + {items.map((item) => ( +
  • +
    + {item} + onChange(items.filter((entry) => entry !== item))} + > + + +
    +
  • + ))} +
+ )} +
+
+ ); +} + +function ComputerHistoryPrivacyDialog({ + open, + onOpenChange, + apps, + websites, + disabled, + onSave, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; + apps: ReadonlyArray; + websites: ReadonlyArray; + disabled?: boolean; + onSave: (next: { apps: string[]; websites: string[] }) => void; +}) { + const [draftApps, setDraftApps] = useState([...apps]); + const [draftWebsites, setDraftWebsites] = useState([...websites]); + + useEffect(() => { + if (!open) return; + setDraftApps([...apps]); + setDraftWebsites([...websites]); + }, [open, apps, websites]); + + return ( + + + + Computer History privacy + + Choose apps and websites that should never be recorded. Everything else is included when + Computer History is on. + + + +
+ + +
+

+ Private-mode web browsing activity is never included in computer history. +

+
+ + }>Cancel + + +
+
+ ); +} + +export function ComputerHistorySettings() { + const settings = usePrimarySettings(); + const updateSettings = useUpdatePrimarySettings(); + const history = settings.computerHistory; + const defaults = DEFAULT_UNIFIED_SETTINGS.computerHistory; + const onDesktop = isDesktopHost(); + const [status, setStatus] = useState(null); + const [items, setItems] = useState([]); + const [busy, setBusy] = useState(false); + const [privacyOpen, setPrivacyOpen] = useState(false); + const [pendingClearScope, setPendingClearScope] = useState( + null, + ); + const [pendingDeletePath, setPendingDeletePath] = useState(null); + + const refresh = useCallback(async () => { + const bridge = window.desktopBridge; + if (!bridge?.getComputerHistoryStatus || !bridge.getComputerHistoryTimeline) return; + const [nextStatus, timeline] = await Promise.all([ + bridge.getComputerHistoryStatus(), + bridge.getComputerHistoryTimeline(), + ]); + setStatus(nextStatus); + setItems([...timeline.items]); + }, []); + + useEffect(() => { + void refresh(); + const id = window.setInterval(() => void refresh(), 5_000); + return () => window.clearInterval(id); + }, [refresh, history.enabled, history.paused]); + + const patchQueue = useRef(Promise.resolve()); + const patch = (partial: Partial) => { + // Persist only the keys being changed so concurrent toggles cannot clobber + // unrelated nested Computer History fields via a stale full-object snapshot. + // Serialize desktop IPC patches so rapid toggles cannot complete out of order. + updateSettings({ + computerHistory: partial, + }); + const bridge = window.desktopBridge; + if (!bridge?.patchComputerHistorySettings) return; + patchQueue.current = patchQueue.current + .catch(() => undefined) + .then(async () => { + const nextStatus = await bridge.patchComputerHistorySettings({ + ...(partial.enabled === undefined ? {} : { enabled: partial.enabled }), + ...(partial.paused === undefined ? {} : { paused: partial.paused }), + ...(partial.mirrorToCodex === undefined ? {} : { mirrorToCodex: partial.mirrorToCodex }), + ...(partial.appFilterMode === undefined ? {} : { appFilterMode: partial.appFilterMode }), + ...(partial.apps === undefined ? {} : { apps: [...partial.apps] }), + ...(partial.websiteFilterMode === undefined + ? {} + : { websiteFilterMode: partial.websiteFilterMode }), + ...(partial.websites === undefined ? {} : { websites: [...partial.websites] }), + }); + setStatus(nextStatus); + const timeline = await bridge.getComputerHistoryTimeline?.(); + if (timeline) setItems([...timeline.items]); + }); + }; + + const hasPrivacyOverrides = + history.apps.length > 0 || + history.websites.length > 0 || + history.appFilterMode !== defaults.appFilterMode || + history.websiteFilterMode !== defaults.websiteFilterMode; + + const exclusionSummary = (() => { + const appCount = history.apps.length; + const siteCount = history.websites.length; + if (appCount === 0 && siteCount === 0) return "No app or website filters"; + const describe = (count: number, noun: string, mode: "exclude" | "includeOnly") => { + const label = `${count} ${noun}${count === 1 ? "" : "s"}`; + return mode === "includeOnly" ? `only ${label}` : `excluding ${label}`; + }; + const parts: string[] = []; + if (appCount > 0) { + parts.push(describe(appCount, "app", history.appFilterMode)); + } + if (siteCount > 0) { + parts.push(describe(siteCount, "website", history.websiteFilterMode)); + } + return parts.join(" · "); + })(); + + return ( + + +

+ Opt-in activity timeline from accessibility events (not screenshots). Summaries become + local memories agents can reference. Requires Accessibility (macOS), UI Automation + (Windows), or AT-SPI (Linux). +

+ {!onDesktop ? ( +

+ Computer History recording runs in the T3 Code desktop app. +

+ ) : null} + + patch({ enabled: defaults.enabled })} + /> + ) : null + } + control={ + patch({ enabled: Boolean(checked) })} + aria-label="Enable Computer History" + /> + } + /> + + Paused} + description="Stop collecting new events without turning the feature off" + control={ + patch({ paused: Boolean(checked) })} + aria-label="Pause Computer History" + /> + } + /> + + Mirror to Codex skysight} + description="Also write memories under ~/.codex/memories/extensions/skysight/" + control={ + patch({ mirrorToCodex: Boolean(checked) })} + aria-label="Mirror Computer History to Codex" + /> + } + /> + + {status ? ( + Recorder status} + description={ + status.lastError + ? status.lastError + : `Phase: ${status.phase} · events in segment: ${status.eventCount} · platform: ${status.platform}` + } + control={ + + } + /> + ) : null} +
+ + +

+ Exclude sensitive apps and websites from Computer History. Private browsing is never + recorded. +

+ Excluded apps & websites} + description={exclusionSummary} + resetAction={ + hasPrivacyOverrides ? ( + + patch({ + apps: [...defaults.apps], + websites: [...defaults.websites], + appFilterMode: defaults.appFilterMode, + websiteFilterMode: defaults.websiteFilterMode, + }) + } + /> + ) : null + } + control={ + + } + /> +
+ + +

+ Summaries from the local event stream. Clearing deletes events and derived memories. +

+
+ {( + [ + ["last_ten_minutes", "Clear 10 min"], + ["last_hour", "Clear hour"], + ["last_day", "Clear day"], + ["all", "Clear all"], + ] as const satisfies ReadonlyArray + ).map(([scope, label]) => ( + + ))} +
+ {items.length === 0 ? ( +

No summaries yet.

+ ) : ( +
    + {items.map((item) => ( +
  • +
    +
    +
    {item.title}
    +
    + {item.level} · {new Date(item.startedAt).toLocaleString()} + {item.applications.length > 0 + ? ` · ${item.applications.slice(0, 3).join(", ")}` + : ""} +
    +

    {item.description}

    + {item.suggestion ? ( +

    + Suggested {item.suggestion.type}: {item.suggestion.name} +

    + ) : null} +
    +
    + + +
    +
    +
  • + ))} +
+ )} +
+ + + // Preserve existing filter modes — Done only edits the lists. + patch({ apps, websites }) + } + /> + + { + if (!open) setPendingClearScope(null); + }} + > + + + Clear Computer History? + + This permanently deletes recorded events and derived memories for the selected range. + + + + }> + Cancel + + + + + + + { + if (!open) setPendingDeletePath(null); + }} + > + + + Delete this memory? + + This permanently removes the selected Computer History memory from disk. + + + + }> + Cancel + + + + + +
+ ); +} diff --git a/apps/web/src/components/settings/ComputerUseMoreBrowsers.tsx b/apps/web/src/components/settings/ComputerUseMoreBrowsers.tsx new file mode 100644 index 000000000000..ee434b50e7ae --- /dev/null +++ b/apps/web/src/components/settings/ComputerUseMoreBrowsers.tsx @@ -0,0 +1,60 @@ +import { useState } from "react"; + +import { Button } from "../ui/button"; +import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from "../ui/collapsible"; +import { BraveIcon, EdgeIcon, FirefoxIcon } from "./browserBrandIcons"; +import { SettingRowTitle, SettingsRow } from "./settingsLayout"; + +/** Chromium/Firefox setup disclosure for Computer Use settings. */ +export function ComputerUseMoreBrowsers() { + const [open, setOpen] = useState(false); + + return ( + + }>More browsers + } + description="Set up the same extension in other Chromium browsers." + control={ + } + aria-controls="computer-use-more-browsers" + > + {open ? "Hide" : "Show"} + + } + > + +
+
+ +
+

Microsoft Edge

+

+ Load the unpacked extension from edge://extensions +

+
+
+
+ +
+

Brave

+

+ Same extension via brave://extensions +

+
+
+
+ +
+

Firefox

+

Not supported yet

+
+
+
+
+
+
+ ); +} diff --git a/apps/web/src/components/settings/ComputerUseSettings.tsx b/apps/web/src/components/settings/ComputerUseSettings.tsx new file mode 100644 index 000000000000..1bfb45fa58de --- /dev/null +++ b/apps/web/src/components/settings/ComputerUseSettings.tsx @@ -0,0 +1,465 @@ +import { MonitorIcon } from "lucide-react"; +import { type ReactNode, useCallback, useEffect, useState } from "react"; +import type { + DesktopComputerUsePermission, + DesktopComputerUsePermissionsState, + DesktopComputerUsePrivacyPane, +} from "@t3tools/contracts"; +import { DEFAULT_UNIFIED_SETTINGS } from "@t3tools/contracts/settings"; + +import { usePrimarySettings, useUpdatePrimarySettings } from "../../hooks/useSettings"; +import { cn } from "../../lib/utils"; +import { Button } from "../ui/button"; +import { + Dialog, + DialogClose, + DialogDescription, + DialogFooter, + DialogHeader, + DialogPanel, + DialogPopup, + DialogTitle, +} from "../ui/dialog"; +import { Switch } from "../ui/switch"; +import { AgentCursorIcon, ChromeIcon } from "./browserBrandIcons"; +import { ComputerUseMoreBrowsers } from "./ComputerUseMoreBrowsers"; +import { + SettingResetButton, + SettingRowTitle, + SettingsPageContainer, + SettingsRow, + SettingsSection, +} from "./settingsLayout"; +import { searchableSetting } from "./settingsSearch"; + +/** Shown when the desktop host lacks the Computer Use permissions bridge API. */ +const BRIDGE_UNSUPPORTED_MESSAGE = "Update T3 Code to check Computer Use permissions"; + +function isDesktopHost(): boolean { + return typeof window !== "undefined" && window.desktopBridge !== undefined; +} + +function isBridgeSupported(): boolean { + return ( + typeof window !== "undefined" && window.desktopBridge?.getComputerUsePermissions !== undefined + ); +} + +function ExtensionStatus({ + tone, + children, +}: { + tone: "ok" | "warn" | "muted"; + children: ReactNode; +}) { + return ( + + + {children} + + ); +} + +function permissionTone(status: DesktopComputerUsePermission["status"]): "ok" | "warn" | "muted" { + if (status === "granted" || status === "notRequired") return "ok"; + if (status === "denied" || status === "notDetermined") return "warn"; + return "muted"; +} + +function permissionLabel(status: DesktopComputerUsePermission["status"]): string { + switch (status) { + case "granted": + return "Granted"; + case "denied": + return "Not granted"; + case "notDetermined": + return "Not determined"; + case "notRequired": + return "Not required on this platform"; + case "unknown": + return "Unknown"; + } +} + +export function ComputerUseSettings() { + const settings = usePrimarySettings(); + const updateSettings = useUpdatePrimarySettings(); + const desktop = settings.desktopControl; + const defaults = DEFAULT_UNIFIED_SETTINGS.desktopControl; + const onDesktop = isDesktopHost(); + const [manageOpen, setManageOpen] = useState(false); + const [permissionPrompt, setPermissionPrompt] = useState( + null, + ); + const [permState, setPermState] = useState(null); + const [permError, setPermError] = useState(null); + + const refreshPermissions = useCallback(async () => { + const bridge = window.desktopBridge; + if (!bridge?.getComputerUsePermissions) { + setPermState(null); + setPermError(BRIDGE_UNSUPPORTED_MESSAGE); + return; + } + try { + const next = await bridge.getComputerUsePermissions(); + setPermState(next); + setPermError(null); + } catch (error) { + setPermError(error instanceof Error ? error.message : "Could not read permissions"); + } + }, []); + + useEffect(() => { + if (!onDesktop) return; + const bridgeSupported = isBridgeSupported(); + if (!bridgeSupported) { + setPermError(BRIDGE_UNSUPPORTED_MESSAGE); + return; + } + void refreshPermissions(); + const onFocus = () => void refreshPermissions(); + window.addEventListener("focus", onFocus); + const id = window.setInterval(() => void refreshPermissions(), 4000); + return () => { + window.removeEventListener("focus", onFocus); + window.clearInterval(id); + }; + }, [onDesktop, refreshPermissions]); + + const openPrivacyPane = async (pane: DesktopComputerUsePrivacyPane): Promise => { + const bridge = window.desktopBridge; + if (!bridge?.openComputerUsePrivacySettings) { + setPermError(BRIDGE_UNSUPPORTED_MESSAGE); + return false; + } + try { + const opened = await bridge.openComputerUsePrivacySettings(pane); + if (!opened) { + setPermError("Could not open System Settings. Open Privacy & Security manually."); + return false; + } + window.setTimeout(() => void refreshPermissions(), 1500); + return true; + } catch { + setPermError("Could not open System Settings. Open Privacy & Security manually."); + return false; + } + }; + + const chromeStatus = permState?.chromeExtension; + const needsMacPrivacy = permState?.platform === "darwin"; + const bridgeSupported = onDesktop && isBridgeSupported(); + + return ( + + + {!onDesktop ? ( +
+ +

+ You are connected to a remote environment. Computer Use settings apply on the host + running the T3 Code desktop app. +

+
+ ) : null} + + + updateSettings({ + desktopControl: { ...desktop, enabled: defaults.enabled }, + }) + } + /> + ) : null + } + control={ + + updateSettings({ + desktopControl: { ...desktop, enabled: Boolean(checked) }, + }) + } + aria-label="Enable Computer Use" + /> + } + /> + + }> + {searchableSetting("computer-use-agent-cursor").title} + + } + description="Show the agent pointer overlay while it works, without moving your mouse." + resetAction={ + desktop.agentCursorEnabled !== defaults.agentCursorEnabled ? ( + + updateSettings({ + desktopControl: { + ...desktop, + agentCursorEnabled: defaults.agentCursorEnabled, + }, + }) + } + /> + ) : null + } + control={ + + updateSettings({ + desktopControl: { + ...desktop, + agentCursorEnabled: Boolean(checked), + }, + }) + } + aria-label="Show agent cursor overlay" + /> + } + /> + + }> + {searchableSetting("computer-use-browser").title} + + } + description="Drive an agent-owned tab group in your signed-in Chrome." + status={ + chromeStatus ? ( + + {chromeStatus.detail} + + ) : onDesktop ? ( + + {bridgeSupported ? "Checking extension…" : BRIDGE_UNSUPPORTED_MESSAGE} + + ) : null + } + resetAction={ + desktop.browserControlEnabled !== defaults.browserControlEnabled ? ( + + updateSettings({ + desktopControl: { + ...desktop, + browserControlEnabled: defaults.browserControlEnabled, + }, + }) + } + /> + ) : null + } + control={ +
+ + + updateSettings({ + desktopControl: { + ...desktop, + browserControlEnabled: Boolean(checked), + }, + }) + } + aria-label="Enable Google Chrome browser control" + /> +
+ } + /> + + +
+ + {onDesktop ? ( + + {(permState?.permissions ?? []).map((permission) => { + const canOpen = + needsMacPrivacy && + permission.status !== "granted" && + permission.status !== "notRequired"; + const search = + permission.kind === "accessibility" + ? searchableSetting("computer-use-accessibility") + : searchableSetting("computer-use-screen-recording"); + return ( + + {permissionLabel(permission.status)} + + } + control={ + canOpen ? ( + + ) : null + } + /> + ); + })} + {permError ?

{permError}

: null} + {!permState && !permError ? ( +

Checking permissions…

+ ) : null} + {needsMacPrivacy ? ( +

+ After enabling a permission in System Settings, return here — status refreshes + automatically. +

+ ) : null} +
+ ) : null} + + + + + Google Chrome + + Load the T3 Code extension so agents can open and drive tabs in a labelled group + without taking over your browsing. + + + +
    +
  1. + Open chrome://extensions in + Chrome and turn on Developer mode. +
  2. +
  3. + Choose Load unpacked and select + the{" "} + + native/t3-chrome-extension + {" "} + folder from this repo (or the copy bundled with the desktop app). +
  4. +
  5. + Confirm the extension id is{" "} + + kgdolgnijopbghhomnblabjkmjhnoage + + . +
  6. +
+ {chromeStatus ? ( + + {chromeStatus.detail} + + ) : null} +
+ + }>Close + +
+
+ + { + if (!open) setPermissionPrompt(null); + }} + > + + + Allow {permissionPrompt?.label ?? "permission"} + + T3 Code needs this macOS privacy permission for Computer Use. System Settings will + open to the right Privacy & Security list — turn on the switch for T3 Code. + + + + {permissionPrompt?.kind === "accessibility" ? ( +

+ Accessibility lets the agent click and type in other apps without stealing your + mouse. +

+ ) : ( +

Screen Recording lets the agent take screenshots of apps and displays.

+ )} +
+ + }>Cancel + + +
+
+
+ ); +} diff --git a/apps/web/src/components/settings/SettingsSidebarNav.tsx b/apps/web/src/components/settings/SettingsSidebarNav.tsx index 174c9e9fe97c..be0caa6ce1f5 100644 --- a/apps/web/src/components/settings/SettingsSidebarNav.tsx +++ b/apps/web/src/components/settings/SettingsSidebarNav.tsx @@ -12,8 +12,10 @@ import { ArrowLeftIcon, BotIcon, GitBranchIcon, + HistoryIcon, KeyboardIcon, Link2Icon, + MonitorIcon, PaletteIcon, SearchIcon, Settings2Icon, @@ -51,6 +53,8 @@ const SETTINGS_SECTION_ICONS: Readonly< "/settings/providers": BotIcon, "/settings/source-control": GitBranchIcon, "/settings/connections": Link2Icon, + "/settings/computer-use": MonitorIcon, + "/settings/computer-history": HistoryIcon, "/settings/archived": ArchiveIcon, }; diff --git a/apps/web/src/components/settings/browserBrandIcons.tsx b/apps/web/src/components/settings/browserBrandIcons.tsx new file mode 100644 index 000000000000..4bf0a3166526 --- /dev/null +++ b/apps/web/src/components/settings/browserBrandIcons.tsx @@ -0,0 +1,47 @@ +import type { ImgHTMLAttributes } from "react"; + +import { cn } from "../../lib/utils"; +import agentCursorUrl from "../../assets/computer-use/agent-cursor-badge.png"; +import braveUrl from "../../assets/computer-use/brave.svg"; +import chromeUrl from "../../assets/computer-use/google-chrome.svg"; +import edgeUrl from "../../assets/computer-use/edge.svg"; +import firefoxUrl from "../../assets/computer-use/firefox.svg"; + +type BrandIconProps = Omit, "src" | "alt"> & { + alt?: string; +}; + +function BrandImg({ src, alt = "", className, ...props }: BrandIconProps & { src: string }) { + return ( + {alt} + ); +} + +export function ChromeIcon({ alt = "", ...props }: BrandIconProps) { + return ; +} + +export function EdgeIcon({ alt = "", ...props }: BrandIconProps) { + return ; +} + +export function BraveIcon({ alt = "", ...props }: BrandIconProps) { + return ; +} + +export function FirefoxIcon({ alt = "", ...props }: BrandIconProps) { + return ; +} + +/** Purple rounded badge with soft radial glow fade (no nested square crop). */ +export function AgentCursorIcon({ alt = "", className, ...props }: BrandIconProps) { + return ( + + ); +} diff --git a/apps/web/src/components/settings/settingsLayout.tsx b/apps/web/src/components/settings/settingsLayout.tsx index 880e03e54754..af9d3e5118c7 100644 --- a/apps/web/src/components/settings/settingsLayout.tsx +++ b/apps/web/src/components/settings/settingsLayout.tsx @@ -83,6 +83,24 @@ function useSettingsSearchTarget(id: string | undefined) return targetRef; } +/** Shared micro icon control used by settings reset/remove/info actions. */ +export function SettingIconAction({ + className, + ...props +}: ComponentPropsWithoutRef) { + return + } /> @@ -211,9 +229,7 @@ export function SettingResetButton({ { @@ -222,7 +238,7 @@ export function SettingResetButton({ }} > - + } /> Reset to default @@ -230,6 +246,16 @@ export function SettingResetButton({ ); } +/** Shared micro icon control used by settings reset/remove actions. */ +export function SettingIconAction({ + className, + ...props +}: ComponentPropsWithoutRef) { + return ( +