From bbc9586d5c3d77e6a7fe72958d8ecb1dab6cd2c4 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Thu, 24 Sep 2026 06:44:33 -0600 Subject: [PATCH 1/2] feat(mobile): preview agent devices with scoped stream access --- apps/mobile/metro.config.js | 29 +- .../mobile/scripts/generate-device-stream.mts | 39 + apps/mobile/src/Stack.tsx | 11 + apps/mobile/src/components/AppSymbol.tsx | 5 + .../devices/DevicePreviewRouteScreen.tsx | 346 ++++++++ .../features/devices/DeviceStreamWebView.tsx | 226 +++++ .../devices/device-preview-button.tsx | 41 + .../devices/device-stream-document.test.ts | 127 +++ .../devices/device-stream-document.ts | 82 ++ .../devices/device-stream.browser.test.ts | 117 +++ .../features/devices/device-stream.browser.ts | 189 +++++ .../devices/deviceHubAccessScope.test.ts | 86 ++ .../features/devices/deviceHubAccessScope.ts | 51 ++ .../devices/threadDevicePreviews.test.ts | 106 +++ .../features/devices/threadDevicePreviews.ts | 28 + .../features/threads/ThreadDetailScreen.tsx | 22 + .../threads/floating-working-control.tsx | 52 +- apps/mobile/src/state/device.ts | 107 +++ .../src/types/mobile-device-stream.d.ts | 4 + .../components/device/DeviceStreamView.tsx | 2 +- .../components/device/deviceStream.test.ts | 2 +- .../preview/ThreadPreviewMiniPlayer.tsx | 2 +- .../preview/previewMiniPlayerLayout.ts | 2 +- docs/user/devices.md | 19 +- knip.jsonc | 9 +- packages/client-runtime/package.json | 8 + .../client-runtime/src/device/hubAccess.ts | 18 + .../client-runtime/src/device/stream.test.ts | 773 ++++++++++++++++++ .../client-runtime/src/device/stream.ts | 12 +- .../src/state/deviceHubAccess.ts | 21 +- 30 files changed, 2487 insertions(+), 49 deletions(-) create mode 100644 apps/mobile/scripts/generate-device-stream.mts create mode 100644 apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx create mode 100644 apps/mobile/src/features/devices/DeviceStreamWebView.tsx create mode 100644 apps/mobile/src/features/devices/device-preview-button.tsx create mode 100644 apps/mobile/src/features/devices/device-stream-document.test.ts create mode 100644 apps/mobile/src/features/devices/device-stream-document.ts create mode 100644 apps/mobile/src/features/devices/device-stream.browser.test.ts create mode 100644 apps/mobile/src/features/devices/device-stream.browser.ts create mode 100644 apps/mobile/src/features/devices/deviceHubAccessScope.test.ts create mode 100644 apps/mobile/src/features/devices/deviceHubAccessScope.ts create mode 100644 apps/mobile/src/features/devices/threadDevicePreviews.test.ts create mode 100644 apps/mobile/src/features/devices/threadDevicePreviews.ts create mode 100644 apps/mobile/src/state/device.ts create mode 100644 apps/mobile/src/types/mobile-device-stream.d.ts create mode 100644 packages/client-runtime/src/device/hubAccess.ts create mode 100644 packages/client-runtime/src/device/stream.test.ts rename apps/web/src/components/device/deviceStream.ts => packages/client-runtime/src/device/stream.ts (98%) diff --git a/apps/mobile/metro.config.js b/apps/mobile/metro.config.js index 02a6e2c866..9025dc4f6b 100644 --- a/apps/mobile/metro.config.js +++ b/apps/mobile/metro.config.js @@ -17,6 +17,7 @@ const licenseGeneratorSource = path.join( ); const escapedWorkspaceRoot = workspaceRoot.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const mobileShikiRoot = path.dirname(require.resolve("shiki/package.json", { paths: [__dirname] })); +const generatedDeviceStreamRoot = path.join(__dirname, ".generated", "device-stream"); const resolveShikiDependencyRoot = (packageName) => { const entryPath = require.resolve(packageName, { paths: [mobileShikiRoot] }); let currentDir = path.dirname(entryPath); @@ -46,6 +47,7 @@ config.resolver = { extraNodeModules: { ...config.resolver?.extraNodeModules, "@t3tools/mobile-third-party-licenses": generatedLicenseModuleRoot, + "@t3tools/mobile-device-stream": generatedDeviceStreamRoot, shiki: mobileShikiRoot, "@shikijs/core": resolveShikiDependencyRoot("@shikijs/core"), "@shikijs/engine-javascript": resolveShikiDependencyRoot("@shikijs/engine-javascript"), @@ -93,7 +95,32 @@ async function generateMobileThirdPartyLicenses() { ]); } -module.exports = generateMobileThirdPartyLicenses().then(() => +async function prepareDeviceStream() { + const { generateDeviceStreamScript } = await import( + pathToFileURL(path.join(__dirname, "scripts", "generate-device-stream.mts")).href + ); + await generateDeviceStreamScript(); + if (process.env.NODE_ENV !== "production") { + let rebuild = Promise.resolve(); + for (const [directory, files] of [ + [path.join(__dirname, "src/features/devices"), ["device-stream.browser.ts"]], + [ + path.join(workspaceRoot, "packages/client-runtime/src/device"), + ["stream.ts", "hubAccess.ts"], + ], + ]) { + // The generated module participates in Metro's normal Fast Refresh. + fs.watch(directory, { persistent: false }, (_event, filename) => { + if (filename && !files.includes(String(filename))) return; + rebuild = rebuild.then(generateDeviceStreamScript).catch((error) => { + console.error("Could not rebuild the device stream:", error); + }); + }); + } + } +} + +module.exports = Promise.all([generateMobileThirdPartyLicenses(), prepareDeviceStream()]).then(() => withUniwindConfig(config, { cssEntryFile: "./global.css", extraThemes, diff --git a/apps/mobile/scripts/generate-device-stream.mts b/apps/mobile/scripts/generate-device-stream.mts new file mode 100644 index 0000000000..3080ca25c5 --- /dev/null +++ b/apps/mobile/scripts/generate-device-stream.mts @@ -0,0 +1,39 @@ +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; +import { build } from "vite-plus"; + +const mobileRoot = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), ".."); + +/** Metro embeds the shared browser transport as a small script in the native WebView. */ +export async function generateDeviceStreamScript() { + const result = await build({ + configFile: false, + logLevel: "silent", + build: { + write: false, + target: "es2022", + minify: true, + lib: { + entry: NodePath.join(mobileRoot, "src/features/devices/device-stream.browser.ts"), + name: "PylonDeviceStream", + formats: ["iife"], + }, + }, + }); + const bundles = Array.isArray(result) ? result : [result]; + const chunk = bundles + .flatMap((bundle) => ("output" in bundle ? bundle.output : [])) + .find((output) => output.type === "chunk"); + if (!chunk) throw new Error("Device stream build did not emit a script."); + const root = NodePath.join(mobileRoot, ".generated/device-stream"); + await NodeFSP.mkdir(root, { recursive: true }); + for (const [name, contents] of [ + ["index.js", `module.exports = ${JSON.stringify(chunk.code)};\n`], + ["package.json", '{"main":"index.js"}\n'], + ] as const) { + const destination = NodePath.join(root, name); + const previous = await NodeFSP.readFile(destination, "utf8").catch(() => null); + if (previous !== contents) await NodeFSP.writeFile(destination, contents); + } +} diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index 22c02e9957..607f5d4377 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -27,6 +27,7 @@ import { HardwareKeyboardCommandProvider } from "./features/keyboard/HardwareKey import { ReviewCommentComposerSheet } from "./features/review/ReviewCommentComposerSheet"; import { ReviewSheet } from "./features/review/ReviewSheet"; import { ThreadTerminalRouteScreen } from "./features/terminal/ThreadTerminalRouteScreen"; +import { DevicePreviewRouteScreen } from "./features/devices/DevicePreviewRouteScreen"; import { GitBranchesSheet } from "./features/threads/git/GitBranchesSheet"; import { GitCommitSheet } from "./features/threads/git/GitCommitSheet"; import { GitConfirmSheet } from "./features/threads/git/GitConfirmSheet"; @@ -377,6 +378,7 @@ const WORKSPACE_OVERLAY_ROUTES = new Set([ "SettingsLegal", "SettingsSheet", "ThreadReviewComment", + "ThreadDevicePreview", "ThreadSettingsSheet", ]); @@ -530,6 +532,15 @@ export const RootStack = createNativeStackNavigator({ linking: `${THREAD_LINKING_PREFIX}/terminal`, options: SOLID_HEADER_OPTIONS, }), + ThreadDevicePreview: createNativeStackScreen({ + screen: DevicePreviewRouteScreen, + linking: `${THREAD_LINKING_PREFIX}/devices`, + options: { + presentation: "fullScreenModal", + headerShown: false, + gestureEnabled: false, + }, + }), ThreadReview: createNativeStackScreen({ screen: ReviewSheet, linking: `${THREAD_LINKING_PREFIX}/review`, diff --git a/apps/mobile/src/components/AppSymbol.tsx b/apps/mobile/src/components/AppSymbol.tsx index 2f65531fa6..c878b09357 100644 --- a/apps/mobile/src/components/AppSymbol.tsx +++ b/apps/mobile/src/components/AppSymbol.tsx @@ -47,6 +47,7 @@ import IconGitBranch from "@tabler/icons-react-native/IconGitBranch"; import IconHammer from "@tabler/icons-react-native/IconHammer"; import IconGitMerge from "@tabler/icons-react-native/IconGitMerge"; import IconGitPullRequest from "@tabler/icons-react-native/IconGitPullRequest"; +import IconHome from "@tabler/icons-react-native/IconHome"; import IconInfoCircle from "@tabler/icons-react-native/IconInfoCircle"; import IconKeyboard from "@tabler/icons-react-native/IconKeyboard"; import IconKeyboardHide from "@tabler/icons-react-native/IconKeyboardHide"; @@ -143,6 +144,9 @@ const ANDROID_ICON_BY_SF_SYMBOL: Partial> = { "folder.badge.plus": IconFolderPlus, "folder.fill": IconFolder, gearshape: IconSettings, + globe: IconWorld, + hammer: IconHammer, + house: IconHome, "info.circle": IconInfoCircle, link: IconLink, "line.3.horizontal": IconMenu2, @@ -168,6 +172,7 @@ const ANDROID_ICON_BY_SF_SYMBOL: Partial> = { "sidebar.right": IconLayoutSidebarRight, "slider.horizontal.3": IconAdjustmentsHorizontal, "square.and.pencil": IconEdit, + "square.on.square": IconCopy, "square.grid.2x2": IconApps, "square.split.2x1": IconLayoutColumns, "sun.max": IconSun, diff --git a/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx b/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx new file mode 100644 index 0000000000..ec7dbf189a --- /dev/null +++ b/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx @@ -0,0 +1,346 @@ +import { useIsFocused, useNavigation, type StaticScreenProps } from "@react-navigation/native"; +import type { MenuAction } from "@react-native-menu/menu"; +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import { useCallback, useEffect, useMemo, useRef, useState, type RefObject } from "react"; +import { ActivityIndicator, Alert, AppState, Platform, Pressable, View } from "react-native"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; + +import { AndroidHeaderIconButton, AndroidScreenHeader } from "../../components/AndroidScreenHeader"; +import { AppText } from "../../components/AppText"; +import { ControlPillMenu } from "../../components/ControlPill"; +import { NativeHeaderToolbar, NativeStackScreenOptions } from "../../native/StackHeader"; +import { + deviceEnvironment, + refreshDeviceHubAccess, + useDeviceHubAccess, + useLiveDeviceState, +} from "../../state/device"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { useAppearancePreferences } from "../settings/appearance/AppearancePreferencesProvider"; +import { DeviceStreamWebView, type DeviceStreamRef } from "./DeviceStreamWebView"; +import { + selectedThreadDevicePreview, + threadDevicePreviews, + type ThreadDevicePreview, +} from "./threadDevicePreviews"; + +type DevicePreviewRouteScreenProps = StaticScreenProps<{ + readonly environmentId: string; + readonly threadId: string; +}>; + +export function DevicePreviewRouteScreen({ route }: DevicePreviewRouteScreenProps) { + const navigation = useNavigation(); + const onClose = useCallback(() => navigation.goBack(), [navigation]); + return ( + + ); +} + +function DevicePreviewScreen({ + environmentId, + threadId, + onClose, +}: { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly onClose: () => void; +}) { + const insets = useSafeAreaInsets(); + const { themeVariables } = useAppearancePreferences(); + const focused = useIsFocused(); + const [foreground, setForeground] = useState(AppState.currentState !== "background"); + const [selectedKey, setSelectedKey] = useState(null); + const [inputConnected, setInputConnected] = useState(false); + const [streamAttempt, setStreamAttempt] = useState(0); + const [shuttingDown, setShuttingDown] = useState(false); + const shutdown = useAtomCommand(deviceEnvironment.shutdown, { reportFailure: false }); + const streamRef = useRef(null); + const state = useLiveDeviceState(environmentId); + const previews = useMemo( + () => threadDevicePreviews(state.data, threadId), + [state.data, threadId], + ); + const preview = selectedThreadDevicePreview(previews, selectedKey); + const onInputConnected = useCallback( + async (connected: boolean) => setInputConnected(connected), + [], + ); + useEffect(() => { + const subscription = AppState.addEventListener("change", (state) => + setForeground(state !== "background"), + ); + return () => subscription.remove(); + }, []); + useEffect(() => { + if (focused && state.data !== null && previews.length === 0) onClose(); + }, [focused, state.data, previews.length, onClose]); + + const shutDownDevice = async () => { + if (!preview || shuttingDown) return; + setShuttingDown(true); + try { + const result = await shutdown({ + environmentId, + input: { + hostId: preview.session.hostId, + deviceId: preview.session.deviceId, + platform: preview.session.platform, + }, + }); + if (result._tag === "Failure") { + Alert.alert("Could not shut down device", String(Cause.squash(result.cause))); + } + } finally { + setShuttingDown(false); + } + }; + + const controls = [ + { + id: "reload", + title: "Reload stream", + icon: "arrow.clockwise", + disabled: !preview || shuttingDown, + onPress: () => { + setInputConnected(false); + setStreamAttempt((attempt) => attempt + 1); + }, + }, + ...(preview?.session.platform === "android" + ? [ + { + id: "back", + title: "Back", + icon: "arrow.left", + disabled: !inputConnected, + onPress: () => streamRef.current?.back(), + }, + ] + : []), + { + id: "app-switcher", + title: "App switcher", + icon: "square.on.square", + disabled: !inputConnected, + onPress: () => streamRef.current?.appSwitcher(), + }, + ...(preview?.session.platform === "ios" + ? [ + { + id: "rotate", + title: "Rotate device", + icon: "arrow.clockwise", + disabled: !inputConnected, + onPress: () => streamRef.current?.rotate(), + }, + ] + : []), + { + id: "shutdown", + title: shuttingDown ? "Shutting down…" : "Shut down device", + icon: "power", + disabled: !preview || shuttingDown, + onPress: () => void shutDownDevice(), + }, + ] as const; + const androidActions: MenuAction[] = [ + ...(previews.length > 1 + ? [ + { + id: "devices", + title: "Devices", + subactions: previews.map((device) => ({ + id: `device:${device.key}`, + title: device.name, + subtitle: device.description, + state: device.key === preview?.key ? ("on" as const) : undefined, + })), + }, + ] + : []), + ...controls.map((control) => ({ + id: control.id, + title: control.title, + attributes: control.disabled ? { disabled: true } : undefined, + })), + ]; + return ( + + + {Platform.OS === "android" ? ( + + streamRef.current?.home()} + /> + { + const id = event.nativeEvent.event; + if (id.startsWith("device:")) setSelectedKey(id.slice("device:".length)); + else controls.find((control) => control.id === id)?.onPress(); + }} + > + + + + } + /> + ) : null} + + streamRef.current?.home()} + separateBackground + /> + + {previews.length > 1 ? ( + + {previews.map((device) => ( + setSelectedKey(device.key)} + > + {device.name} + + ))} + + ) : null} + {controls.map((control) => ( + + {control.title} + + ))} + + + {Platform.OS === "ios" ? ( + + + + ) : null} + {preview && focused && foreground ? ( + + ) : ( + + {state.error ? ( + <> + + {state.error} + + + Retry + + + ) : focused && foreground ? ( + + ) : null} + + )} + + ); +} + +function OpenDevicePreview({ + environmentId, + preview, + streamRef, + onInputConnected, +}: { + readonly environmentId: EnvironmentId; + readonly preview: ThreadDevicePreview; + readonly streamRef: RefObject; + readonly onInputConnected: (connected: boolean) => Promise; +}) { + const { session } = preview; + const { themeVariables } = useAppearancePreferences(); + const { access, error, refresh } = useDeviceHubAccess(environmentId, session.hostId); + const onUnauthorized = useCallback( + async () => refreshDeviceHubAccess(environmentId), + [environmentId], + ); + useEffect(() => { + refreshDeviceHubAccess(environmentId); + return () => void onInputConnected(false); + }, [environmentId, onInputConnected]); + return access ? ( + + ) : ( + + {error ? ( + <> + + {error} + + + Retry + + + ) : ( + <> + + Connecting to device... + + )} + + ); +} diff --git a/apps/mobile/src/features/devices/DeviceStreamWebView.tsx b/apps/mobile/src/features/devices/DeviceStreamWebView.tsx new file mode 100644 index 0000000000..c77b49b9bd --- /dev/null +++ b/apps/mobile/src/features/devices/DeviceStreamWebView.tsx @@ -0,0 +1,226 @@ +import deviceStreamScript from "@t3tools/mobile-device-stream"; +import { + useEffect, + useEffectEvent, + useImperativeHandle, + useLayoutEffect, + useMemo, + useRef, + useState, + type Ref, +} from "react"; +import { ActivityIndicator, Pressable, View } from "react-native"; +import { WebView } from "react-native-webview"; +import type { DeviceStreamStatus } from "@t3tools/client-runtime/device/stream"; + +import { AppText } from "../../components/AppText"; +import { uuidv4 } from "../../lib/uuid"; + +import { + deviceStreamDocument, + deviceStreamDocumentBaseUrl, + deviceStreamFailureDetail, + deviceStreamMessage, + isDeviceStreamDocumentNavigation, + type DeviceStreamConfiguration, +} from "./device-stream-document"; + +export interface DeviceStreamRef { + home: () => void; + back: () => void; + appSwitcher: () => void; + rotate: () => void; +} + +type NativeStreamBridge = { + readonly ref?: Ref; + readonly onUnauthorized: () => Promise; + readonly onInputConnected: (connected: boolean) => Promise; +}; + +export function DeviceStreamWebView({ + ref, + ...props +}: DeviceStreamConfiguration & NativeStreamBridge) { + const [attempt, setAttempt] = useState(0); + const processRetried = useRef(false); + const configuration = JSON.stringify({ + access: props.access, + platform: props.platform, + deviceId: props.deviceId, + colors: props.colors, + }); + // A credential change gets a new opaque key only when React commits this + // render. Abandoned renders never replace the active WebView, and keys do + // not expose the ticket in diagnostics. + const documentIdentity = useMemo(() => ({ configuration, id: uuidv4() }), [configuration]); + return ( + { + processRetried.current = false; + setAttempt((attempt) => attempt + 1); + void props.onUnauthorized(); + }} + onStreaming={() => { + processRetried.current = false; + }} + onRecoverProcess={() => { + if (processRetried.current) return false; + processRetried.current = true; + setAttempt((attempt) => attempt + 1); + void props.onUnauthorized(); + return true; + }} + /> + ); +} + +function DeviceStreamDocumentView({ + ref, + configuration, + documentBaseUrl, + useSessionCookies, + background, + onUnauthorized, + onInputConnected, + onRetry, + onStreaming, + onRecoverProcess, +}: NativeStreamBridge & { + readonly configuration: string; + readonly documentBaseUrl: string; + readonly useSessionCookies: boolean; + readonly background: string; + readonly onRetry: () => void; + readonly onStreaming: () => void; + readonly onRecoverProcess: () => boolean; +}) { + const webView = useRef(null); + const active = useRef(true); + const failed = useRef(false); + const [status, setStatus] = useState("connecting"); + const [error, setError] = useState(null); + const [started, setStarted] = useState(false); + const fail = (message: string) => { + if (!active.current || failed.current) return; + failed.current = true; + void onInputConnected(false); + webView.current?.injectJavaScript("window.PylonDeviceStream?.stop(); true;"); + setError(message); + setStatus("error"); + }; + // The shared transport owns video timeouts once the document acknowledges startup. + const bootstrapTimedOut = useEffectEvent(() => + fail("Device viewer could not start. Reconnect to try again."), + ); + useEffect(() => { + if (started) return; + const timer = setTimeout(bootstrapTimedOut, 15_000); + return () => clearTimeout(timer); + }, [started]); + const source = useMemo( + () => ({ + html: deviceStreamDocument(configuration, deviceStreamScript), + baseUrl: documentBaseUrl, + }), + [configuration, documentBaseUrl], + ); + const command = (button: keyof DeviceStreamRef) => { + webView.current?.injectJavaScript( + `window.PylonDeviceStream?.command(${JSON.stringify(button)}); true;`, + ); + }; + useImperativeHandle(ref, () => ({ + home: () => command("home"), + back: () => command("back"), + appSwitcher: () => command("appSwitcher"), + rotate: () => command("rotate"), + })); + const resetInput = useEffectEvent(() => void onInputConnected(false)); + useLayoutEffect(() => { + active.current = true; + resetInput(); + const view = webView.current; + return () => { + active.current = false; + view?.injectJavaScript("window.PylonDeviceStream?.stop(); true;"); + }; + }, []); + const processTerminated = () => { + if (!active.current || failed.current) return; + void onInputConnected(false); + if (!onRecoverProcess()) fail("Device viewer stopped. Reconnect to try again."); + }; + return ( + + fail("Device viewer could not load. Reconnect to try again.")} + onHttpError={() => fail("Device viewer could not load. Reconnect to try again.")} + onContentProcessDidTerminate={processTerminated} + onRenderProcessGone={processTerminated} + onShouldStartLoadWithRequest={(request) => + isDeviceStreamDocumentNavigation(request.url, source.baseUrl) + } + onMessage={(event) => { + if (!active.current || failed.current) return; + if (!isDeviceStreamDocumentNavigation(event.nativeEvent.url, source.baseUrl)) return; + const message = deviceStreamMessage(event.nativeEvent.data); + if (message?.type === "unauthorized") void onUnauthorized(); + else if (message?.type === "input") void onInputConnected(message.connected); + else if (message?.type === "retry") onRetry(); + else if (message?.type === "status") { + setStarted(true); + if (message.status === "error") fail(deviceStreamFailureDetail(message.detail)); + else { + setStatus(message.status); + if (message.status === "streaming") onStreaming(); + } + } + }} + /> + {status !== "streaming" ? ( + + {status === "connecting" ? : null} + + {status === "error" ? error : "Connecting to device..."} + + {status === "error" ? ( + + Reconnect + + ) : null} + + ) : null} + + ); +} diff --git a/apps/mobile/src/features/devices/device-preview-button.tsx b/apps/mobile/src/features/devices/device-preview-button.tsx new file mode 100644 index 0000000000..01e7899467 --- /dev/null +++ b/apps/mobile/src/features/devices/device-preview-button.tsx @@ -0,0 +1,41 @@ +import { Pressable, View } from "react-native"; + +import { AppText } from "../../components/AppText"; +import { SymbolView } from "../../components/AppSymbol"; + +export function DevicePreviewButton(props: { + readonly count: number; + readonly onPress: () => void; + readonly compact?: boolean; +}) { + const compact = props.compact ?? true; + return ( + + + {!compact ? ( + + {props.count === 1 ? "One device open" : `${props.count} devices open`} + + ) : props.count > 1 ? ( + + {props.count} + + ) : null} + + ); +} diff --git a/apps/mobile/src/features/devices/device-stream-document.test.ts b/apps/mobile/src/features/devices/device-stream-document.test.ts new file mode 100644 index 0000000000..7b9bd9d096 --- /dev/null +++ b/apps/mobile/src/features/devices/device-stream-document.test.ts @@ -0,0 +1,127 @@ +import * as NodeVM from "node:vm"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { + deviceStreamDocument, + deviceStreamDocumentBaseUrl, + deviceStreamFailureDetail, + deviceStreamMessage, + isDeviceStreamDocumentNavigation, +} from "./device-stream-document"; + +describe("native device stream origin", () => { + it.each([ + ["http://127.0.0.1:3710/api/device-hub", "http://127.0.0.1:3710/api/device-hub/"], + ["https://relay.example/prefix/api/device-hub", "https://relay.example/api/device-hub/"], + ])("uses the environment origin for %s", (httpBase, expected) => { + const base = deviceStreamDocumentBaseUrl({ + httpBase, + wsBase: httpBase.replace(/^http/, "ws"), + query: {}, + credentials: false, + }); + expect(base).toBe(expected); + expect(isDeviceStreamDocumentNavigation(base, base)).toBe(true); + expect(isDeviceStreamDocumentNavigation("about:blank", base)).toBe(true); + expect(isDeviceStreamDocumentNavigation(`${base}other`, base)).toBe(false); + expect(isDeviceStreamDocumentNavigation("https://attacker.example/", base)).toBe(false); + }); +}); + +it("does not show ticketed URLs from stream exceptions", () => { + expect(deviceStreamFailureDetail("fetch ws://host/api/device-hub?wsTicket=secret failed")).toBe( + "Device stream failed. Reconnect to try again.", + ); + expect( + deviceStreamFailureDetail( + "This viewer cannot decode the Android stream (WebCodecs unavailable).", + ), + ).toBe("This viewer cannot decode the Android stream (WebCodecs unavailable)."); +}); + +describe("native device stream document", () => { + it("keeps ticket and device values from terminating the embedded script", () => { + const deviceId = ''; + const configuration = JSON.stringify({ deviceId, ticket: "" }); + const html = deviceStreamDocument( + configuration, + "var PylonDeviceStream={start(input){return input}};", + ); + const script = html.match(/";'); + expect(html.match(/<\/script>/g)).toHaveLength(1); + expect(html).toContain('var text="<\\/script>";'); + }); + + it("reports a bootstrap exception to the native recovery UI", () => { + const postMessage = vi.fn(); + const html = deviceStreamDocument( + "{}", + 'var PylonDeviceStream={start(){throw new Error("startup")}};', + ); + NodeVM.runInNewContext(html.match(/`; +} + +export function deviceStreamMessage(data: string) { + try { + const message: unknown = JSON.parse(data); + if (typeof message !== "object" || message === null || !("type" in message)) return null; + if (message.type === "unauthorized" || message.type === "retry") { + return { type: message.type } as const; + } + if ( + message.type === "input" && + "connected" in message && + typeof message.connected === "boolean" + ) { + return { type: message.type, connected: message.connected } as const; + } + if ( + message.type === "status" && + "status" in message && + (message.status === "connecting" || + message.status === "streaming" || + message.status === "error") && + (!("detail" in message) || typeof message.detail === "string") + ) { + return { + type: message.type, + status: message.status, + detail: + "detail" in message && typeof message.detail === "string" ? message.detail : undefined, + } as const; + } + } catch { + // Ignore messages that are not part of the stream bridge. + } + return null; +} diff --git a/apps/mobile/src/features/devices/device-stream.browser.test.ts b/apps/mobile/src/features/devices/device-stream.browser.test.ts new file mode 100644 index 0000000000..5aeefdac02 --- /dev/null +++ b/apps/mobile/src/features/devices/device-stream.browser.test.ts @@ -0,0 +1,117 @@ +import { afterEach, expect, it, vi } from "vite-plus/test"; +import { start, stop } from "./device-stream.browser"; + +class Element extends EventTarget { + readonly style = {}; + naturalWidth = 0; + naturalHeight = 0; + src = ""; + readonly tag: string; + constructor(tag: string) { + super(); + this.tag = tag; + } + setAttribute() {} + removeAttribute(name: string) { + if (name === "src") this.src = ""; + } + append() {} +} + +async function setup() { + vi.useFakeTimers(); + const elements: Element[] = []; + vi.stubGlobal("document", { + documentElement: { style: {} }, + body: { style: {}, replaceChildren() {} }, + createElement: (tag: string) => { + const element = new Element(tag); + elements.push(element); + return element; + }, + }); + const postMessage = vi.fn(); + vi.stubGlobal("window", { ReactNativeWebView: { postMessage }, addEventListener() {} }); + vi.stubGlobal("fetch", () => Promise.resolve(new Response("prime"))); + const sockets: Socket[] = []; + class Socket { + static OPEN = 1; + readyState = 1; + onopen: (() => void) | null = null; + close = vi.fn(); + send = vi.fn(); + constructor() { + sockets.push(this); + } + } + vi.stubGlobal("WebSocket", Socket); + const configuration = { + platform: "ios" as const, + deviceId: "fixture-device", + access: { + httpBase: "https://device.test", + wsBase: "wss://device.test", + credentials: false, + query: {}, + }, + colors: { + background: "white", + foreground: "black", + muted: "gray", + buttonBackground: "gray", + buttonForeground: "black", + buttonBorder: "gray", + }, + }; + start(configuration); + await vi.advanceTimersByTimeAsync(0); + return { + configuration, + elements, + sockets, + messages: () => + postMessage.mock.calls.map( + ([message]) => JSON.parse(message as string) as { type: string; status?: string }, + ), + }; +} + +afterEach(() => { + stop(); + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +it("bridges shared first-frame readiness, image failure, and a successful fresh attempt to native", async () => { + const { elements, messages, sockets, configuration } = await setup(); + sockets[0]!.onopen?.(); + expect(messages()).not.toContainEqual({ type: "status", status: "streaming" }); + expect(messages()).toContainEqual({ type: "input", connected: true }); + const image = elements.find((element) => element.tag === "img")!; + image.naturalWidth = 400; + image.naturalHeight = 800; + await vi.advanceTimersByTimeAsync(250); + expect(messages()).toContainEqual({ type: "status", status: "streaming" }); + image.dispatchEvent(new Event("error")); + expect(messages()).toContainEqual({ + type: "status", + status: "error", + detail: "Could not receive the device stream. Reconnect to try again.", + }); + expect(messages()).toContainEqual({ type: "input", connected: false }); + expect(messages()).not.toContainEqual({ type: "unauthorized" }); + expect(image.src).toBe(""); + expect(sockets[0]!.close).toHaveBeenCalledOnce(); + start(configuration); + await vi.advanceTimersByTimeAsync(0); + const replacement = elements.findLast((element) => element.tag === "img")!; + replacement.naturalWidth = 400; + replacement.naturalHeight = 800; + replacement.dispatchEvent(new Event("load")); + expect(messages().at(-1)).toEqual({ type: "status", status: "streaming" }); + image.dispatchEvent(new Event("error")); + expect(sockets[1]!.close).not.toHaveBeenCalled(); + stop(); + expect(replacement.src).toBe(""); + expect(vi.getTimerCount()).toBe(0); +}); diff --git a/apps/mobile/src/features/devices/device-stream.browser.ts b/apps/mobile/src/features/devices/device-stream.browser.ts new file mode 100644 index 0000000000..0b9e98a51b --- /dev/null +++ b/apps/mobile/src/features/devices/device-stream.browser.ts @@ -0,0 +1,189 @@ +import { + createDeviceStreamClient, + type DeviceScreenSize, +} from "@t3tools/client-runtime/device/stream"; + +import type { DeviceStreamConfiguration } from "./device-stream-document"; + +declare global { + interface Window { + ReactNativeWebView: { postMessage: (message: string) => void }; + } +} + +let activeClient: ReturnType | null = null; + +export function stop() { + activeClient?.stop(); + activeClient = null; +} + +export function command(button: "home" | "back" | "appSwitcher" | "rotate") { + if (button === "rotate") activeClient?.rotate(); + else activeClient?.pressButton(button); +} + +/** Bundled into the existing native WebView without React or Expo's web runtime. */ +export function start(configuration: DeviceStreamConfiguration) { + stop(); + // oxlint-disable-next-line unicorn/require-post-message-target-origin -- The native WebView bridge takes one string. + const post = (message: object) => window.ReactNativeWebView.postMessage(JSON.stringify(message)); + const { colors, platform } = configuration; + Object.assign(document.documentElement.style, { height: "100%", overflow: "hidden" }); + Object.assign(document.body.style, { + margin: "0", + height: "100%", + overflow: "hidden", + background: colors.background, + color: colors.foreground, + fontFamily: "system-ui", + }); + const container = document.createElement("div"); + Object.assign(container.style, { + position: "fixed", + inset: "0", + containerType: "size", + display: "flex", + alignItems: "center", + justifyContent: "center", + }); + const frame = document.createElement("div"); + frame.setAttribute("role", "application"); + frame.setAttribute( + "aria-label", + platform === "ios" ? "iOS Simulator screen" : "Android Emulator screen", + ); + frame.tabIndex = 0; + Object.assign(frame.style, { + position: "relative", + touchAction: "none", + userSelect: "none", + webkitUserSelect: "none", + webkitTouchCallout: "none", + outline: "none", + }); + const canvas = document.createElement("canvas"); + const image = document.createElement("img"); + image.alt = ""; + image.draggable = false; + image.style.display = "none"; + const inputStatus = document.createElement("div"); + inputStatus.setAttribute("role", "status"); + inputStatus.textContent = "Reconnecting device controls..."; + Object.assign(inputStatus.style, { + position: "fixed", + bottom: "12px", + left: "0", + right: "0", + textAlign: "center", + pointerEvents: "none", + fontSize: "13px", + color: colors.muted, + background: colors.background, + }); + frame.append(canvas, image); + container.append(frame); + document.body.replaceChildren(container, inputStatus); + + let pointerId: number | null = null; + let inputConnected = false; + let streaming = false; + const reportStatus = (status: "connecting" | "streaming" | "error", detail?: string) => { + if (activeClient !== client) return; + streaming = status === "streaming"; + inputStatus.style.display = streaming && !inputConnected ? "block" : "none"; + post({ type: "status", status, detail }); + }; + const layout = (screen: DeviceScreenSize | null) => { + const landscape = + screen?.orientation === "landscape_left" || screen?.orientation === "landscape_right"; + const aspect = screen + ? landscape + ? Math.max(screen.width, screen.height) / Math.min(screen.width, screen.height) + : Math.min(screen.width, screen.height) / Math.max(screen.width, screen.height) + : 9 / 19.5; + const rotation = + platform === "ios" && screen && screen.width <= screen.height + ? screen.orientation === "landscape_left" + ? 90 + : screen.orientation === "landscape_right" + ? -90 + : screen.orientation === "portrait_upside_down" + ? 180 + : 0 + : 0; + const sideways = Math.abs(rotation) === 90; + frame.style.width = `min(100cqw, ${aspect * 100}cqh)`; + frame.style.height = `min(100cqh, ${100 / aspect}cqw)`; + for (const media of [canvas, image]) { + Object.assign(media.style, { + position: "absolute", + width: sideways ? `${100 / aspect}%` : "100%", + height: sideways ? `${100 * aspect}%` : "100%", + left: "50%", + top: "50%", + transform: `translate(-50%, -50%) rotate(${rotation}deg)`, + pointerEvents: "none", + }); + } + }; + layout(null); + const unauthorized = () => { + if (activeClient === client) post({ type: "unauthorized" }); + }; + const client = createDeviceStreamClient( + { ...configuration, preferMjpeg: platform === "ios" }, + canvas, + { + onStatus: reportStatus, + onScreen: layout, + onMjpegFallback: () => { + canvas.style.display = "none"; + image.style.display = "block"; + }, + onUnauthorized: unauthorized, + onInputConnected: (connected) => { + inputConnected = connected; + inputStatus.style.display = streaming && !connected ? "block" : "none"; + post({ type: "input", connected }); + }, + }, + ); + activeClient = client; + client.setMjpegImage(image); + const touch = (event: PointerEvent, phase: "begin" | "move" | "end") => { + const rect = frame.getBoundingClientRect(); + client.sendTouch( + phase, + Math.max(0, Math.min(1, (event.clientX - rect.left) / rect.width)), + Math.max(0, Math.min(1, (event.clientY - rect.top) / rect.height)), + ); + }; + frame.addEventListener("pointerdown", (event) => { + if (!inputConnected || pointerId !== null) return; + event.preventDefault(); + pointerId = event.pointerId; + frame.setPointerCapture(event.pointerId); + frame.focus(); + touch(event, "begin"); + }); + frame.addEventListener("pointermove", (event) => { + if (pointerId === event.pointerId) touch(event, "move"); + }); + const endTouch = (event: PointerEvent) => { + if (pointerId !== event.pointerId) return; + pointerId = null; + touch(event, "end"); + }; + frame.addEventListener("pointerup", endTouch); + frame.addEventListener("pointercancel", endTouch); + frame.addEventListener("lostpointercapture", endTouch); + frame.addEventListener("keydown", (event) => { + event.preventDefault(); + client.sendKey(event, "down"); + }); + frame.addEventListener("keyup", (event) => client.sendKey(event, "up")); + window.addEventListener("pagehide", stop, { once: true }); + post({ type: "input", connected: false }); + client.start(); +} diff --git a/apps/mobile/src/features/devices/deviceHubAccessScope.test.ts b/apps/mobile/src/features/devices/deviceHubAccessScope.test.ts new file mode 100644 index 0000000000..0621da75ab --- /dev/null +++ b/apps/mobile/src/features/devices/deviceHubAccessScope.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from "vite-plus/test"; +import type { SupervisorConnectionState } from "@t3tools/client-runtime/connection"; + +import { currentDeviceHubAccess, currentDeviceState } from "./deviceHubAccessScope"; + +const connected = (generation: number): SupervisorConnectionState => ({ + desired: true, + network: "online", + phase: "connected", + stage: null, + attempt: 1, + generation, + lastFailure: null, + retryAt: null, +}); + +const access = { + httpBase: "https://environment.test/api/device-hub", + wsBase: "wss://environment.test/api/device-hub", + credentials: false, + query: { wsTicket: "fixture-ticket" }, +}; + +describe("mobile device access scope", () => { + it("hides A's pending ticket and preview through a B reconnect until B's receipt arrives", () => { + const preparedA = { session: "A" }; + const preparedB = { session: "B" }; + const readingA = { prepared: preparedA, generation: 1, access }; + const view = (input: { + prepared: typeof preparedA; + generation: number; + reading: typeof readingA; + pending?: boolean; + }) => + currentDeviceHubAccess({ + prepared: input.prepared, + connection: connected(input.generation), + connectionPending: false, + reading: input.reading, + readingPending: input.pending ?? false, + readingError: null, + hostId: "selected-host", + }); + + expect(view({ prepared: preparedA, generation: 1, reading: readingA })).toEqual({ + ...access, + query: { wsTicket: "fixture-ticket", hostId: "selected-host" }, + }); + expect( + view({ prepared: preparedA, generation: 1, reading: readingA, pending: true }), + ).toBeNull(); + expect(view({ prepared: preparedB, generation: 1, reading: readingA })).toBeNull(); + expect(view({ prepared: preparedB, generation: 2, reading: readingA })).toBeNull(); + const readingB = { prepared: preparedB, generation: 2, access }; + // The supervisor may already be connected to B while the separate + // prepared-connection stream still exposes A's prior value. + expect(view({ prepared: preparedA, generation: 2, reading: readingB })).toBeNull(); + expect( + view({ + prepared: preparedB, + generation: 2, + reading: readingB, + }), + ).not.toBeNull(); + + const oldPreview = { sessions: ["A"] }; + expect( + currentDeviceState({ + connection: connected(2), + connectionPending: false, + state: oldPreview, + statePending: true, + stateError: null, + }), + ).toBeNull(); + expect( + currentDeviceState({ + connection: { ...connected(2), phase: "connecting" }, + connectionPending: false, + state: oldPreview, + statePending: false, + stateError: null, + }), + ).toBeNull(); + }); +}); diff --git a/apps/mobile/src/features/devices/deviceHubAccessScope.ts b/apps/mobile/src/features/devices/deviceHubAccessScope.ts new file mode 100644 index 0000000000..d1a8058536 --- /dev/null +++ b/apps/mobile/src/features/devices/deviceHubAccessScope.ts @@ -0,0 +1,51 @@ +import type { DeviceHubAccess } from "@t3tools/client-runtime/device/hub-access"; +import type { SupervisorConnectionState } from "@t3tools/client-runtime/connection"; + +export interface ScopedDeviceHubAccess { + readonly prepared: Prepared; + readonly generation: number; + readonly access: DeviceHubAccess; +} + +/** A waiting atom may carry the previous session's still-valid ticket. */ +export function currentDeviceHubAccess(input: { + readonly prepared: Prepared | null; + readonly connection: SupervisorConnectionState | null; + readonly connectionPending: boolean; + readonly reading: ScopedDeviceHubAccess | null; + readonly readingPending: boolean; + readonly readingError: string | null; + readonly hostId: string; +}): DeviceHubAccess | null { + if ( + input.connectionPending || + input.connection?.phase !== "connected" || + input.prepared === null || + input.readingPending || + input.readingError !== null || + input.reading === null || + input.reading.prepared !== input.prepared || + input.reading.generation !== input.connection.generation + ) { + return null; + } + return { + ...input.reading.access, + query: { ...input.reading.access.query, hostId: input.hostId }, + }; +} + +export function currentDeviceState(input: { + readonly connection: SupervisorConnectionState | null; + readonly connectionPending: boolean; + readonly state: State | null; + readonly statePending: boolean; + readonly stateError: string | null; +}): State | null { + return !input.connectionPending && + input.connection?.phase === "connected" && + !input.statePending && + input.stateError === null + ? input.state + : null; +} diff --git a/apps/mobile/src/features/devices/threadDevicePreviews.test.ts b/apps/mobile/src/features/devices/threadDevicePreviews.test.ts new file mode 100644 index 0000000000..90582dbdc5 --- /dev/null +++ b/apps/mobile/src/features/devices/threadDevicePreviews.test.ts @@ -0,0 +1,106 @@ +import { ThreadId, type DeviceServiceState } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { selectedThreadDevicePreview, threadDevicePreviews } from "./threadDevicePreviews"; + +const threadId = ThreadId.make("thread"); +const state: DeviceServiceState = { + hosts: [ + { + id: "local", + kind: "local", + label: "Local", + platforms: [], + hubInstalled: true, + agentDeviceInstalled: true, + }, + { + id: "remote", + kind: "ssh", + label: "Mac mini", + platforms: [], + hubInstalled: true, + agentDeviceInstalled: true, + }, + ], + hostStatus: "ready", + hostStatuses: {}, + devices: [ + { + hostId: "local", + id: "emulator-5554", + platform: "android", + name: "Pixel 9", + version: "Android 16", + booted: true, + physical: false, + }, + { + hostId: "remote", + id: "emulator-5554", + platform: "android", + name: "Pixel 8", + version: "Android 15", + booted: true, + physical: false, + }, + ], + sessions: [ + { + threadId, + hostId: "local", + deviceId: "emulator-5554", + platform: "android", + openedAt: "2026-09-18T00:00:00Z", + }, + { + threadId, + hostId: "remote", + deviceId: "emulator-5554", + platform: "android", + openedAt: "2026-09-18T00:01:00Z", + }, + { + threadId: ThreadId.make("another-thread"), + hostId: "local", + deviceId: "iphone", + platform: "ios", + openedAt: "2026-09-18T00:02:00Z", + }, + ], + onboardingCompleted: true, + agentAccessEnabled: true, + hubBasePath: "/api/device-hub", + revision: 1, +}; + +describe("thread device previews", () => { + it("keeps device names and selection distinct when hosts have the same Android serial", () => { + const previews = threadDevicePreviews(state, threadId); + expect(previews.map((preview) => preview.name)).toEqual(["Pixel 9", "Pixel 8"]); + expect(new Set(previews.map((preview) => preview.key)).size).toBe(2); + expect(selectedThreadDevicePreview(previews, previews[1]!.key)?.description).toBe( + "Android 15 · Mac mini", + ); + }); + + it("selects another open device when the agent closes the selected session", () => { + const previews = threadDevicePreviews(state, threadId); + const selectedKey = previews[1]!.key; + const afterClose = threadDevicePreviews({ ...state, sessions: [state.sessions[0]!] }, threadId); + expect(selectedThreadDevicePreview(afterClose, selectedKey)?.name).toBe("Pixel 9"); + expect(selectedThreadDevicePreview([], selectedKey)).toBeNull(); + }); + + it("can view an open session before device discovery metadata arrives", () => { + const previews = threadDevicePreviews({ ...state, hosts: [], devices: [] }, threadId); + expect(previews[0]?.name).toBe("Android Emulator"); + expect(previews[0]?.session.deviceId).toBe("emulator-5554"); + expect(previews[0]?.description).toBe(""); + }); + + it("offers no devices before state arrives or in a thread without sessions", () => { + expect(threadDevicePreviews(null, threadId)).toEqual([]); + expect(threadDevicePreviews(state, ThreadId.make("empty-thread"))).toEqual([]); + }); +}); diff --git a/apps/mobile/src/features/devices/threadDevicePreviews.ts b/apps/mobile/src/features/devices/threadDevicePreviews.ts new file mode 100644 index 0000000000..aaf87ca978 --- /dev/null +++ b/apps/mobile/src/features/devices/threadDevicePreviews.ts @@ -0,0 +1,28 @@ +import type { DeviceServiceState, ThreadId } from "@t3tools/contracts"; + +/** Host identity is part of the selection because Android serials repeat across hosts. */ +export function threadDevicePreviews(state: DeviceServiceState | null, threadId: ThreadId) { + return (state?.sessions ?? []) + .filter((session) => session.threadId === threadId) + .map((session) => { + const device = state?.devices.find( + (device) => device.hostId === session.hostId && device.id === session.deviceId, + ); + const host = state?.hosts.find((host) => host.id === session.hostId); + return { + key: JSON.stringify([session.hostId, session.deviceId]), + session, + name: device?.name ?? (session.platform === "ios" ? "iOS Simulator" : "Android Emulator"), + description: [device?.version, host?.label].filter(Boolean).join(" · "), + }; + }); +} + +export type ThreadDevicePreview = ReturnType[number]; + +export function selectedThreadDevicePreview( + previews: ReadonlyArray, + selectedKey: string | null, +) { + return previews.find((preview) => preview.key === selectedKey) ?? previews[0] ?? null; +} diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index ad5c6c4a52..463066ded4 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -24,6 +24,7 @@ import { useKeyboardChatComposerInset, useKeyboardScrollToEnd } from "@legendapp import { resolveProviderSkillsForCwd } from "@t3tools/client-runtime/providerSkills"; import type { LegendListRef } from "@legendapp/list/react-native"; import { HeaderHeightContext } from "@react-navigation/elements"; +import { useNavigation } from "@react-navigation/native"; import type { ApprovalRequestId, EnvironmentId, @@ -92,6 +93,8 @@ import type { DraftComposerAttachment } from "../../lib/composerImages"; import { CHAT_CONTENT_MAX_WIDTH, type LayoutVariant } from "../../lib/layout"; import { IOS_NAV_BAR_HEIGHT } from "../../lib/layoutMetrics"; import { editPendingThreadMessage } from "../../state/edit-pending-thread-message"; +import { useLiveDeviceState } from "../../state/device"; +import { threadDevicePreviews } from "../devices/threadDevicePreviews"; import type { QueuedThreadMessage } from "../../state/thread-outbox-model"; import { scopedThreadKey } from "../../lib/scopedEntities"; import type { @@ -392,6 +395,19 @@ function RollbackStatusSurface(props: { } export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: ThreadDetailScreenProps) { + const navigation = useNavigation(); + const deviceState = useLiveDeviceState(props.environmentId); + const devicePreviews = useMemo( + () => threadDevicePreviews(deviceState.data, props.selectedThread.id), + [deviceState.data, props.selectedThread.id], + ); + const openDevicePreview = useCallback(() => { + Keyboard.dismiss(); + navigation.navigate("ThreadDevicePreview", { + environmentId: props.environmentId, + threadId: props.selectedThread.id, + }); + }, [navigation, props.environmentId, props.selectedThread.id]); const insets = useSafeAreaInsets(); const isKeyboardVisible = useKeyboardState((state) => state.isVisible); const liveKeyboardHeight = useKeyboardState((state) => state.height); @@ -539,6 +555,7 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread // reserves no status space. const showFloatingStatus = showWorkingControl || + devicePreviews.length > 0 || props.connectionStateLabel !== "connected" || props.queuedMessages.some((message) => message.deliveryHold === undefined) || props.selectedThreadFeed.some( @@ -1059,6 +1076,11 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread 0 + ? { count: devicePreviews.length, onPress: openDevicePreview } + : null + } showScrollToEnd={showScrollToEndButton} onScrollToEnd={handleScrollToEnd} /> diff --git a/apps/mobile/src/features/threads/floating-working-control.tsx b/apps/mobile/src/features/threads/floating-working-control.tsx index bb55de1b00..fb3a3edd17 100644 --- a/apps/mobile/src/features/threads/floating-working-control.tsx +++ b/apps/mobile/src/features/threads/floating-working-control.tsx @@ -28,6 +28,7 @@ import { SymbolView } from "../../components/AppSymbol"; import type { FloatingWorkingStatus } from "./floating-working-status"; import { ControlPill } from "../../components/ControlPill"; import { NATIVE_LIQUID_GLASS_SUPPORTED } from "../../native/native-glass"; +import { DevicePreviewButton } from "../devices/device-preview-button"; import { ShimmeringWorkContent } from "./thread-work-log"; const CONTROL_HEIGHT = 38.5; // h-11 with the mobile 14px rem @@ -67,12 +68,18 @@ export const FLOATING_WORKING_CONTROL_COVERAGE = CONTROL_OVERLAY_OFFSET + CONTRO export function FloatingWorkingControl(props: { readonly colorScheme: "light" | "dark"; readonly status: FloatingWorkingStatus | null; + readonly devicePreview: { readonly count: number; readonly onPress: () => void } | null; readonly showScrollToEnd: boolean; readonly onScrollToEnd: () => void; }) { const { width: windowWidth } = useWindowDimensions(); const [overlayWidth, setOverlayWidth] = useState(windowWidth); - const labelWidth = Math.max(0, Math.min(overlayWidth, windowWidth) - CONTROL_HEIGHT - 16); + const deviceControlWidth = props.devicePreview !== null ? CONTROL_HEIGHT : 0; + const hasCapsule = props.status !== null || props.devicePreview !== null; + const labelWidth = Math.max( + 0, + Math.min(overlayWidth, windowWidth) - CONTROL_HEIGHT - 16 - deviceControlWidth, + ); const separationProgress = useSharedValue(props.showScrollToEnd ? 1 : 0); useEffect(() => { @@ -115,25 +122,30 @@ export function FloatingWorkingControl(props: { })); // Zero until the first measurement lands, so the capsule never paints around // a label it has not sized to yet. - const capsuleSizerStyle = useAnimatedStyle(() => ({ width: capsuleWidth.value ?? 0 })); + const capsuleSizerStyle = useAnimatedStyle(() => ({ + width: (capsuleWidth.value ?? 0) + deviceControlWidth, + })); - if (props.status === null && !props.showScrollToEnd) { + if (!hasCapsule && !props.showScrollToEnd) { return null; } // Only the connection label is a button (tap to reconnect); the others // pass touches through to the feed like before. const statusInteractive = props.status?.kind === "connection"; + const capsuleInteractive = statusInteractive || props.devicePreview !== null; // The host stays centered on the capsule, but its measurement constraint // comes from the overlay, independent of the capsule's current width. - const statusContent = - props.status !== null ? ( + const capsuleContent = + props.status === null && props.devicePreview !== null ? ( + + ) : props.status !== null ? ( <> + {props.devicePreview !== null ? ( + <> + + + + + + ) : null} ) : null; @@ -157,7 +181,7 @@ export function FloatingWorkingControl(props: { entering={NATIVE_LIQUID_GLASS_SUPPORTED ? undefined : CONTROL_ENTERING} exiting={NATIVE_LIQUID_GLASS_SUPPORTED ? undefined : CONTROL_EXITING} > - {props.status !== null && NATIVE_LIQUID_GLASS_SUPPORTED ? ( + {hasCapsule && NATIVE_LIQUID_GLASS_SUPPORTED ? ( - {statusContent} + {capsuleContent} - ) : props.status !== null ? ( + ) : hasCapsule ? ( - {statusContent} + {capsuleContent} + connectionAtomRuntime + .atom((get) => { + const connectionResult = get(environmentCatalog.stateAtom(environmentId)); + const connection = Option.getOrNull(AsyncResult.value(connectionResult)); + // Follow credential changes as a trigger, but never trust this stream's + // waitingFrom value as the ticket's identity. Read the live supervisor. + get(environmentSession.preparedConnectionValueAtom(environmentId)); + if (connectionResult.waiting || connection?.phase !== "connected") return Effect.never; + return EnvironmentRegistry.pipe( + Effect.flatMap((registry) => + registry.run( + environmentId, + EnvironmentSupervisor.pipe( + Effect.flatMap((supervisor) => + Effect.all([ + SubscriptionRef.get(supervisor.state), + SubscriptionRef.get(supervisor.prepared), + ]).pipe( + Effect.flatMap(([liveState, livePrepared]) => + liveState.phase === "connected" && + liveState.generation === connection.generation && + Option.isSome(livePrepared) + ? resolveDeviceHubAccess({ + prepared: livePrepared.value, + hubBasePath: "/api/device-hub", + }).pipe( + Effect.map((access) => ({ + prepared: livePrepared.value, + generation: liveState.generation, + access, + })), + ) + : Effect.never, + ), + ), + ), + ), + ), + ), + ); + }) + .pipe(Atom.setIdleTTL(60_000), Atom.withLabel(`mobile-device-hub-access:${environmentId}`)), +); + +export function refreshDeviceHubAccess(environmentId: EnvironmentId) { + appAtomRegistry.refresh(deviceHubAccessAtom(environmentId)); +} + +export function useDeviceHubAccess(environmentId: EnvironmentId, hostId: string) { + const prepared = usePreparedConnection(environmentId); + const connectionResult = useAtomValue(environmentCatalog.stateAtom(environmentId)); + const query = useEnvironmentQuery(deviceHubAccessAtom(environmentId)); + const access = useMemo( + () => + currentDeviceHubAccess({ + prepared: Option.getOrNull(prepared), + connection: Option.getOrNull(AsyncResult.value(connectionResult)), + connectionPending: connectionResult.waiting, + reading: query.data, + readingPending: query.isPending, + readingError: query.error, + hostId, + }), + [query.data, query.error, query.isPending, prepared, connectionResult, hostId], + ); + return { access, error: query.error, refresh: query.refresh }; +} diff --git a/apps/mobile/src/types/mobile-device-stream.d.ts b/apps/mobile/src/types/mobile-device-stream.d.ts new file mode 100644 index 0000000000..eddedfac36 --- /dev/null +++ b/apps/mobile/src/types/mobile-device-stream.d.ts @@ -0,0 +1,4 @@ +declare module "@t3tools/mobile-device-stream" { + const script: string; + export default script; +} diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index a4c73bbd7d..b9e60528e1 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -13,7 +13,7 @@ import { type DeviceScreenSize, type DeviceStreamClient, type DeviceStreamStatus, -} from "./deviceStream"; +} from "@t3tools/client-runtime/device/stream"; const AX_POLL_INTERVAL_MS = 2_000; diff --git a/apps/web/src/components/device/deviceStream.test.ts b/apps/web/src/components/device/deviceStream.test.ts index 530773575b..86d81e8644 100644 --- a/apps/web/src/components/device/deviceStream.test.ts +++ b/apps/web/src/components/device/deviceStream.test.ts @@ -6,7 +6,7 @@ import { avcCodecString, parseSemuPacket, scanAccessUnit, -} from "./deviceStream"; +} from "@t3tools/client-runtime/device/stream"; const envelope = (tag: number, payload: number[]) => { const length = 1 + payload.length; diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index 3d262b3081..53fac8470e 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -36,7 +36,7 @@ import { useRightPanelStore } from "~/rightPanelStore"; import { useDeviceState } from "~/state/device"; import { DeviceStreamView } from "../device/DeviceStreamView"; -import type { DeviceScreenSize } from "../device/deviceStream"; +import type { DeviceScreenSize } from "@t3tools/client-runtime/device/stream"; import { previewBridge } from "./previewBridge"; import { clampPreviewMiniPlayerPosition, diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.ts index 13ba26f06f..faa529ae91 100644 --- a/apps/web/src/components/preview/previewMiniPlayerLayout.ts +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.ts @@ -7,7 +7,7 @@ import { } from "~/browser/browserViewportLayout"; import type { PreviewMiniPlayerPosition, PreviewMiniPlayerSize } from "~/previewMiniPlayerStore"; -import type { DeviceScreenSize } from "../device/deviceStream"; +import type { DeviceScreenSize } from "@t3tools/client-runtime/device/stream"; export const PREVIEW_MINI_PLAYER_EDGE_GAP = 12; export const PREVIEW_MINI_PLAYER_CORNER_RADIUS = 12; diff --git a/docs/user/devices.md b/docs/user/devices.md index fb26486e96..25948d7a52 100644 --- a/docs/user/devices.md +++ b/docs/user/devices.md @@ -55,9 +55,18 @@ back from the device after a change. When an agent opens a device, it floats over the chat in web and desktop clients connected to the thread, the same way an agent-driven browser does. Turn off **Auto-show floating preview** in **Settings → Integrations → Browser** to open a -right-panel tab instead. Mobile clients show device activity in the thread -timeline. Agents drive the device through the `agent-device` command line. Pylon -Code installs and starts it only after **Agent device access** is enabled. iOS +right-panel tab instead. In the mobile app, open the agent's thread and tap the +device button above the composer to watch the live screen and control it. +If the thread has several devices open, choose one in the viewer. Closing the +viewer stops streaming and leaves the device available to the agent. Device +activity also appears in the thread timeline. +Android video in the mobile viewer needs a secure WebView context: use HTTPS for +remote environments, or a loopback address for local connections. If decoding +is unavailable, the viewer shows an error instead of a blank screen. iOS can +use its MJPEG stream when WebCodecs is unavailable. + +Agents drive the device through the `agent-device` command line. Pylon Code +installs and starts it only after **Agent device access** is enabled. iOS taps build a small test runner on first use, which takes a couple of minutes once per server. Restart an existing agent session after granting access so it receives the device CLI environment. @@ -98,6 +107,6 @@ localhost without forwarding or another reachable address. ## Device tool updates -The connected T3 server manages the device hub and agent tools on its own machine and configured SSH hosts. Required versions install automatically the next time those tools are used. Settings → Integrations → Check device tool versions reads installed versions without installing tools or starting devices. +The connected Pylon server manages the device hub and agent tools on its own machine and configured SSH hosts. Required versions install automatically the next time those tools are used. Settings → Integrations → Check device tool versions reads installed versions without installing tools or starting devices. -To receive newer tool versions on a remote environment, update that environment's T3 server. Updating only the browser or mobile app does not update the remote server. An offline host keeps its installed files, but an update needs network access before device support can start; T3 does not fall back to an older version. Reconnect the host and use Retry if installation fails. Existing device and agent-access settings are preserved. +To receive newer tool versions on a remote environment, update that environment's Pylon server. Updating only the browser or mobile app does not update the remote server. An offline host keeps its installed files, but an update needs network access before device support can start; Pylon does not fall back to an older version. Reconnect the host and use Retry if installation fails. Existing device and agent-access settings are preserved. diff --git a/knip.jsonc b/knip.jsonc index 2a0cb036e6..500d599c5b 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -65,8 +65,13 @@ "ignoreIssues": { "src/components/ui/*.tsx": ["exports", "nsExports", "duplicates"] }, }, "apps/mobile": { - // Expo loads local config plugins by string; Metro handles platform variants. - "entry": ["index.ts!", "plugins/*.cjs"], + // Expo loads plugins by string; Metro embeds the browser entry in the native WebView. + "entry": [ + "index.ts!", + "plugins/*.cjs", + "scripts/generate-device-stream.mts", + "src/features/devices/device-stream.browser.ts!", + ], // Fonts are configured as asset paths; Expo autolinks the native packages. "ignoreDependencies": [ "@expo-google-fonts/dm-sans", diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index dc4bced3ab..b9f7427239 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -3,6 +3,14 @@ "private": true, "type": "module", "exports": { + "./device/stream": { + "types": "./src/device/stream.ts", + "default": "./src/device/stream.ts" + }, + "./device/hub-access": { + "types": "./src/device/hubAccess.ts", + "default": "./src/device/hubAccess.ts" + }, "./worktree-setup": { "types": "./src/worktreeSetup.ts", "default": "./src/worktreeSetup.ts" diff --git a/packages/client-runtime/src/device/hubAccess.ts b/packages/client-runtime/src/device/hubAccess.ts new file mode 100644 index 0000000000..9c010a2583 --- /dev/null +++ b/packages/client-runtime/src/device/hubAccess.ts @@ -0,0 +1,18 @@ +/** Credentials for media requests that cannot set bearer or DPoP headers. */ +export interface DeviceHubAccess { + /** Absolute environment URL ending in `/api/device-hub`. */ + readonly httpBase: string; + /** Same base with the `ws(s)` scheme. */ + readonly wsBase: string; + /** Empty for cookie sessions; includes a short-lived ticket for bearer and DPoP sessions. */ + readonly query: Readonly>; + /** Whether requests must include session cookies. */ + readonly credentials: boolean; +} + +export const withDeviceHubQuery = (url: string, access: DeviceHubAccess): string => { + const entries = Object.entries(access.query); + if (entries.length === 0) return url; + const separator = url.includes("?") ? "&" : "?"; + return `${url}${separator}${new URLSearchParams(entries).toString()}`; +}; diff --git a/packages/client-runtime/src/device/stream.test.ts b/packages/client-runtime/src/device/stream.test.ts new file mode 100644 index 0000000000..a4757f620f --- /dev/null +++ b/packages/client-runtime/src/device/stream.test.ts @@ -0,0 +1,773 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { + createDeviceStreamClient, + AvccDemuxer, + avcCodecString, + parseSemuPacket, + scanAccessUnit, +} from "./stream.ts"; + +const envelope = (tag: number, payload: number[]) => { + const length = 1 + payload.length; + return [ + (length >>> 24) & 0xff, + (length >>> 16) & 0xff, + (length >>> 8) & 0xff, + length & 0xff, + tag, + ...payload, + ]; +}; + +describe("AvccDemuxer", () => { + it("reassembles envelopes split across reads", () => { + const demuxer = new AvccDemuxer(); + const bytes = new Uint8Array([ + ...envelope(1, [1, 0x64, 0x00, 0x1f]), + ...envelope(2, [9, 9, 9]), + ...envelope(0x7f, [0]), + ...envelope(3, [4]), + ]); + const first = demuxer.push(bytes.subarray(0, 7)); + const rest = demuxer.push(bytes.subarray(7)); + const chunks = [...first, ...rest]; + expect(chunks.map((chunk) => chunk.type)).toEqual(["description", "keyframe", "delta"]); + expect(Array.from(chunks[0]!.payload)).toEqual([1, 0x64, 0x00, 0x1f]); + expect(Array.from(chunks[1]!.payload)).toEqual([9, 9, 9]); + }); + + it("derives the WebCodecs codec string from the avcC record", () => { + expect(avcCodecString(new Uint8Array([1, 0x64, 0x00, 0x1f]))).toBe("avc1.64001f"); + expect(avcCodecString(new Uint8Array([1]))).toBe("avc1.42E01E"); + }); +}); + +describe("native device stream transport", () => { + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + function setup(platform: "ios" | "android") { + vi.useFakeTimers(); + vi.stubGlobal("VideoDecoder", vi.fn()); + vi.stubGlobal("EncodedVideoChunk", vi.fn()); + let resolveOpened!: (socket: FakeSocket) => void; + const waitForSocket = () => + new Promise((resolve) => { + resolveOpened = resolve; + }); + const opened = waitForSocket(); + class FakeSocket { + static OPEN = 1; + readyState = 1; + binaryType = ""; + onopen: (() => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = null; + send = vi.fn(); + close = vi.fn(); + readonly url: string; + constructor(url: string) { + this.url = url; + resolveOpened(this); + } + } + vi.stubGlobal("WebSocket", FakeSocket); + const fetch = vi.fn(() => Promise.resolve(new Response("frame"))); + vi.stubGlobal("fetch", fetch); + const events = { + onStatus: vi.fn(), + onScreen: vi.fn(), + onUnauthorized: vi.fn(), + onMjpegFallback: vi.fn(), + onInputConnected: vi.fn(), + }; + const client = createDeviceStreamClient( + { + platform, + deviceId: "test device", + preferMjpeg: platform === "ios", + access: { + httpBase: "https://environment.test/api/device-hub", + wsBase: "wss://environment.test/api/device-hub", + credentials: false, + query: { wsTicket: "stream-ticket", hostId: "ssh-host" }, + }, + }, + { getContext: () => null } as unknown as HTMLCanvasElement, + events, + ); + return { client, opened, waitForSocket, events, fetch }; + } + + it("reports an unavailable Android decoder instead of leaving a blank viewer", () => { + vi.stubGlobal("VideoDecoder", undefined); + vi.stubGlobal("EncodedVideoChunk", undefined); + Reflect.deleteProperty(globalThis, "VideoDecoder"); + Reflect.deleteProperty(globalThis, "EncodedVideoChunk"); + const events = { + onStatus: vi.fn(), + onScreen: vi.fn(), + onUnauthorized: vi.fn(), + onMjpegFallback: vi.fn(), + onInputConnected: vi.fn(), + }; + const client = createDeviceStreamClient( + { + platform: "android", + deviceId: "remote-device", + access: { + httpBase: "http://remote.example/api/device-hub", + wsBase: "ws://remote.example/api/device-hub", + credentials: false, + query: { wsTicket: "ticket" }, + }, + }, + { getContext: () => null } as unknown as HTMLCanvasElement, + events, + ); + client.start(); + expect(events.onStatus).toHaveBeenLastCalledWith( + "error", + "This viewer cannot decode the Android stream (WebCodecs unavailable).", + ); + expect(events.onInputConnected).toHaveBeenLastCalledWith( + false, + "This viewer cannot decode the Android stream (WebCodecs unavailable).", + ); + client.stop(); + }); + + it("uses authenticated iOS MJPEG and forwards controls without a cross-origin video fetch", async () => { + const { client, opened, events, fetch } = setup("ios"); + client.start(); + const socket = await opened; + const imageUrl = new URL(events.onMjpegFallback.mock.calls[0]![0] as string); + expect(imageUrl.pathname).toContain("test%20device/stream.mjpeg"); + expect(imageUrl.searchParams.get("wsTicket")).toBe("stream-ticket"); + expect(imageUrl.searchParams.get("hostId")).toBe("ssh-host"); + expect(fetch).toHaveBeenCalledTimes(1); + socket.onopen?.(); + client.pressButton("home"); + client.sendTouch("begin", 0.25, 0.75); + const messages = socket.send.mock.calls.slice(1).map(([bytes]) => { + const packet = bytes as Uint8Array; + return { + tag: packet[0], + body: JSON.parse(new TextDecoder().decode(packet.subarray(1))) as unknown, + }; + }); + expect(messages).toEqual([ + { tag: 0x04, body: { button: "home" } }, + { tag: 0x03, body: { type: "begin", x: 0.25, y: 0.75 } }, + ]); + client.stop(); + expect(socket.close).toHaveBeenCalledTimes(1); + expect(vi.getTimerCount()).toBe(0); + }); + + it("forwards Android gestures and device controls through the ticketed host socket", async () => { + const { client, opened } = setup("android"); + client.start(); + const socket = await opened; + const url = new URL(socket.url); + expect(url.searchParams.get("device")).toBe("test device"); + expect(url.searchParams.get("wsTicket")).toBe("stream-ticket"); + expect(url.searchParams.get("hostId")).toBe("ssh-host"); + client.sendTouch("begin", 0.2, 0.8); + client.sendTouch("move", 0.2, 0.4); + client.sendTouch("end", 0.2, 0.1); + client.pressButton("home"); + client.pressButton("appSwitcher"); + expect( + socket.send.mock.calls.map(([message]) => JSON.parse(message as string) as unknown), + ).toEqual([ + { type: "touch", action: "down", x: 0.2, y: 0.8 }, + { type: "touch", action: "move", x: 0.2, y: 0.4 }, + { type: "touch", action: "up", x: 0.2, y: 0.1 }, + { type: "home" }, + { type: "recents" }, + ]); + client.stop(); + expect(socket.close).toHaveBeenCalledTimes(1); + }); + + it("renews an expired Android ticket when the HTTP upgrade is rejected instead of retrying it forever", async () => { + const { client, opened, events } = setup("android"); + client.start(); + const socket = await opened; + socket.onclose?.({ code: 1006, reason: "" }); + expect(events.onUnauthorized).toHaveBeenCalledTimes(1); + expect(vi.getTimerCount()).toBe(0); + client.stop(); + }); + + it.each(["ios", "android"] as const)( + "keeps a restarted %s stream connected when the discarded socket closes late", + async (platform) => { + const { client, opened, waitForSocket, events } = setup(platform); + client.start(); + const discarded = await opened; + client.stop(); + const reopened = waitForSocket(); + client.start(); + const replacement = await reopened; + replacement.onopen?.(); + replacement.send.mockClear(); + events.onInputConnected.mockClear(); + + discarded.onclose?.({ code: 1006, reason: "" }); + + expect(events.onUnauthorized).not.toHaveBeenCalled(); + expect(events.onInputConnected).not.toHaveBeenCalled(); + expect(replacement.close).not.toHaveBeenCalled(); + client.pressButton("home"); + expect(replacement.send).toHaveBeenCalledTimes(1); + client.stop(); + expect(replacement.close).toHaveBeenCalledTimes(1); + expect(vi.getTimerCount()).toBe(0); + }, + ); + + it.each(["ios", "android"] as const)( + "does not renew credentials or reconnect a stopped %s stream", + async (platform) => { + const { client, opened, events } = setup(platform); + client.start(); + const socket = await opened; + events.onInputConnected.mockClear(); + client.stop(); + socket.onclose?.({ code: 1006, reason: "" }); + expect(events.onUnauthorized).not.toHaveBeenCalled(); + expect(events.onInputConnected).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }, + ); +}); + +describe("serve-emu frames", () => { + it("strips the SEMU header and reads the keyframe flag and timestamp", () => { + const buffer = new ArrayBuffer(16 + 3); + const view = new DataView(buffer); + view.setUint32(0, 0x53454d55); + view.setUint8(4, 1); + view.setUint8(5, 1); + view.setBigUint64(8, 123456n); + new Uint8Array(buffer).set([7, 8, 9], 16); + const packet = parseSemuPacket(buffer); + expect(packet.isKey).toBe(true); + expect(packet.timestamp).toBe(123456); + expect(Array.from(packet.data)).toEqual([7, 8, 9]); + }); + + it("treats a frame without the header as raw data", () => { + const packet = parseSemuPacket(new Uint8Array([0, 0, 1, 0x65]).buffer); + expect(packet.isKey).toBeNull(); + expect(packet.data.length).toBe(4); + }); + + it("finds the SPS and IDR NAL units in an Annex-B access unit", () => { + const unit = new Uint8Array([0, 0, 0, 1, 0x67, 0x64, 0x00, 0x1f, 0, 0, 1, 0x65, 0xaa]); + const scanned = scanAccessUnit(unit); + expect(scanned.isKey).toBe(true); + expect(scanned.sps && avcCodecString(scanned.sps)).toBe("avc1.64001f"); + expect(scanAccessUnit(new Uint8Array([0, 0, 1, 0x41, 0x00])).isKey).toBe(false); + }); +}); + +describe("iOS input startup", () => { + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + const setup = () => { + vi.useFakeTimers(); + const sockets: FakeSocket[] = []; + class FakeSocket { + static OPEN = 1; + readyState = 1; + binaryType = ""; + onopen: (() => void) | null = null; + onmessage: ((event: { data: ArrayBuffer }) => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = null; + send = vi.fn(); + close = vi.fn(); + constructor() { + sockets.push(this); + } + } + vi.stubGlobal("WebSocket", FakeSocket); + const signals: AbortSignal[] = []; + vi.stubGlobal( + "fetch", + vi.fn((_url, init: RequestInit) => { + const signal = init.signal!; + signals.push(signal); + return Promise.resolve( + new Response( + new ReadableStream({ + start(controller) { + signal.addEventListener("abort", () => controller.error(new Error("aborted"))); + }, + }), + ), + ); + }), + ); + const client = createDeviceStreamClient( + { + platform: "ios", + deviceId: "test-device", + access: { + httpBase: "http://test/api/device-hub", + wsBase: "ws://test/api/device-hub", + credentials: true, + query: {}, + }, + }, + { getContext: () => null } as unknown as HTMLCanvasElement, + { + onStatus: vi.fn(), + onScreen: vi.fn(), + onUnauthorized: vi.fn(), + onMjpegFallback: vi.fn(), + onInputConnected: vi.fn(), + }, + ); + return { client, sockets, signals }; + }; + + it("connects input when the MJPEG prime never produces a frame", async () => { + const { client, sockets, signals } = setup(); + client.start(); + await vi.advanceTimersByTimeAsync(2_000); + expect(signals[0]?.aborted).toBe(true); + expect(sockets).toHaveLength(1); + client.stop(); + }); + + it("aborts priming immediately when hidden without opening a socket later", async () => { + const { client, sockets, signals } = setup(); + client.start(); + await vi.advanceTimersByTimeAsync(0); + client.stop(); + expect(signals[0]?.aborted).toBe(true); + await vi.advanceTimersByTimeAsync(2_000); + expect(sockets).toHaveLength(0); + }); + + it.each([ + ["landscape_left", 0.7, 1 - 0.2], + ["landscape_right", 1 - 0.7, 0.2], + ])("maps %s touches back to the raw iOS framebuffer", async (orientation, x, y) => { + const { client, sockets } = setup(); + client.start(); + await vi.advanceTimersByTimeAsync(2_000); + const socket = sockets[0]!; + const json = new TextEncoder().encode(JSON.stringify({ width: 400, height: 800, orientation })); + const packet = new Uint8Array(1 + json.length); + packet[0] = 0x82; + packet.set(json, 1); + socket.onmessage?.({ data: packet.buffer }); + client.sendTouch("begin", 0.2, 0.7); + const sent = socket.send.mock.calls[0]![0] as Uint8Array; + expect(JSON.parse(new TextDecoder().decode(sent.subarray(1)))).toEqual({ + type: "begin", + x, + y, + }); + client.stop(); + }); +}); + +class MjpegImage extends EventTarget { + naturalWidth = 0; + naturalHeight = 0; + src = ""; + removeAttribute(name: string) { + if (name === "src") { + this.src = ""; + this.naturalWidth = 0; + this.naturalHeight = 0; + } + } +} + +function recoveryFixture(platform: "ios" | "android" = "ios", preferMjpeg = true) { + vi.useFakeTimers(); + const sockets: Socket[] = []; + class Socket { + static OPEN = 1; + readyState = 1; + binaryType = ""; + onopen: (() => void) | null = null; + onmessage: ((event: { data: ArrayBuffer | string }) => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = null; + send = vi.fn(); + close = vi.fn(); + constructor() { + sockets.push(this); + } + } + vi.stubGlobal("WebSocket", Socket); + const decoders: Decoder[] = []; + class Decoder { + static isConfigSupported = vi.fn(async () => ({ supported: true })); + state = "unconfigured"; + decodeQueueSize = 0; + configure() { + this.state = "configured"; + } + close() { + this.state = "closed"; + } + decode() {} + readonly callbacks: { output: (frame: VideoFrame) => void; error: () => void }; + constructor(callbacks: { output: (frame: VideoFrame) => void; error: () => void }) { + this.callbacks = callbacks; + decoders.push(this); + } + } + vi.stubGlobal("VideoDecoder", Decoder); + vi.stubGlobal("EncodedVideoChunk", vi.fn()); + const videoBodies: ReadableStreamDefaultController[] = []; + const signals: AbortSignal[] = []; + const fetch = vi.fn((url: string, init: RequestInit) => { + const signal = init.signal!; + signals.push(signal); + if (!url.includes("stream.avcc")) return Promise.resolve(new Response("prime")); + return Promise.resolve( + new Response( + new ReadableStream({ + start(body) { + videoBodies.push(body); + signal.addEventListener("abort", () => body.error(new Error("aborted")), { + once: true, + }); + }, + }), + ), + ); + }); + vi.stubGlobal("fetch", fetch); + const events = { + onStatus: vi.fn(), + onScreen: vi.fn(), + onUnauthorized: vi.fn(), + onMjpegFallback: vi.fn(), + onInputConnected: vi.fn(), + }; + const drawImage = vi.fn(); + const image = new MjpegImage(); + const client = createDeviceStreamClient( + { + platform, + preferMjpeg, + deviceId: "test-device", + access: { + httpBase: "https://test/api/device-hub", + wsBase: "wss://test/api/device-hub", + credentials: true, + query: {}, + }, + }, + { getContext: () => ({ drawImage }) } as unknown as HTMLCanvasElement, + events, + ); + client.setMjpegImage(image as unknown as HTMLImageElement); + const imageReady = () => { + image.naturalWidth = 400; + image.naturalHeight = 800; + image.dispatchEvent(new Event("load")); + }; + const decodedFrame = (index = decoders.length - 1) => { + const close = vi.fn(); + decoders[index]!.callbacks.output({ + displayWidth: 400, + displayHeight: 800, + close, + } as unknown as VideoFrame); + return close; + }; + const sps = new Uint8Array([0, 0, 0, 1, 0x67, 0x64, 0, 0x1f, 0, 0, 1, 0x65]).buffer; + return { + client, + events, + image, + imageReady, + sockets, + signals, + fetch, + videoBodies, + decoders, + Decoder, + drawImage, + decodedFrame, + sps, + }; +} + +describe("shared device stream readiness and recovery", () => { + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + it("waits for actual MJPEG dimensions without requiring a multipart load event", async () => { + const { client, events, image } = recoveryFixture(); + client.start(); + await vi.advanceTimersByTimeAsync(0); + expect(image.src).toContain("stream.mjpeg"); + expect(events.onStatus).not.toHaveBeenCalledWith("streaming", undefined); + image.naturalWidth = 400; + image.naturalHeight = 800; + await vi.advanceTimersByTimeAsync(250); + expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined); + expect(vi.getTimerCount()).toBe(0); + await vi.advanceTimersByTimeAsync(20_000); + expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined); + client.stop(); + }); + + it("observes a web image attached after MJPEG fallback has already begun", async () => { + const { client, image, imageReady, events } = recoveryFixture(); + client.setMjpegImage(null); + client.start(); + expect(image.src).toBe(""); + await vi.advanceTimersByTimeAsync(1000); + client.setMjpegImage(image as unknown as HTMLImageElement); + imageReady(); + expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined); + client.stop(); + expect(image.src).toBe(""); + expect(vi.getTimerCount()).toBe(0); + }); + + it("stops input and video on an image error without treating it as expired credentials", async () => { + const { client, image, imageReady, sockets, events } = recoveryFixture(); + client.start(); + await vi.advanceTimersByTimeAsync(0); + sockets[0]!.onopen?.(); + imageReady(); + image.dispatchEvent(new Event("error")); + expect(events.onStatus).toHaveBeenLastCalledWith("error", expect.stringContaining("Reconnect")); + expect(events.onInputConnected).toHaveBeenLastCalledWith(false, expect.any(String)); + expect(events.onUnauthorized).not.toHaveBeenCalled(); + expect(sockets[0]!.close).toHaveBeenCalledOnce(); + expect(image.src).toBe(""); + client.pressButton("home"); + expect(sockets[0]!.send).toHaveBeenCalledTimes(1); + expect(vi.getTimerCount()).toBe(0); + }); + + it.each([ + ["ios", true], + ["ios", false], + ["android", false], + ] as const)( + "bounds missing first frames for %s with preferMjpeg=%s", + async (platform, preferMjpeg) => { + const { client, events, signals, sockets, image } = recoveryFixture(platform, preferMjpeg); + client.start(); + await vi.advanceTimersByTimeAsync(14_999); + expect(events.onStatus.mock.calls.some(([status]) => status === "error")).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(events.onStatus).toHaveBeenLastCalledWith( + "error", + expect.stringContaining("No video"), + ); + expect(events.onInputConnected).toHaveBeenLastCalledWith(false, expect.any(String)); + expect(signals.every((signal) => signal.aborted)).toBe(true); + expect(sockets[0]!.close).toHaveBeenCalledOnce(); + expect(image.src).toBe(""); + expect(vi.getTimerCount()).toBe(0); + }, + ); + + it("does not extend the first-frame deadline each time a socket reconnects", async () => { + const { client, sockets, events } = recoveryFixture("android"); + client.start(); + sockets[0]!.onopen?.(); + sockets[0]!.onclose?.({ code: 1000, reason: "restart" }); + await vi.advanceTimersByTimeAsync(14_999); + sockets[1]!.onopen?.(); + await vi.advanceTimersByTimeAsync(1); + expect(events.onStatus).toHaveBeenLastCalledWith("error", expect.stringContaining("No video")); + expect(sockets[1]!.close).toHaveBeenCalledOnce(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("gives a manual restart a fresh deadline and ignores queued image events from its old binding", async () => { + const { client, image, events } = recoveryFixture(); + const listeners: EventListener[] = []; + const add = image.addEventListener.bind(image); + vi.spyOn(image, "addEventListener").mockImplementation((name, callback, options) => { + if (typeof callback === "function") listeners.push(callback); + add(name, callback, options); + }); + client.start(); + await vi.advanceTimersByTimeAsync(10_000); + client.stop(); + client.start(); + events.onStatus.mockClear(); + for (const listener of listeners.slice(0, 2)) listener(new Event("error")); + await vi.advanceTimersByTimeAsync(10_000); + expect(events.onStatus).not.toHaveBeenCalled(); + const replacement = new MjpegImage(); + client.setMjpegImage(replacement as unknown as HTMLImageElement); + replacement.naturalWidth = 400; + replacement.naturalHeight = 800; + replacement.dispatchEvent(new Event("load")); + expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined); + client.stop(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("falls back from unsupported iOS H.264 without publishing a terminal error", async () => { + const { client, videoBodies, Decoder, events, imageReady } = recoveryFixture("ios", false); + Decoder.isConfigSupported.mockResolvedValue({ supported: false }); + client.start(); + await vi.advanceTimersByTimeAsync(0); + videoBodies[0]!.enqueue(new Uint8Array(envelope(1, [1, 0x64, 0, 0x1f]))); + await vi.advanceTimersByTimeAsync(0); + expect(events.onMjpegFallback).toHaveBeenCalledOnce(); + expect(events.onStatus.mock.calls.some(([status]) => status === "error")).toBe(false); + imageReady(); + expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined); + client.stop(); + }); + + it("publishes a terminal unsupported-profile error for Android and closes its socket", async () => { + const { client, sockets, sps, Decoder, events } = recoveryFixture("android"); + Decoder.isConfigSupported.mockResolvedValue({ supported: false }); + client.start(); + sockets[0]!.onmessage?.({ data: sps }); + await vi.advanceTimersByTimeAsync(0); + expect(events.onStatus).toHaveBeenLastCalledWith( + "error", + expect.stringContaining("cannot decode"), + ); + expect(sockets[0]!.close).toHaveBeenCalledOnce(); + expect(events.onMjpegFallback).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("waits for decoded Android output and bounds a rotation that never produces another frame", async () => { + const { client, sockets, sps, events, decodedFrame, drawImage } = recoveryFixture("android"); + client.start(); + sockets[0]!.onmessage?.({ data: sps }); + await vi.advanceTimersByTimeAsync(0); + expect(events.onStatus.mock.calls.some(([status]) => status === "streaming")).toBe(false); + expect(decodedFrame()).toHaveBeenCalledOnce(); + expect(drawImage).toHaveBeenCalledOnce(); + expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined); + sockets[0]!.onmessage?.({ data: JSON.stringify({ type: "video-session" }) }); + await vi.advanceTimersByTimeAsync(15_000); + expect(events.onStatus).toHaveBeenLastCalledWith("error", expect.stringContaining("No video")); + expect(vi.getTimerCount()).toBe(0); + }); + + it("discards decoder support results and output from a previous Android socket", async () => { + const { client, sockets, sps, events, Decoder, decoders, decodedFrame, drawImage } = + recoveryFixture("android"); + let resolveSupport!: (support: { supported: boolean }) => void; + Decoder.isConfigSupported.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveSupport = resolve; + }), + ); + client.start(); + sockets[0]!.onmessage?.({ data: sps }); + client.stop(); + client.start(); + resolveSupport({ supported: true }); + await vi.advanceTimersByTimeAsync(0); + expect(decoders).toHaveLength(0); + sockets[1]!.onmessage?.({ data: sps }); + await vi.advanceTimersByTimeAsync(0); + client.stop(); + client.start(); + events.onStatus.mockClear(); + expect(decodedFrame()).toHaveBeenCalledOnce(); + expect(drawImage).not.toHaveBeenCalled(); + expect(events.onStatus).not.toHaveBeenCalled(); + client.stop(); + }); + + it("ignores an old AVCC response rejected after a new attempt is connected", async () => { + const { client, fetch, events } = recoveryFixture("ios", false); + let resolveResponse!: (response: Response) => void; + fetch.mockImplementationOnce(() => Promise.resolve(new Response("prime"))); + fetch.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveResponse = resolve; + }), + ); + client.start(); + await vi.advanceTimersByTimeAsync(0); + client.stop(); + client.start(); + await vi.advanceTimersByTimeAsync(0); + events.onStatus.mockClear(); + resolveResponse(new Response(null, { status: 401 })); + await vi.advanceTimersByTimeAsync(0); + expect(events.onUnauthorized).not.toHaveBeenCalled(); + expect(events.onStatus).not.toHaveBeenCalled(); + client.stop(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("closes a delayed JPEG seed from a stopped AVCC attempt without painting it", async () => { + const { client, videoBodies, drawImage, events } = recoveryFixture("ios", false); + let resolveBitmap!: (bitmap: ImageBitmap) => void; + vi.stubGlobal( + "createImageBitmap", + () => + new Promise((resolve) => { + resolveBitmap = resolve; + }), + ); + client.start(); + await vi.advanceTimersByTimeAsync(0); + videoBodies[0]!.enqueue(new Uint8Array(envelope(4, [1]))); + await vi.advanceTimersByTimeAsync(0); + client.stop(); + client.start(); + events.onStatus.mockClear(); + const close = vi.fn(); + resolveBitmap({ width: 400, height: 800, close } as unknown as ImageBitmap); + await vi.advanceTimersByTimeAsync(0); + expect(close).toHaveBeenCalledOnce(); + expect(drawImage).not.toHaveBeenCalled(); + expect(events.onStatus).not.toHaveBeenCalled(); + client.stop(); + }); +}); + +it("reports a stalled AVCC body after its initial image instead of leaving a frozen streaming state", async () => { + const { client, videoBodies, events, signals } = recoveryFixture("ios", false); + const close = vi.fn(); + vi.stubGlobal("createImageBitmap", () => Promise.resolve({ width: 400, height: 800, close })); + try { + client.start(); + await vi.advanceTimersByTimeAsync(0); + videoBodies[0]!.enqueue(new Uint8Array(envelope(4, [1]))); + await vi.advanceTimersByTimeAsync(0); + expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined); + await vi.advanceTimersByTimeAsync(15_000); + expect(events.onStatus).toHaveBeenLastCalledWith( + "error", + expect.stringContaining("stopped receiving video"), + ); + expect(signals.every((signal) => signal.aborted)).toBe(true); + expect(vi.getTimerCount()).toBe(0); + } finally { + client.stop(); + vi.useRealTimers(); + vi.unstubAllGlobals(); + } +}); diff --git a/apps/web/src/components/device/deviceStream.ts b/packages/client-runtime/src/device/stream.ts similarity index 98% rename from apps/web/src/components/device/deviceStream.ts rename to packages/client-runtime/src/device/stream.ts index 2faa2445bb..21da0b74ee 100644 --- a/apps/web/src/components/device/deviceStream.ts +++ b/packages/client-runtime/src/device/stream.ts @@ -1,6 +1,9 @@ +// @effect-diagnostics globalFetch:off globalTimers:off - This browser and WebView transport runs without an Effect runtime. +/* oxlint-disable unicorn/prefer-add-event-listener -- Each client owns its sockets and their handlers. */ + /** * Framework-free client for expo-device-hub's per-device streams, reached - * through the T3 proxy. One class handles both platforms because the hub + * through the Pylon proxy. One class handles both platforms because the hub * vendors two servers with different wire formats: * * - iOS (serve-sim): video is an HTTP `stream.avcc` body of length-prefixed @@ -16,8 +19,7 @@ * The decoder only runs while frames arrive and the viewer is attached; a * hidden panel calls `stop()` so an idle device costs nothing on the GPU. */ -import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; -import { withDeviceHubQuery } from "@t3tools/client-runtime/state/deviceHubAccess"; +import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts"; import type { DevicePlatform } from "@t3tools/contracts"; export type DeviceStreamStatus = "connecting" | "streaming" | "error"; @@ -461,7 +463,7 @@ export function createDeviceStreamClient( const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false })); if (!isCurrent() || epoch !== decoderEpoch) return false; if (!support.supported) { - if (platform === "android") fail(`This browser cannot decode ${config.codec}.`); + if (platform === "android") fail(`This viewer cannot decode ${config.codec}.`); return false; } try { @@ -761,7 +763,7 @@ export function createDeviceStreamClient( } else if (useWebCodecs) { connectAndroid(); } else { - fail("This browser cannot decode the Android stream (WebCodecs unavailable)."); + fail("This viewer cannot decode the Android stream (WebCodecs unavailable)."); } }; diff --git a/packages/client-runtime/src/state/deviceHubAccess.ts b/packages/client-runtime/src/state/deviceHubAccess.ts index 4c87aa538a..9ab65b25e6 100644 --- a/packages/client-runtime/src/state/deviceHubAccess.ts +++ b/packages/client-runtime/src/state/deviceHubAccess.ts @@ -20,19 +20,11 @@ import { environmentEndpointUrl } from "../environment/endpoint.ts"; import { ManagedRelayDpopSigner } from "../relay/managedRelay.ts"; import type { RemoteEnvironmentRequestError } from "../rpc/http.ts"; import { executeAuthenticatedEnvironmentHttpRequest } from "./environmentHttpAuth.ts"; +import type { DeviceHubAccess } from "../device/hubAccess.ts"; -const TICKET_TIMEOUT_MS = 8_000; +export { type DeviceHubAccess, withDeviceHubQuery } from "../device/hubAccess.ts"; -export interface DeviceHubAccess { - /** Absolute origin-relative base, e.g. `https://env.example/api/device-hub`. */ - readonly httpBase: string; - /** Same base with the `ws(s)` scheme. */ - readonly wsBase: string; - /** Query parameters to append to every hub request; empty for cookie sessions. */ - readonly query: Readonly>; - /** Whether requests must include cookies (same-origin session). */ - readonly credentials: boolean; -} +const TICKET_TIMEOUT_MS = 8_000; export const resolveDeviceHubAccess = Effect.fn("clientRuntime.state.resolveDeviceHubAccess")( function* (input: { @@ -64,10 +56,3 @@ export const resolveDeviceHubAccess = Effect.fn("clientRuntime.state.resolveDevi }; }, ); - -export const withDeviceHubQuery = (url: string, access: DeviceHubAccess): string => { - const entries = Object.entries(access.query); - if (entries.length === 0) return url; - const separator = url.includes("?") ? "&" : "?"; - return `${url}${separator}${new URLSearchParams(entries).toString()}`; -}; From 7ebae6929e28c67ec03b5e32d6ca903a5d3df740 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Sat, 26 Sep 2026 18:02:40 -0600 Subject: [PATCH 2/2] fix(mobile): show session-scoped device previews and iOS controls --- apps/mobile/src/Stack.tsx | 2 +- .../devices/deviceHubAccessScope.test.ts | 32 ++++++++--- .../features/devices/deviceHubAccessScope.ts | 20 ++++--- apps/mobile/src/state/device.ts | 57 ++++++++++++++++--- 4 files changed, 86 insertions(+), 25 deletions(-) diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index 930e61f66b..7f1877f818 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -548,7 +548,7 @@ const RootStackConfig = createNativeStackNavigator({ linking: `${THREAD_LINKING_PREFIX}/devices`, options: { presentation: "fullScreenModal", - headerShown: false, + headerShown: Platform.OS === "ios", gestureEnabled: false, }, }), diff --git a/apps/mobile/src/features/devices/deviceHubAccessScope.test.ts b/apps/mobile/src/features/devices/deviceHubAccessScope.test.ts index 0621da75ab..0a2607eb58 100644 --- a/apps/mobile/src/features/devices/deviceHubAccessScope.test.ts +++ b/apps/mobile/src/features/devices/deviceHubAccessScope.test.ts @@ -35,7 +35,6 @@ describe("mobile device access scope", () => { currentDeviceHubAccess({ prepared: input.prepared, connection: connected(input.generation), - connectionPending: false, reading: input.reading, readingPending: input.pending ?? false, readingError: null, @@ -63,22 +62,39 @@ describe("mobile device access scope", () => { }), ).not.toBeNull(); + const ownerA = {}; + const ownerB = {}; const oldPreview = { sessions: ["A"] }; + const readingAState = { state: oldPreview, sessionOwner: ownerA, generation: 1 }; + expect( + currentDeviceState({ + connection: connected(1), + sessionOwner: ownerA, + reading: readingAState, + stateError: null, + }), + ).toBe(oldPreview); + expect( + currentDeviceState({ + connection: connected(2), + sessionOwner: ownerA, + reading: readingAState, + stateError: null, + }), + ).toBeNull(); expect( currentDeviceState({ connection: connected(2), - connectionPending: false, - state: oldPreview, - statePending: true, + sessionOwner: ownerB, + reading: readingAState, stateError: null, }), ).toBeNull(); expect( currentDeviceState({ - connection: { ...connected(2), phase: "connecting" }, - connectionPending: false, - state: oldPreview, - statePending: false, + connection: { ...connected(1), phase: "connecting" }, + sessionOwner: ownerA, + reading: readingAState, stateError: null, }), ).toBeNull(); diff --git a/apps/mobile/src/features/devices/deviceHubAccessScope.ts b/apps/mobile/src/features/devices/deviceHubAccessScope.ts index d1a8058536..823e9ab878 100644 --- a/apps/mobile/src/features/devices/deviceHubAccessScope.ts +++ b/apps/mobile/src/features/devices/deviceHubAccessScope.ts @@ -11,14 +11,12 @@ export interface ScopedDeviceHubAccess { export function currentDeviceHubAccess(input: { readonly prepared: Prepared | null; readonly connection: SupervisorConnectionState | null; - readonly connectionPending: boolean; readonly reading: ScopedDeviceHubAccess | null; readonly readingPending: boolean; readonly readingError: string | null; readonly hostId: string; }): DeviceHubAccess | null { if ( - input.connectionPending || input.connection?.phase !== "connected" || input.prepared === null || input.readingPending || @@ -37,15 +35,19 @@ export function currentDeviceHubAccess(input: { export function currentDeviceState(input: { readonly connection: SupervisorConnectionState | null; - readonly connectionPending: boolean; - readonly state: State | null; - readonly statePending: boolean; + readonly sessionOwner: object | null; + readonly reading: { + readonly state: State; + readonly sessionOwner: object; + readonly generation: number; + } | null; readonly stateError: string | null; }): State | null { - return !input.connectionPending && - input.connection?.phase === "connected" && - !input.statePending && + return input.connection?.phase === "connected" && + input.sessionOwner !== null && + input.reading?.sessionOwner === input.sessionOwner && + input.reading.generation === input.connection.generation && input.stateError === null - ? input.state + ? input.reading.state : null; } diff --git a/apps/mobile/src/state/device.ts b/apps/mobile/src/state/device.ts index abe4417198..3a27fab61a 100644 --- a/apps/mobile/src/state/device.ts +++ b/apps/mobile/src/state/device.ts @@ -1,7 +1,8 @@ import { createDeviceEnvironmentAtoms } from "@t3tools/client-runtime/state/device"; import { resolveDeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; -import type { EnvironmentId } from "@t3tools/contracts"; +import { type EnvironmentId, WS_METHODS } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; import { useAtomValue } from "@effect/atom-react"; import { AsyncResult } from "effect/unstable/reactivity"; import * as Option from "effect/Option"; @@ -11,6 +12,9 @@ import { useMemo } from "react"; import { environmentCatalog } from "../connection/catalog"; import { EnvironmentRegistry, EnvironmentSupervisor } from "@t3tools/client-runtime/connection"; +import { subscribeDynamicWithSession } from "@t3tools/client-runtime/rpc"; +import { rpcSessionOwner } from "@t3tools/client-runtime/rpc/session-owner"; +import { createEnvironmentSubscriptionAtomFamily } from "@t3tools/client-runtime/state/runtime"; import { currentDeviceHubAccess, currentDeviceState, @@ -22,16 +26,56 @@ import { environmentSession, usePreparedConnection } from "./session"; export const deviceEnvironment = createDeviceEnvironmentAtoms(connectionAtomRuntime); +// A live subscription stays "waiting" after its first value. Bind that value to +// the exact RPC session and generation instead of treating waiting as loading. +const liveDeviceStateAtom = createEnvironmentSubscriptionAtomFamily(connectionAtomRuntime, { + label: "mobile-device-state-scoped", + subscribe: () => + Stream.unwrap( + EnvironmentSupervisor.pipe( + Effect.map((supervisor) => + subscribeDynamicWithSession(WS_METHODS.subscribeDeviceState, () => + Effect.succeed({}), + ).pipe( + Stream.mapEffect(([session, state]) => + Effect.all([ + SubscriptionRef.get(supervisor.session), + SubscriptionRef.get(supervisor.state), + ]).pipe( + Effect.map(([currentSession, connection]) => + Option.isSome(currentSession) && + currentSession.value === session && + connection.phase === "connected" + ? Option.some({ + state, + sessionOwner: rpcSessionOwner(session), + generation: connection.generation, + }) + : Option.none(), + ), + ), + ), + Stream.filter(Option.isSome), + Stream.map((value) => value.value), + ), + ), + ), + ), +}); + export function useLiveDeviceState(environmentId: EnvironmentId) { const connectionResult = useAtomValue(environmentCatalog.stateAtom(environmentId)); - const query = useEnvironmentQuery(deviceEnvironment.state({ environmentId, input: {} })); + const sessionOwnerResult = useAtomValue(environmentSession.rpcSessionOwnerAtom(environmentId)); + const sessionOwner = Option.getOrNull( + Option.getOrElse(AsyncResult.value(sessionOwnerResult), () => Option.none()), + ); + const query = useEnvironmentQuery(liveDeviceStateAtom({ environmentId, input: {} })); return { ...query, data: currentDeviceState({ connection: Option.getOrNull(AsyncResult.value(connectionResult)), - connectionPending: connectionResult.waiting, - state: query.data, - statePending: query.isPending, + sessionOwner, + reading: query.data, stateError: query.error, }), }; @@ -45,7 +89,7 @@ const deviceHubAccessAtom = Atom.family((environmentId: EnvironmentId) => // Follow credential changes as a trigger, but never trust this stream's // waitingFrom value as the ticket's identity. Read the live supervisor. get(environmentSession.preparedConnectionValueAtom(environmentId)); - if (connectionResult.waiting || connection?.phase !== "connected") return Effect.never; + if (connection?.phase !== "connected") return Effect.never; return EnvironmentRegistry.pipe( Effect.flatMap((registry) => registry.run( @@ -95,7 +139,6 @@ export function useDeviceHubAccess(environmentId: EnvironmentId, hostId: string) currentDeviceHubAccess({ prepared: Option.getOrNull(prepared), connection: Option.getOrNull(AsyncResult.value(connectionResult)), - connectionPending: connectionResult.waiting, reading: query.data, readingPending: query.isPending, readingError: query.error,