diff --git a/apps/mobile/src/App.tsx b/apps/mobile/src/App.tsx index 001c9d597cc8..645bc0b8a2fb 100644 --- a/apps/mobile/src/App.tsx +++ b/apps/mobile/src/App.tsx @@ -1,7 +1,7 @@ import * as Linking from "expo-linking"; import * as SplashScreen from "expo-splash-screen"; import { useEffect } from "react"; -import { StatusBar, View } from "react-native"; +import { StatusBar } from "react-native"; import { GestureHandlerRootView } from "react-native-gesture-handler"; import { KeyboardProvider } from "react-native-keyboard-controller"; import { SafeAreaProvider } from "react-native-safe-area-context"; @@ -23,6 +23,8 @@ import { OverlayPortalHost } from "./components/OverlayPortal"; import { shouldHandleAppLink } from "./lib/appLinking"; import { useMobileNavigationTheme } from "./lib/useMobileNavigationTheme"; import { SubscriptionUsageCoordinator } from "./widgets/SubscriptionUsageCoordinator"; +import { VoiceInputProvider } from "./features/voice-input/VoiceInputProvider"; +import { GlobalVoiceInputControl } from "./features/voice-input/GlobalVoiceInputControl"; import "../global.css"; @@ -76,22 +78,24 @@ function AppContent() { - - {/* The navigation theme drives the NATIVE header appearance: native-stack + + + {/* The navigation theme drives the NATIVE header appearance: native-stack forwards `dark` as the nav bar's overrideUserInterfaceStyle. Without this, React Navigation defaults to its light theme and every native header (glass buttons, title, materials) is forced light even when the system is in dark mode. */} - - - - - - - - {/* Anchored-menu overlays render here — in-window, so the + + + + + + + + {/* Anchored-menu overlays render here — in-window, so the keyboard stays up while a dropdown is open. */} - + + diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index 91b1d0923e6c..a3219888241a 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -15,7 +15,6 @@ import { import { useCallback, useEffect, - useId, useLayoutEffect, useMemo, useRef, @@ -63,6 +62,8 @@ import { } from "./scheduledTaskDraft"; import { settingsTargetsForProject } from "./settings-environment-filter.logic"; import { useScheduledTaskEditor } from "./scheduled-task-editor"; +import { scheduledTaskEditorSessionAtom } from "./scheduled-task-editor-state"; +import { appAtomRegistry } from "../../state/atom-registry"; import { formatNextScheduledTaskRun, formatScheduledTaskInterval, @@ -372,14 +373,21 @@ export function SettingsScheduledTaskEditRouteScreen() { } function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string }) { - const { editor, setEditor, hasChanges, draftForEnvironment } = useScheduledTaskEditor(); + const { + editor, + voiceOwnerKey, + readEditor, + setEditor, + startEditor, + hasChanges, + draftForEnvironment, + } = useScheduledTaskEditor(); const { availableTargets } = useSettingsEnvironmentFilter(); const navigation = useNavigation(); const insets = useSafeAreaInsets(); const [saving, setSaving] = useState(false); const [saved, setSaved] = useState(false); - const voiceOwnerId = useId(); - const ownerKey = editor ? `${voiceOwnerId}:${editor.environmentId}` : null; + const ownerKey = editor ? `${voiceOwnerKey}:${editor.environmentId}` : null; const prompt = editor?.draft.prompt ?? ""; const [selectionState, setSelectionState] = useState<{ readonly ownerKey: string | null; @@ -402,32 +410,40 @@ function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string } ); const voiceInput = useVoiceInputController({ ownerKey, - draftMessage: prompt, + label: editor?.draft.title.trim() || title, + subscribeToDraftChanges: (onChange) => + appAtomRegistry.subscribe(scheduledTaskEditorSessionAtom, onChange), selection, + readDraftMessage: () => { + const current = readEditor(); + return current?.environmentId === editor?.environmentId + ? (current?.draft.prompt ?? null) + : null; + }, onChangeSelection: setSelection, - onChangeDraftMessage: setPrompt, + onChangeDraftMessage: (text) => { + if (readEditor()?.environmentId === editor?.environmentId) setPrompt(text); + }, disabled: saving, }); - const preventRemove = !saved && (hasChanges || saving || voiceInput.isBusy); + const preventRemove = !saved && (hasChanges || saving); usePreventRemove(preventRemove, ({ data }) => { if (saving) { Alert.alert("Saving task", "Wait for the task to finish saving before leaving."); return; } - Alert.alert( - "Discard changes?", - voiceInput.isBusy - ? "Your dictation and unsaved changes will be lost." - : "Your unsaved changes will be lost.", - [ - { text: "Keep editing", style: "cancel" }, - { - text: "Discard changes", - style: "destructive", - onPress: () => navigation.dispatch(data.action), + Alert.alert("Discard changes?", "Your unsaved changes will be lost.", [ + { text: "Keep editing", style: "cancel" }, + { + text: "Discard changes", + style: "destructive", + onPress: () => { + if (voiceInput.isBusy) voiceInput.cancel(); + startEditor(null); + navigation.dispatch(data.action); }, - ], - ); + }, + ]); }); useEffect(() => { if (!saved) return; diff --git a/apps/mobile/src/features/settings/scheduled-task-editor-state.test.ts b/apps/mobile/src/features/settings/scheduled-task-editor-state.test.ts new file mode 100644 index 000000000000..97a7d38a44bf --- /dev/null +++ b/apps/mobile/src/features/settings/scheduled-task-editor-state.test.ts @@ -0,0 +1,57 @@ +import { EnvironmentId } from "@t3tools/contracts"; +import { afterEach, describe, expect, it } from "vite-plus/test"; + +import { appAtomRegistry } from "../../state/atom-registry"; +import { createDraft } from "./scheduledTaskDraft"; +import { + readScheduledTaskEditor, + scheduledTaskEditorSessionAtom, + startScheduledTaskEditor, + updateScheduledTaskEditor, +} from "./scheduled-task-editor-state"; + +afterEach(() => appAtomRegistry.set(scheduledTaskEditorSessionAtom, null)); + +describe("scheduled task voice draft", () => { + const editor = { + environmentId: EnvironmentId.make("environment"), + environmentLabel: "Environment", + draft: createDraft(null, null), + }; + + it("keeps the prompt outside the screen and retains its original change baseline", () => { + startScheduledTaskEditor(editor); + const original = appAtomRegistry.get(scheduledTaskEditorSessionAtom)!; + const ownerKey = `scheduled-task:${original.id}`; + updateScheduledTaskEditor( + (current) => + current && { + ...current, + draft: { ...current.draft, prompt: "spoken prompt" }, + }, + null, + ); + + expect(readScheduledTaskEditor(ownerKey, null)?.draft.prompt).toBe("spoken prompt"); + expect(appAtomRegistry.get(scheduledTaskEditorSessionAtom)?.initial).toBe(editor); + }); + + it("rejects an old voice target when another task opens in the same environment", () => { + startScheduledTaskEditor(editor); + const original = appAtomRegistry.get(scheduledTaskEditorSessionAtom)!; + startScheduledTaskEditor({ ...editor, draft: { ...editor.draft, prompt: "other task" } }); + + expect(readScheduledTaskEditor(`scheduled-task:${original.id}`, editor)).toBeNull(); + expect(appAtomRegistry.get(scheduledTaskEditorSessionAtom)?.current?.draft.prompt).toBe( + "other task", + ); + }); + + it("does not restore the default prompt after the editor is discarded", () => { + startScheduledTaskEditor(null); + const session = appAtomRegistry.get(scheduledTaskEditorSessionAtom)!; + expect(readScheduledTaskEditor(`scheduled-task:${session.id}`, editor)).toBeNull(); + updateScheduledTaskEditor((current) => current, editor); + expect(appAtomRegistry.get(scheduledTaskEditorSessionAtom)?.current).toBeNull(); + }); +}); diff --git a/apps/mobile/src/features/settings/scheduled-task-editor-state.ts b/apps/mobile/src/features/settings/scheduled-task-editor-state.ts new file mode 100644 index 000000000000..3ac81c08b13d --- /dev/null +++ b/apps/mobile/src/features/settings/scheduled-task-editor-state.ts @@ -0,0 +1,55 @@ +import type { EnvironmentId } from "@t3tools/contracts"; +import { Atom } from "effect/unstable/reactivity"; + +import { appAtomRegistry } from "../../state/atom-registry"; +import type { ScheduledTaskDraft } from "./scheduledTaskDraft"; + +export type ScheduledTaskEditor = { + readonly environmentId: EnvironmentId; + readonly environmentLabel: string; + readonly draft: ScheduledTaskDraft; +}; + +type ScheduledTaskEditorSession = { + readonly id: number; + readonly initial: ScheduledTaskEditor | null; + readonly current: ScheduledTaskEditor | null; +}; + +export const scheduledTaskEditorSessionAtom = Atom.make( + null, +).pipe(Atom.keepAlive); +let nextEditorId = 0; + +export function startScheduledTaskEditor(next: ScheduledTaskEditor | null): void { + appAtomRegistry.set(scheduledTaskEditorSessionAtom, { + id: ++nextEditorId, + initial: next, + current: next, + }); +} + +export function updateScheduledTaskEditor( + update: + | ScheduledTaskEditor + | null + | ((current: ScheduledTaskEditor | null) => ScheduledTaskEditor | null), + defaultEditor: ScheduledTaskEditor | null, +): void { + const session = appAtomRegistry.get(scheduledTaskEditorSessionAtom); + const previous = session ? session.current : defaultEditor; + appAtomRegistry.set(scheduledTaskEditorSessionAtom, { + id: session?.id ?? 0, + initial: session ? session.initial : previous, + current: typeof update === "function" ? update(previous) : update, + }); +} + +export function readScheduledTaskEditor( + voiceOwnerKey: string, + defaultEditor: ScheduledTaskEditor | null, +): ScheduledTaskEditor | null { + const session = appAtomRegistry.get(scheduledTaskEditorSessionAtom); + if (`scheduled-task:${session?.id ?? 0}` !== voiceOwnerKey) return null; + return session ? session.current : defaultEditor; +} diff --git a/apps/mobile/src/features/settings/scheduled-task-editor.tsx b/apps/mobile/src/features/settings/scheduled-task-editor.tsx index f5d8337beeb4..265f42e0059d 100644 --- a/apps/mobile/src/features/settings/scheduled-task-editor.tsx +++ b/apps/mobile/src/features/settings/scheduled-task-editor.tsx @@ -1,10 +1,10 @@ +import { useAtomValue } from "@effect/atom-react"; import type { EnvironmentId } from "@t3tools/contracts"; import { createContext, use, useMemo, useCallback, - useState, type Dispatch, type ReactNode, type SetStateAction, @@ -20,14 +20,18 @@ import { import { useSettingsEnvironmentFilter } from "./settings-environment-filter"; import { settingsTargetsForProject } from "./settings-environment-filter.logic"; -type ScheduledTaskEditor = { - readonly environmentId: EnvironmentId; - readonly environmentLabel: string; - readonly draft: ScheduledTaskDraft; -}; +import { + scheduledTaskEditorSessionAtom, + startScheduledTaskEditor, + updateScheduledTaskEditor, + readScheduledTaskEditor, + type ScheduledTaskEditor, +} from "./scheduled-task-editor-state"; const ScheduledTaskEditorContext = createContext<{ readonly editor: ScheduledTaskEditor | null; + readonly voiceOwnerKey: string; + readonly readEditor: () => ScheduledTaskEditor | null; readonly setEditor: Dispatch>; readonly startEditor: (editor: ScheduledTaskEditor | null) => void; readonly hasChanges: boolean; @@ -76,12 +80,10 @@ export function ScheduledTaskEditorProvider({ children }: { readonly children: R : null, [defaultTarget, draftForEnvironment], ); - const [session, setSession] = useState<{ - readonly initial: ScheduledTaskEditor | null; - readonly current: ScheduledTaskEditor | null; - } | null>(null); - const editor = session?.current ?? defaultEditor; - const initial = session?.initial ?? defaultEditor; + const session = useAtomValue(scheduledTaskEditorSessionAtom); + const voiceOwnerKey = `scheduled-task:${session?.id ?? 0}`; + const editor = session ? session.current : defaultEditor; + const initial = session ? session.initial : defaultEditor; const hasChanges = Boolean( initial && editor && @@ -90,23 +92,37 @@ export function ScheduledTaskEditorProvider({ children }: { readonly children: R ); const setEditor = useCallback>>( (update) => { - setSession((current) => { - const previous = current?.current ?? defaultEditor; - return { - initial: current?.initial ?? previous, - current: typeof update === "function" ? update(previous) : update, - }; - }); + updateScheduledTaskEditor(update, defaultEditor); }, [defaultEditor], ); - const startEditor = useCallback((next: ScheduledTaskEditor | null) => { - setSession({ initial: next, current: next }); - }, []); + const startEditor = startScheduledTaskEditor; const resetEditor = useCallback(() => startEditor(defaultEditor), [defaultEditor, startEditor]); + const readEditor = useCallback( + () => readScheduledTaskEditor(voiceOwnerKey, defaultEditor), + [defaultEditor, voiceOwnerKey], + ); const value = useMemo( - () => ({ editor, setEditor, startEditor, hasChanges, resetEditor, draftForEnvironment }), - [editor, setEditor, startEditor, hasChanges, resetEditor, draftForEnvironment], + () => ({ + editor, + voiceOwnerKey, + readEditor, + setEditor, + startEditor, + hasChanges, + resetEditor, + draftForEnvironment, + }), + [ + editor, + voiceOwnerKey, + readEditor, + setEditor, + startEditor, + hasChanges, + resetEditor, + draftForEnvironment, + ], ); return {children}; } diff --git a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx index 1ee4e9bc4e85..3e2e1837e1da 100644 --- a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx +++ b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx @@ -96,6 +96,7 @@ import { captureComposerDraftInsertion, countComposerDraftAttachmentsAfterSelection, getComposerDraftSnapshot, + composerDraftsAtom, mergeComposerDraftContent, restoreComposerDraftSnapshot, updateComposerDraftSettings, @@ -473,7 +474,9 @@ export function NewTaskDraftScreen(props: { }); const voiceInput = useVoiceInputController({ ownerKey: flow.draftKey, - draftMessage: flow.prompt, + label: selectedProject ? `New task in ${selectedProject.title}` : "New task", + readDraftMessage: () => (flow.draftKey ? getComposerDraftSnapshot(flow.draftKey).text : null), + subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange), selection: composerMenu.selection, disabled: isIncomingShareTransferPending || isImportingShare || flow.submitting, onChangeDraftMessage: flow.setPrompt, diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 16b09e7c1b1a..5ea142d08b4d 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -54,9 +54,13 @@ import { themeColorWithAlpha } from "../../lib/mobileTheme"; import { armAgentAwarenessLiveActivityForLocalWork } from "../agent-awareness/remoteRegistration"; import { scopedThreadKey } from "../../lib/scopedEntities"; import { + getComposerDraftSnapshot, + composerDraftsAtom, + setComposerDraftText, composerContextImportsAtom, countComposerDraftAttachmentsAfterSelection, } from "../../state/use-composer-drafts"; +import { appAtomRegistry } from "../../state/atom-registry"; import type { ComposerDocumentAttachment } from "../../lib/composerContext"; import { useProject, useThreadShells } from "../../state/entities"; import { scopeProjectRef } from "@t3tools/client-runtime/environment"; @@ -504,10 +508,12 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer usageLimitsOffered && props.draftAttachments.length === 0 ? openUsageLimits : undefined, }); const voiceInput = useVoiceInputController({ - ownerKey: composerOwnerKey, - draftMessage: props.draftMessage, + ownerKey: composerDraftKey, + label: props.selectedThread.title || "Untitled thread", + readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text, + subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange), selection: composerMenu.selection, - onChangeDraftMessage: props.onChangeDraftMessage, + onChangeDraftMessage: (text) => setComposerDraftText(composerDraftKey, text), onChangeSelection: composerMenu.onSelectionChange, }); const voicePresentation = resolveVoiceComposerPresentation( diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index 8b9e061f39a2..013bf900912d 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -132,6 +132,7 @@ import { ComposerQueuedEditBanner } from "./ComposerQueuedEdit"; import { useThreadQueuedCount } from "./ThreadQueueControl"; import type { ThreadContentPresentation } from "./threadContentPresentation"; import { resolveThreadFeedSubmissionAnchor } from "./thread-feed-live-follow"; +import { useGlobalVoiceInput } from "../voice-input/VoiceInputProvider"; export interface ThreadDetailScreenProps { readonly worktreeSetup?: WorktreeSetupCardProps | null; @@ -304,6 +305,7 @@ const USER_INPUT_TOGGLE_TIMING = { export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: ThreadDetailScreenProps) { const navigation = useNavigation(); + const { session: voiceInputSession } = useGlobalVoiceInput(); const reportedModelSelection = useThreadReportedModelSelection({ environmentId: props.environmentId, threadId: props.selectedThread.id, @@ -1169,7 +1171,10 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread > { + voiceInputSession.cancel(props.composerDraftKey); + props.onCancelQueuedRunEdit(); + }} /> ) : null} diff --git a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx index 602cbb158b54..f27d2359639a 100644 --- a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx +++ b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx @@ -157,8 +157,10 @@ export function ComposerDictationToolbar(props: { const WaveformBar = memo(function WaveformBar(props: { readonly audioLevels: SharedValue; readonly sampleIndex: number; + readonly height?: number; }) { const { audioLevels, sampleIndex } = props; + const height = props.height ?? WAVEFORM_BAR_HEIGHT; const animatedStyle = useAnimatedStyle(() => { const level = audioLevels.value[sampleIndex] ?? 0; return { @@ -166,8 +168,7 @@ const WaveformBar = memo(function WaveformBar(props: { transform: [ { scaleY: withTiming( - (WAVEFORM_MIN_BAR_HEIGHT + level * (WAVEFORM_BAR_HEIGHT - WAVEFORM_MIN_BAR_HEIGHT)) / - WAVEFORM_BAR_HEIGHT, + (WAVEFORM_MIN_BAR_HEIGHT + level * (height - WAVEFORM_MIN_BAR_HEIGHT)) / height, WAVEFORM_TIMING, ), }, @@ -178,11 +179,37 @@ const WaveformBar = memo(function WaveformBar(props: { return ( ); }); +const COMPACT_WAVEFORM_BAR_COUNT = 8; +const COMPACT_WAVEFORM_BAR_HEIGHT = 14; + +/** A fixed-size waveform of the latest samples, for tight spaces like the global dictation pill. */ +export const CompactVoiceWaveform = memo(function CompactVoiceWaveform(props: { + readonly audioLevels: SharedValue; +}) { + return ( + + {Array.from({ length: COMPACT_WAVEFORM_BAR_COUNT }, (_, index) => ( + + ))} + + ); +}); + const VoiceWaveform = memo(function VoiceWaveform(props: { readonly audioLevels: SharedValue; }) { diff --git a/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx b/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx new file mode 100644 index 000000000000..4159041e2853 --- /dev/null +++ b/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx @@ -0,0 +1,259 @@ +import { useEffect, useRef, useState, type ReactNode, type RefObject } from "react"; +import { + ActivityIndicator, + Linking, + Platform, + Pressable, + useWindowDimensions, + View, +} from "react-native"; +import Animated, { + Easing, + Extrapolation, + FadeIn, + interpolate, + ReduceMotion, + useAnimatedStyle, + useSharedValue, + withTiming, +} from "react-native-reanimated"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { FullWindowOverlay } from "react-native-screens"; + +import { AppText as Text } from "../../components/AppText"; +import { SymbolView } from "../../components/AppSymbol"; +import { useAndroidControlSizing } from "../../components/useAndroidControlSizing"; +import { cn } from "../../lib/cn"; +import { CompactVoiceWaveform } from "./ComposerDictationControl"; +import { useGlobalVoiceInput } from "./VoiceInputProvider"; +import { resolveVoiceComposerPresentation } from "./voiceInputPresentation"; + +const PILL_HEIGHT = 36; +const COLLAPSED_WIDTH = 64; +const EXPANDED_MAX_WIDTH = 280; +// Clears Home's floating bottom toolbar (56) and a collapsed thread composer (60). +const IOS_BOTTOM_CHROME_CLEARANCE = 64; +const MORPH_TIMING = { + duration: 260, + easing: Easing.out(Easing.cubic), + reduceMotion: ReduceMotion.System, +} as const; +const ENTERING = FadeIn.duration(180).reduceMotion(ReduceMotion.System); + +/** + * Keeps an off-screen dictation reachable as a pill on the screen's trailing + * edge. Wraps the app content so any touch that starts outside the pill, + * including a scroll, collapses it without claiming the touch. + */ +export function GlobalVoiceInputControl(props: { readonly children: ReactNode }) { + const voice = useGlobalVoiceInput(); + const collapseRef = useRef<(() => void) | null>(null); + const presentation = resolveVoiceComposerPresentation(voice.state, voice.elapsedSeconds); + const visible = + presentation.statusLabel !== null && + !(voice.ownerKey && voice.focusedOwners.has(voice.ownerKey)); + // Mounted only while visible, so each dictation starts collapsed. + const pill = visible ? : null; + return ( + <> + { + collapseRef.current?.(); + return false; + }} + > + {props.children} + + {pill && Platform.OS === "ios" ? {pill} : pill} + + ); +} + +function EdgeDictationPill(props: { readonly collapseRef: RefObject<(() => void) | null> }) { + const voice = useGlobalVoiceInput(); + const insets = useSafeAreaInsets(); + const { width } = useWindowDimensions(); + const { fabSize } = useAndroidControlSizing(); + const [expanded, setExpanded] = useState(false); + const progress = useSharedValue(0); + useEffect(() => { + progress.value = withTiming(expanded ? 1 : 0, MORPH_TIMING); + }, [expanded, progress]); + const { collapseRef } = props; + useEffect(() => { + if (!expanded) return; + collapseRef.current = () => setExpanded(false); + return () => { + collapseRef.current = null; + }; + }, [collapseRef, expanded]); + + const expandedWidth = Math.min(width - insets.left - insets.right - 32, EXPANDED_MAX_WIDTH); + // Each face keeps its final width and stays pinned to the edge, so the + // morphing pill reveals the expanded row instead of reflowing it. + const pillStyle = useAnimatedStyle(() => ({ + width: interpolate(progress.value, [0, 1], [COLLAPSED_WIDTH, expandedWidth]), + })); + const collapsedStyle = useAnimatedStyle(() => ({ + opacity: interpolate(progress.value, [0, 0.4], [1, 0], Extrapolation.CLAMP), + })); + const expandedStyle = useAnimatedStyle(() => ({ + opacity: interpolate(progress.value, [0.3, 1], [0, 1], Extrapolation.CLAMP), + })); + + // Sit just above the bottom chrome: the toolbar or composer on iOS, the + // new-thread FAB on Android. + const bottom = + Platform.OS === "android" + ? Math.max(insets.bottom, 16) + 16 + fabSize + 12 + : Math.max(insets.bottom, 12) + IOS_BOTTOM_CHROME_CLEARANCE; + + const presentation = resolveVoiceComposerPresentation(voice.state, voice.elapsedSeconds); + const phase = voice.state.phase; + const isError = phase === "error"; + const openSettings = isError && voice.state.errorAction === "settings"; + const label = voice.label ?? "Draft"; + const elapsedLabel = `${Math.floor(voice.elapsedSeconds / 60)}:${String(voice.elapsedSeconds % 60).padStart(2, "0")}`; + + return ( + + + + + setExpanded(true)} + > + {isError ? ( + + ) : phase === "recording" ? ( + <> + + + {elapsedLabel} + + + ) : ( + + )} + + + + + + + + setExpanded(false)} + > + + {isError ? presentation.statusLabel : label} + + + {phase === "recording" ? ( + <> + + + {elapsedLabel} + + + + + + + + ) : isError ? ( + voice.isAvailable ? ( + { + if (!openSettings) { + void voice.session.retry(); + return; + } + voice.cancel(); + void Linking.openSettings(); + }} + > + + + ) : null + ) : ( + + + + )} + + + + + ); +} diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx new file mode 100644 index 000000000000..f6deefcea87b --- /dev/null +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -0,0 +1,222 @@ +import { + RecordingPresets, + requestRecordingPermissionsAsync, + setAudioModeAsync, + setIsAudioActiveAsync, + useAudioRecorder, + type RecordingStatus, +} from "expo-audio"; +import { File } from "expo-file-system"; +import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; +import { + createContext, + use, + useCallback, + useEffect, + useId, + useRef, + useState, + type ReactNode, +} from "react"; +import { AppState } from "react-native"; +import { useSharedValue } from "react-native-reanimated"; + +import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; +import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene"; +import { + VOICE_RECORDING_LIMIT_SECONDS, + voiceInputBlocksSubmission, + type VoiceInputState, +} from "@t3tools/client-runtime/voice-input"; +import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; +import { VoiceInputSession } from "./voiceInputSession"; + +const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; +const VOICE_METERING_INTERVAL_MS = 80; +const VOICE_RECORDING_OPTIONS = { + ...RecordingPresets.HIGH_QUALITY, + isMeteringEnabled: true, +}; + +async function releaseVoiceRecordingAudio(): Promise { + try { + await setAudioModeAsync({ allowsRecording: false }); + } finally { + // Expo does not deactivate AVAudioSession when recording stops or its + // category changes. Explicit deactivation resumes interrupted app audio. + await setIsAudioActiveAsync(false); + } +} + +async function configureVoiceRecordingAudio(): Promise { + try { + await setAudioModeAsync({ + allowsRecording: true, + interruptionMode: "doNotMix", + playsInSilentMode: true, + shouldPlayInBackground: false, + }); + await setIsAudioActiveAsync(true); + } catch (error) { + try { + await releaseVoiceRecordingAudio(); + } catch { + // Keep the setup error. The controller has not started a recorder yet. + } + throw error; + } +} + +const VoiceInputContext = createContext | null>(null); + +export function VoiceInputProvider({ children }: { readonly children: ReactNode }) { + const runtime = useVoiceInputRuntime(); + return {children}; +} + +export function useGlobalVoiceInput() { + const context = use(VoiceInputContext); + if (!context) throw new Error("Voice input provider is missing."); + return context; +} + +function useVoiceInputRuntime() { + const [{ state, ownerKey, label }, setState] = useState({ + state: INITIAL_STATE, + ownerKey: null as string | null, + label: null as string | null, + }); + const [focusedOwners, setFocusedOwners] = useState>(new Set()); + const setOwnerFocused = useCallback((key: string, focused: boolean) => { + setFocusedOwners((current) => { + const next = new Set(current); + if (focused) next.add(key); + else next.delete(key); + return next; + }); + }, []); + const [elapsedSeconds, setElapsedSeconds] = useState(0); + const keepAwakeId = useId(); + const keepAwakeSessionRef = useRef(0); + const elapsedSecondsRef = useRef(0); + const audioLevelsRef = useRef(Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0)); + const audioLevels = useSharedValue(audioLevelsRef.current); + const sessionRef = useRef(null); + const handleRecorderStatus = useCallback((status: RecordingStatus) => { + sessionRef.current?.controller.handleRecorderStatus({ + isFinished: status.isFinished, + hasError: status.hasError || status.mediaServicesDidReset === true, + error: status.error, + url: status.url, + }); + }, []); + const recorder = useAudioRecorder(VOICE_RECORDING_OPTIONS, handleRecorderStatus); + + if (!sessionRef.current) { + sessionRef.current = new VoiceInputSession({ + recorder, + getTranscriber: getLocalVoiceTranscriber, + requestPermission: async () => { + const permission = await requestRecordingPermissionsAsync(); + return { granted: permission.granted, canAskAgain: permission.canAskAgain }; + }, + configureRecording: configureVoiceRecordingAudio, + releaseRecording: releaseVoiceRecordingAudio, + deleteRecording: (uri) => new File(uri).delete(), + onStateChange: (nextState) => + setState({ + state: nextState, + ownerKey: sessionRef.current?.ownerKey ?? null, + label: sessionRef.current?.label ?? null, + }), + }); + } + + const session = sessionRef.current; + const controller = session.controller; + + useEffect(() => { + const subscription = AppState.addEventListener("change", (nextState) => { + // iOS reports `inactive` while its permission dialog is open. Only the + // real background state cancels preparation; recorder status handles + // calls and route interruptions during capture. + if (nextState === "background") controller.appMovedToBackground(); + }); + return () => subscription.remove(); + }, [controller]); + + useEffect(() => () => controller.dispose(), [controller]); + + useEffect(() => { + if (state.phase !== "recording") return; + + const tag = `voice-input:${keepAwakeId}:${++keepAwakeSessionRef.current}`; + const activation = activateKeepAwakeAsync(tag); + void activation.catch(() => {}); + return () => { + // Release after activation settles, even if the recording ends immediately. + void activation.then(() => deactivateKeepAwake(tag)).catch(() => {}); + }; + }, [keepAwakeId, state.phase]); + + useEffect(() => { + if (state.phase !== "preparing" && state.phase !== "recording") return; + + if (audioLevelsRef.current.some((level) => level !== 0)) { + audioLevelsRef.current = Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0); + audioLevels.value = audioLevelsRef.current; + } + if (elapsedSecondsRef.current !== 0) { + elapsedSecondsRef.current = 0; + setElapsedSeconds(0); + } + if (state.phase !== "recording") return; + + const sampleRecording = () => { + if (controller.currentState.phase !== "recording") return; + const status = recorder.getStatus(); + if (!status.isRecording) return; + + const level = normalizeVoiceInputDecibels(status.metering); + const history = audioLevelsRef.current; + if (level !== 0 || history.some((sample) => sample !== 0)) { + const nextLevels = [...history.slice(1), level]; + audioLevelsRef.current = nextLevels; + audioLevels.value = nextLevels; + } + + const nextElapsedSeconds = Math.min( + VOICE_RECORDING_LIMIT_SECONDS, + Math.max(0, Math.floor(status.durationMillis / 1_000)), + ); + if (nextElapsedSeconds !== elapsedSecondsRef.current) { + elapsedSecondsRef.current = nextElapsedSeconds; + setElapsedSeconds(nextElapsedSeconds); + } + }; + + sampleRecording(); + const intervalId = setInterval(sampleRecording, VOICE_METERING_INTERVAL_MS); + return () => clearInterval(intervalId); + }, [audioLevels, controller, recorder, state.phase]); + + const stop = useCallback(() => controller.stop(), [controller]); + const cancel = useCallback(() => controller.cancel(), [controller]); + + return { + // Store screenshots show the dictation button even on simulators, whose + // on-device transcription is unavailable. + isAvailable: getLocalVoiceTranscriber() !== null || getNativeShowcaseScene() !== null, + state, + audioLevels, + elapsedSeconds, + isBusy: voiceInputBlocksSubmission(state), + ownerKey, + label, + focusedOwners, + setOwnerFocused, + session, + stop, + cancel, + }; +} diff --git a/apps/mobile/src/features/voice-input/useVoiceInputController.ts b/apps/mobile/src/features/voice-input/useVoiceInputController.ts index 5fd15482d037..d1dc234f39f5 100644 --- a/apps/mobile/src/features/voice-input/useVoiceInputController.ts +++ b/apps/mobile/src/features/voice-input/useVoiceInputController.ts @@ -1,235 +1,79 @@ -import { - RecordingPresets, - requestRecordingPermissionsAsync, - setAudioModeAsync, - setIsAudioActiveAsync, - useAudioRecorder, - type RecordingStatus, -} from "expo-audio"; -import { File } from "expo-file-system"; -import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; import { useFocusEffect } from "@react-navigation/native"; -import { useCallback, useEffect, useId, useRef, useState } from "react"; -import { AppState } from "react-native"; -import { useSharedValue } from "react-native-reanimated"; - -import type { ComposerEditorSelection } from "../../components/ComposerEditor"; -import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; -import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene"; +import { useCallback, useEffect, useRef } from "react"; import { - VoiceInputController, - VOICE_RECORDING_LIMIT_SECONDS, voiceInputBlocksSubmission, - voiceInputFreezesEditor, - type VoiceDraftSnapshot, type VoiceInputState, } from "@t3tools/client-runtime/voice-input"; -import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; -const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; -const VOICE_METERING_INTERVAL_MS = 80; -const VOICE_RECORDING_OPTIONS = { - ...RecordingPresets.HIGH_QUALITY, - isMeteringEnabled: true, -}; - -async function releaseVoiceRecordingAudio(): Promise { - try { - await setAudioModeAsync({ allowsRecording: false }); - } finally { - // Expo does not deactivate AVAudioSession when recording stops or its - // category changes. Explicit deactivation resumes interrupted app audio. - await setIsAudioActiveAsync(false); - } -} +import type { ComposerEditorSelection } from "../../components/ComposerEditor"; +import { useGlobalVoiceInput } from "./VoiceInputProvider"; +import { createVoiceInputTarget } from "./voiceInputSession"; -async function configureVoiceRecordingAudio(): Promise { - try { - await setAudioModeAsync({ - allowsRecording: true, - interruptionMode: "doNotMix", - playsInSilentMode: true, - shouldPlayInBackground: false, - }); - await setIsAudioActiveAsync(true); - } catch (error) { - try { - await releaseVoiceRecordingAudio(); - } catch { - // Keep the setup error. The controller has not started a recorder yet. - } - throw error; - } -} +const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; export function useVoiceInputController(input: { readonly ownerKey: string | null; - readonly draftMessage: string; + /** Shown by the global dictation pill when this composer is off screen. */ + readonly label: string; + readonly readDraftMessage: () => string | null; + readonly subscribeToDraftChanges: (onChange: () => void) => () => void; readonly selection: ComposerEditorSelection; readonly disabled?: boolean; readonly onChangeDraftMessage: (value: string) => void; readonly onChangeSelection: (selection: ComposerEditorSelection) => void; }) { - const [state, setState] = useState(INITIAL_STATE); - const [elapsedSeconds, setElapsedSeconds] = useState(0); - const keepAwakeId = useId(); - const keepAwakeSessionRef = useRef(0); - const elapsedSecondsRef = useRef(0); - const audioLevelsRef = useRef(Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0)); - const audioLevels = useSharedValue(audioLevelsRef.current); - const controllerRef = useRef(null); - const previousDraftRef = useRef({ ownerKey: input.ownerKey, text: input.draftMessage }); - const revisionRef = useRef(0); - if ( - previousDraftRef.current.ownerKey !== input.ownerKey || - previousDraftRef.current.text !== input.draftMessage - ) { - previousDraftRef.current = { ownerKey: input.ownerKey, text: input.draftMessage }; - revisionRef.current += 1; - } - const latestInputRef = useRef(input); - latestInputRef.current = input; - - const handleRecorderStatus = useCallback((status: RecordingStatus) => { - controllerRef.current?.handleRecorderStatus({ - isFinished: status.isFinished, - hasError: status.hasError || status.mediaServicesDidReset === true, - error: status.error, - url: status.url, - }); - }, []); - const recorder = useAudioRecorder(VOICE_RECORDING_OPTIONS, handleRecorderStatus); - - if (!controllerRef.current) { - controllerRef.current = new VoiceInputController({ - recorder, - getTranscriber: getLocalVoiceTranscriber, - requestPermission: async () => { - const permission = await requestRecordingPermissionsAsync(); - return { granted: permission.granted, canAskAgain: permission.canAskAgain }; - }, - configureRecording: configureVoiceRecordingAudio, - releaseRecording: releaseVoiceRecordingAudio, - deleteRecording: (uri) => new File(uri).delete(), - readDraft: (): VoiceDraftSnapshot | null => { - const current = latestInputRef.current; - if (!current.ownerKey) return null; - return { - ownerKey: current.ownerKey, - text: current.draftMessage, - selection: current.selection, - revision: revisionRef.current, - }; - }, - commitDraft: (text, selection) => { - const current = latestInputRef.current; - current.onChangeSelection(selection); - current.onChangeDraftMessage(text); - }, - onStateChange: setState, - }); - } - - const controller = controllerRef.current; - const previousOwnerRef = useRef(input.ownerKey); - useEffect(() => { - if (previousOwnerRef.current === input.ownerKey) return; - previousOwnerRef.current = input.ownerKey; - controller.ownerChanged(); - }, [controller, input.ownerKey]); - - useFocusEffect( - useCallback( - () => () => { - controller.dispose(); - }, - [controller], - ), - ); - - useEffect(() => { - const subscription = AppState.addEventListener("change", (nextState) => { - // iOS reports `inactive` while its permission dialog is open. Only the - // real background state cancels preparation; recorder status handles - // calls and route interruptions during capture. - if (nextState === "background") controller.appMovedToBackground(); - }); - return () => subscription.remove(); - }, [controller]); - - useEffect(() => () => controller.dispose(), [controller]); - + const global = useGlobalVoiceInput(); + const { setOwnerFocused, session } = global; + const latestInput = useRef(input); + latestInput.current = input; + const mounted = useRef(true); useEffect(() => { - if (state.phase !== "recording") return; - - const tag = `voice-input:${keepAwakeId}:${++keepAwakeSessionRef.current}`; - const activation = activateKeepAwakeAsync(tag); - void activation.catch(() => {}); + mounted.current = true; return () => { - // Release after activation settles, even if the recording ends immediately. - void activation.then(() => deactivateKeepAwake(tag)).catch(() => {}); - }; - }, [keepAwakeId, state.phase]); - - useEffect(() => { - if (state.phase !== "preparing" && state.phase !== "recording") return; - - if (audioLevelsRef.current.some((level) => level !== 0)) { - audioLevelsRef.current = Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0); - audioLevels.value = audioLevelsRef.current; - } - if (elapsedSecondsRef.current !== 0) { - elapsedSecondsRef.current = 0; - setElapsedSeconds(0); - } - if (state.phase !== "recording") return; - - const sampleRecording = () => { - if (controller.currentState.phase !== "recording") return; - const status = recorder.getStatus(); - if (!status.isRecording) return; - - const level = normalizeVoiceInputDecibels(status.metering); - const history = audioLevelsRef.current; - if (level !== 0 || history.some((sample) => sample !== 0)) { - const nextLevels = [...history.slice(1), level]; - audioLevelsRef.current = nextLevels; - audioLevels.value = nextLevels; - } - - const nextElapsedSeconds = Math.min( - VOICE_RECORDING_LIMIT_SECONDS, - Math.max(0, Math.floor(status.durationMillis / 1_000)), - ); - if (nextElapsedSeconds !== elapsedSecondsRef.current) { - elapsedSecondsRef.current = nextElapsedSeconds; - setElapsedSeconds(nextElapsedSeconds); - } + mounted.current = false; }; + }, []); - sampleRecording(); - const intervalId = setInterval(sampleRecording, VOICE_METERING_INTERVAL_MS); - return () => clearInterval(intervalId); - }, [audioLevels, controller, recorder, state.phase]); + useFocusEffect( + useCallback(() => { + const ownerKey = input.ownerKey; + if (!ownerKey) return; + setOwnerFocused(ownerKey, true); + return () => setOwnerFocused(ownerKey, false); + }, [input.ownerKey, setOwnerFocused]), + ); const start = useCallback(() => { - if (!latestInputRef.current.disabled) void controller.start(); - }, [controller]); - const stop = useCallback(() => controller.stop(), [controller]); - const cancel = useCallback(() => controller.cancel(), [controller]); - + const captured = latestInput.current; + if (!captured.ownerKey || captured.disabled) return; + void session.start({ + ...createVoiceInputTarget( + captured.ownerKey, + captured.readDraftMessage, + (text, selection) => { + captured.onChangeDraftMessage(text); + if (mounted.current && latestInput.current.ownerKey === captured.ownerKey) { + latestInput.current.onChangeSelection(selection); + } + }, + captured.selection, + captured.subscribeToDraftChanges, + ), + label: captured.label, + }); + }, [session]); + const state = global.ownerKey === input.ownerKey ? global.state : IDLE_STATE; + const isBusy = voiceInputBlocksSubmission(state); return { - // Store screenshots show the dictation button even on simulators, whose - // on-device transcription is unavailable. - isAvailable: getLocalVoiceTranscriber() !== null || getNativeShowcaseScene() !== null, + isAvailable: global.isAvailable && (!global.isBusy || global.ownerKey === input.ownerKey), state, - audioLevels, - elapsedSeconds, - isBusy: voiceInputBlocksSubmission(state), - freezesEditor: voiceInputFreezesEditor(state), - blocksSubmission: voiceInputBlocksSubmission(state), + audioLevels: global.audioLevels, + elapsedSeconds: global.elapsedSeconds, + isBusy, + freezesEditor: isBusy, + blocksSubmission: isBusy, start, - stop, - cancel, + stop: global.stop, + cancel: global.cancel, }; } diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts new file mode 100644 index 000000000000..07fc6348f8af --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts @@ -0,0 +1,272 @@ +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import type { PreparedVoiceTranscription } from "@t3tools/client-runtime/voice-input"; +import { Atom, AtomRegistry } from "effect/unstable/reactivity"; +import { resetVoiceInputGlobalsForTests } from "../../../../../packages/client-runtime/src/voice-input/controller"; + +import { + createVoiceInputTarget, + VoiceInputSession, + type VoiceInputTarget, +} from "./voiceInputSession"; + +function createTarget( + ownerKey: string, + readText: () => string | null, + commit: VoiceInputTarget["commitDraft"], + selection: { start: number; end: number }, +) { + return createVoiceInputTarget(ownerKey, readText, commit, selection, () => () => {}); +} + +function createSession() { + const recorder = { + uri: "file:///voice.m4a", + prepareToRecordAsync: vi.fn(async () => {}), + record: vi.fn(), + stop: vi.fn(async () => {}), + }; + const prepare = vi.fn(async (): Promise => ({ + locale: "en-US", + transcribe: async () => "spoken text", + })); + const session = new VoiceInputSession({ + recorder, + getTranscriber: () => ({ prepare }), + requestPermission: async () => ({ granted: true, canAskAgain: true }), + configureRecording: async () => {}, + releaseRecording: vi.fn(async () => {}), + deleteRecording: vi.fn(), + onStateChange: vi.fn(), + }); + return { session, recorder, prepare }; +} + +describe("global voice input", () => { + beforeEach(() => resetVoiceInputGlobalsForTests()); + + it.each([ + { selection: { start: 6, end: 11 }, expected: "hello spoken text", cursor: 17 }, + { selection: { start: 6, end: 6 }, expected: "hello spoken text world", cursor: 18 }, + ])( + "keeps the starting selection $selection after navigation", + async ({ selection, expected, cursor }) => { + const { session } = createSession(); + const commit = vi.fn(); + await session.start(createTarget("first", () => "hello world", commit, selection)); + await session.start( + createTarget("second", () => "other prompt", vi.fn(), { start: 12, end: 12 }), + ); + await session.controller.stop(); + expect(commit).toHaveBeenCalledWith(expected, { + start: cursor, + end: cursor, + }); + }, + ); + + it("appends to the starting draft after its screen leaves and another draft opens", async () => { + const { session, recorder } = createSession(); + const drafts = new Map([ + ["first", "original prompt"], + ["second", "other prompt"], + ]); + let visibleDraft = "first"; + const targetKey = visibleDraft; + await session.start( + createTarget( + targetKey, + () => drafts.get(targetKey) ?? null, + (text) => drafts.set(targetKey, text), + { start: 15, end: 15 }, + ), + ); + visibleDraft = "second"; + expect(session.controller.currentState.phase).toBe("recording"); + expect(recorder.stop).not.toHaveBeenCalled(); + await session.controller.stop(); + + expect(drafts.get("first")).toBe("original prompt spoken text"); + expect(drafts.get(visibleDraft)).toBe("other prompt"); + expect(session.controller.currentState.phase).toBe("idle"); + }); + + it.each(["preparing", "recording", "transcribing"] as const)( + "keeps one recorder and its original target during %s", + async (phase) => { + const preparation = Promise.withResolvers(); + const preparationEntered = Promise.withResolvers(); + const transcription = Promise.withResolvers(); + const transcriptionEntered = Promise.withResolvers(); + const { session, recorder, prepare } = createSession(); + prepare.mockImplementationOnce(() => { + preparationEntered.resolve(); + return preparation.promise; + }); + const firstCommit = vi.fn(); + const secondCommit = vi.fn(); + const starting = session.start( + createTarget("first", () => "first", firstCommit, { start: 5, end: 5 }), + ); + await preparationEntered.promise; + let stopping: Promise | null = null; + if (phase !== "preparing") { + preparation.resolve({ + locale: "en-US", + transcribe: () => { + transcriptionEntered.resolve(); + return transcription.promise; + }, + }); + await starting; + } + if (phase === "transcribing") { + stopping = session.controller.stop(); + await transcriptionEntered.promise; + } + await session.start( + createTarget("second", () => "second", secondCommit, { start: 6, end: 6 }), + ); + expect(session.ownerKey).toBe("first"); + expect(session.controller.currentState.phase).toBe(phase); + expect(prepare).toHaveBeenCalledTimes(1); + preparation.resolve({ locale: "en-US", transcribe: async () => "spoken text" }); + await starting; + transcription.resolve("spoken text"); + await (stopping ?? session.controller.stop()); + expect(recorder.record).toHaveBeenCalledTimes(1); + expect(firstCommit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); + expect(secondCommit).not.toHaveBeenCalled(); + }, + ); + + it.each(["changed", "removed"] as const)( + "does not overwrite a %s starting draft", + async (change) => { + const { session } = createSession(); + let text: string | null = "first"; + const commit = vi.fn(); + await session.start(createTarget("first", () => text, commit, { start: 5, end: 5 })); + text = change === "removed" ? null : "edited prompt"; + await session.controller.stop(); + expect(commit).not.toHaveBeenCalled(); + expect(session.controller.currentState.error).toContain("draft changed"); + }, + ); + + it("finishes the original draft at the recording limit while it is off screen", async () => { + const { session, recorder } = createSession(); + const commit = vi.fn(); + await session.start(createTarget("first", () => "first", commit, { start: 5, end: 5 })); + await session.controller.handleRecorderStatus({ + isFinished: true, + hasError: false, + error: null, + url: recorder.uri, + }); + expect(commit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); + expect(session.controller.currentState.phase).toBe("idle"); + }); + + it("rejects a transcript when its off-screen draft changes and returns to the original text", async () => { + const registry = AtomRegistry.make(); + const draft = Atom.make("hello world"); + const unsubscribe = vi.fn(); + const { session } = createSession(); + const commit = vi.fn(); + const target = createVoiceInputTarget( + "first", + () => registry.get(draft), + commit, + { start: 6, end: 6 }, + (onChange) => { + const stop = registry.subscribe(draft, onChange); + return () => { + stop(); + unsubscribe(); + }; + }, + ); + await session.start(target); + registry.set(draft, "changed"); + registry.set(draft, "hello world"); + await session.controller.stop(); + expect(commit).not.toHaveBeenCalled(); + expect(session.controller.currentState.error).toContain("draft changed"); + expect(unsubscribe).toHaveBeenCalledTimes(1); + registry.dispose(); + }); + + it("stops recording when its queued edit is discarded", async () => { + const { session, recorder } = createSession(); + const commit = vi.fn(); + let draft: string | null = "queued prompt"; + const ownerKey = "thread~queued-edit~run"; + await session.start(createTarget(ownerKey, () => draft, commit, { start: 13, end: 13 })); + session.cancel(ownerKey); + draft = null; + await session.controller.stop(); + expect(recorder.stop).toHaveBeenCalledTimes(1); + expect(session.controller.currentState).toEqual({ + phase: "idle", + error: null, + errorAction: null, + }); + expect(commit).not.toHaveBeenCalled(); + }); + + it.each(["complete", "cancel"] as const)( + "releases draft observation after %s", + async (finish) => { + const { session } = createSession(); + const unsubscribe = vi.fn(); + const subscribe = vi.fn(() => unsubscribe); + await session.start( + createVoiceInputTarget("first", () => "first", vi.fn(), { start: 5, end: 5 }, subscribe), + ); + expect(subscribe).toHaveBeenCalledTimes(1); + if (finish === "complete") await session.controller.stop(); + else session.cancel("first"); + expect(unsubscribe).toHaveBeenCalledTimes(1); + }, + ); + + it("keeps another prompt's recording when a queued edit is discarded", async () => { + const { session, recorder } = createSession(); + const commit = vi.fn(); + await session.start(createTarget("other prompt", () => "hello", commit, { start: 5, end: 5 })); + session.cancel("thread~queued-edit~run"); + expect(recorder.stop).not.toHaveBeenCalled(); + expect(session.controller.currentState.phase).toBe("recording"); + await session.controller.stop(); + expect(commit).toHaveBeenCalledWith("hello spoken text", { start: 17, end: 17 }); + }); + + it("waits for canceled native work before starting a recording for another draft", async () => { + const preparation = Promise.withResolvers(); + const preparationEntered = Promise.withResolvers(); + const { session, recorder, prepare } = createSession(); + prepare.mockImplementationOnce(() => { + preparationEntered.resolve(); + return preparation.promise; + }); + const oldCommit = vi.fn(); + const nextCommit = vi.fn(); + const firstStart = session.start( + createTarget("first", () => "first", oldCommit, { start: 5, end: 5 }), + ); + await preparationEntered.promise; + session.cancel("first"); + const nextStart = session.start( + createTarget("second", () => "second", nextCommit, { start: 6, end: 6 }), + ); + expect(prepare).toHaveBeenCalledTimes(1); + expect(recorder.record).not.toHaveBeenCalled(); + preparation.resolve({ locale: "en-US", transcribe: async () => "old transcript" }); + await firstStart; + await nextStart; + await session.controller.stop(); + expect(recorder.record).toHaveBeenCalledTimes(1); + expect(oldCommit).not.toHaveBeenCalled(); + expect(nextCommit).toHaveBeenCalledWith("second spoken text", { start: 18, end: 18 }); + }); +}); diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.ts b/apps/mobile/src/features/voice-input/voiceInputSession.ts new file mode 100644 index 000000000000..45983654601f --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceInputSession.ts @@ -0,0 +1,88 @@ +import { + VoiceInputController, + voiceInputBlocksSubmission, + type VoiceDraftSnapshot, + type VoiceInputControllerDependencies, +} from "@t3tools/client-runtime/voice-input"; + +export type VoiceInputTarget = { + readonly ownerKey: string; + /** Names the draft being dictated into while its composer is off screen. */ + readonly label?: string; + readonly readDraft: () => VoiceDraftSnapshot | null; + readonly commitDraft: VoiceInputControllerDependencies["commitDraft"]; + readonly subscribe: () => () => void; +}; + +export function createVoiceInputTarget( + ownerKey: string, + readText: () => string | null, + commitDraft: VoiceInputTarget["commitDraft"], + selection: VoiceDraftSnapshot["selection"], + subscribeToChanges: (onChange: () => void) => () => void, +): VoiceInputTarget { + let revision = 0; + return { + ownerKey, + readDraft: () => { + const text = readText(); + if (text === null) return null; + return { ownerKey, text, selection, revision }; + }, + commitDraft, + subscribe: () => { + let previousText = readText(); + return subscribeToChanges(() => { + const text = readText(); + if (text !== previousText) { + previousText = text; + revision += 1; + } + }); + }, + }; +} + +export class VoiceInputSession { + readonly controller: VoiceInputController; + private target: VoiceInputTarget | null = null; + private unsubscribe: (() => void) | null = null; + + constructor(dependencies: Omit) { + this.controller = new VoiceInputController({ + ...dependencies, + onStateChange: (state) => { + if (!voiceInputBlocksSubmission(state)) { + this.unsubscribe?.(); + this.unsubscribe = null; + } + dependencies.onStateChange(state); + }, + readDraft: () => this.target?.readDraft() ?? null, + commitDraft: (text, selection) => this.target?.commitDraft(text, selection), + }); + } + + get ownerKey(): string | null { + return this.target?.ownerKey ?? null; + } + + get label(): string | null { + return this.target?.label ?? null; + } + + cancel(ownerKey: string | null): void { + if (ownerKey !== null && this.ownerKey === ownerKey) this.controller.cancel(); + } + + retry(): Promise { + return this.target ? this.start(this.target) : Promise.resolve(); + } + + start(target: VoiceInputTarget): Promise { + if (voiceInputBlocksSubmission(this.controller.currentState)) return Promise.resolve(); + this.target = target; + this.unsubscribe = target.subscribe(); + return this.controller.start(); + } +} diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index 5f26b882b694..d73c8e7505ba 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -298,11 +298,21 @@ describe("VoiceInputController", () => { }); it.each(["cancel", "dispose", "ownerChanged"] as const)( - "holds the session after %s until non-abortable transcription settles", + "queues the next recording after %s until transcription and audio cleanup settle", async (action) => { const transcription = deferred(); const transcriptionEntered = deferred(); + const audioRelease = deferred(); + const audioReleaseEntered = deferred(); + const releaseRecording = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error("busy")) + .mockImplementationOnce(() => { + audioReleaseEntered.resolve(undefined); + return audioRelease.promise; + }); const harness = createHarness({ + releaseRecording, getTranscriber: () => ({ prepare: async () => preparedTranscription((_uri, { signal }) => { @@ -321,20 +331,31 @@ describe("VoiceInputController", () => { harness.controller[action](); expect(signal.aborted).toBe(true); - const next = createHarness(); - await next.controller.start(); - expect(next.controller.currentState.error).toContain("already active"); + const prepare = vi.fn(async () => preparedTranscription()); + const next = createHarness({ getTranscriber: () => ({ prepare }) }); + const nextStart = next.controller.start(); + expect(next.controller.currentState).toEqual({ + phase: "preparing", + error: null, + errorAction: null, + }); + expect(prepare).not.toHaveBeenCalled(); expect(next.recorder.record).not.toHaveBeenCalled(); transcription.resolve("late text"); + await audioReleaseEntered.promise; + expect(prepare).not.toHaveBeenCalled(); + expect(next.recorder.prepareToRecordAsync).not.toHaveBeenCalled(); + audioRelease.resolve(undefined); await stopping; + await nextStart; expect(harness.commits).toEqual([]); expect(harness.deleted).toEqual(["file:///voice.m4a"]); expect(harness.controller.currentState.phase).toBe("idle"); - await next.controller.start(); expect(next.controller.currentState.phase).toBe("recording"); + expect(prepare).toHaveBeenCalledTimes(1); await next.controller.interruptRecording(); }, ); @@ -431,7 +452,7 @@ describe("VoiceInputController", () => { expect(harness.controller.currentState.error).toContain("draft changed"); }); - it("keeps the app-wide session locked until canceled preparation settles", async () => { + it("queues the next recording until canceled preparation settles", async () => { const preparation = deferred(); const preparationEntered = deferred(); const first = createHarness({ @@ -447,21 +468,105 @@ describe("VoiceInputController", () => { first.controller.cancel(); expect(signal.aborted).toBe(true); - const blocked = createHarness(); - await blocked.controller.start(); - expect(blocked.controller.currentState.error).toContain("already active"); + const next = createHarness(); + const nextStart = next.controller.start(); + expect(next.controller.currentState.phase).toBe("preparing"); + expect(next.recorder.prepareToRecordAsync).not.toHaveBeenCalled(); preparation.resolve(preparedTranscription()); await firstStart; expect(first.recorder.record).not.toHaveBeenCalled(); - blocked.controller.cancel(); + await nextStart; + expect(next.controller.currentState.phase).toBe("recording"); + await next.controller.interruptRecording(); + }); + + it("still rejects a second start while the first recording is active", async () => { + const first = createHarness(); + await first.controller.start(); + const next = createHarness(); + await next.controller.start(); + expect(next.controller.currentState.error).toContain("already active"); + expect(next.recorder.record).not.toHaveBeenCalled(); + await first.controller.interruptRecording(); + }); + + it("does not start a canceled waiter or let it release another recording", async () => { + const preparation = deferred(); + const preparationEntered = deferred(); + const first = createHarness({ + getTranscriber: () => ({ + prepare: () => { + preparationEntered.resolve(undefined); + return preparation.promise; + }, + }), + }); + const firstStart = first.controller.start(); + await preparationEntered.promise; + first.controller.dispose(); + + const next = createHarness(); + const canceledStart = next.controller.start(); + next.controller.cancel(); + const nextStart = next.controller.start(); + preparation.resolve(preparedTranscription()); + await Promise.all([firstStart, canceledStart, nextStart]); + + expect(next.recorder.record).toHaveBeenCalledTimes(1); + expect(next.deleted).toEqual([]); + const blocked = createHarness(); + await blocked.controller.start(); + expect(blocked.controller.currentState.error).toContain("already active"); + await next.controller.interruptRecording(); + }); + it("releases an abandoned recording even if its native recorder was already disposed", async () => { + const harness = createHarness(); + await harness.controller.start(); + harness.recorder.stop.mockRejectedValueOnce(new Error("recorder released")); + Object.defineProperty(harness.recorder, "uri", { + get() { + throw new Error("recorder released"); + }, + }); + + await expect(harness.controller.interruptRecording()).resolves.toBeUndefined(); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); const next = createHarness(); await next.controller.start(); expect(next.controller.currentState.phase).toBe("recording"); await next.controller.interruptRecording(); }); + it("keeps the new abort controller when restarting the same composer after cancellation", async () => { + const transcription = deferred(); + const transcriptionEntered = deferred(); + const prepare = vi + .fn() + .mockResolvedValueOnce( + preparedTranscription(() => { + transcriptionEntered.resolve(undefined); + return transcription.promise; + }), + ) + .mockResolvedValue(preparedTranscription()); + const harness = createHarness({ getTranscriber: () => ({ prepare }) }); + await harness.controller.start(); + const stopping = harness.controller.stop(); + await transcriptionEntered.promise; + harness.controller.cancel(); + const restarting = harness.controller.start(); + transcription.resolve("late text"); + await Promise.all([stopping, restarting]); + await harness.controller.stop(); + + expect(harness.commits).toEqual([ + { text: "hello new text", selection: { start: 14, end: 14 } }, + ]); + expect(harness.controller.currentState.phase).toBe("idle"); + }); + it("does not start the microphone for an owner that changed during preparation", async () => { const preparation = deferred(); const preparationEntered = deferred(); diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index cb284ad9f85b..80a8a528f335 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -121,18 +121,28 @@ export function resolveTranscriptCommit( }; } -let activeSession: symbol | null = null; +type VoiceInputSession = { + abandoned: boolean; + readonly released: Promise; + readonly release: () => void; +}; + +let activeSession: VoiceInputSession | null = null; let activeTranscriptionOperation: Promise | null = null; -function acquireSession(): symbol | null { +function acquireSession(): VoiceInputSession | null { if (activeSession) return null; - const token = Symbol("voice-input-session"); + const { promise, resolve } = Promise.withResolvers(); + const token = { abandoned: false, released: promise, release: resolve }; activeSession = token; return token; } -function releaseSession(token: symbol | null): void { - if (token && activeSession === token) activeSession = null; +function releaseSession(token: VoiceInputSession | null): void { + if (token && activeSession === token) { + activeSession = null; + token.release(); + } } async function runTranscriptionOperation(operation: () => Promise): Promise { @@ -177,7 +187,7 @@ export class VoiceInputController { private readonly dependencies: VoiceInputControllerDependencies; private state: VoiceInputState = IDLE_STATE; private operationToken = 0; - private sessionToken: symbol | null = null; + private sessionToken: VoiceInputSession | null = null; private transcription: PreparedVoiceTranscription | null = null; private transcriptionAbortController: AbortController | null = null; private capturedDraft: VoiceDraftSnapshot | null = null; @@ -201,19 +211,28 @@ export class VoiceInputController { this.setError("This draft is no longer available.", "retry"); return; } - const sessionToken = acquireSession(); - if (!sessionToken) { - this.setError("Another voice recording is already active.", "retry"); - return; - } - - this.sessionToken = sessionToken; + let sessionToken: VoiceInputSession | null = null; const operationToken = ++this.operationToken; - const abortController = new AbortController(); - this.transcriptionAbortController = abortController; this.setState({ phase: "preparing", error: null, errorAction: null }); try { + for ( + let previousSession = activeSession; + previousSession?.abandoned; + previousSession = activeSession + ) { + await previousSession.released; + if (!this.isCurrent(operationToken)) return; + } + sessionToken = acquireSession(); + if (!sessionToken) { + this.setError("Another voice recording is already active.", "retry"); + return; + } + this.sessionToken = sessionToken; + const abortController = new AbortController(); + this.transcriptionAbortController = abortController; + const transcriber = this.dependencies.getTranscriber(); if (!transcriber) { this.setError("Voice transcription is not available.", null); @@ -260,10 +279,12 @@ export class VoiceInputController { if (this.isCurrent(operationToken)) this.setError("Could not start voice recording.", "retry"); } finally { - if (this.isCurrent(operationToken) && this.state.phase === "error") { - await this.releaseResources(); - } else if (!this.isCurrent(operationToken) && !this.finishing) { - await this.releaseResources(); + if (sessionToken && this.sessionToken === sessionToken) { + if (this.isCurrent(operationToken) && this.state.phase === "error") { + await this.releaseResources(); + } else if (!this.isCurrent(operationToken) && !this.finishing) { + await this.releaseResources(); + } } } } @@ -426,9 +447,7 @@ export class VoiceInputController { ); try { await this.dependencies.recorder.stop(); - this.rememberRecordingUri(this.dependencies.recorder.uri); } catch { - this.rememberRecordingUri(this.dependencies.recorder.uri); } finally { await this.releaseResources(); } @@ -436,7 +455,9 @@ export class VoiceInputController { private async releaseResources(): Promise { this.rememberRecordingUri(this.recordingUri); - this.rememberRecordingUri(this.dependencies.recorder.uri); + try { + this.rememberRecordingUri(this.dependencies.recorder.uri); + } catch {} this.recordingUri = null; for (const uri of this.ownedRecordingUris) { try { @@ -470,6 +491,7 @@ export class VoiceInputController { private invalidateOperation(): void { this.operationToken += 1; + if (this.sessionToken) this.sessionToken.abandoned = true; this.transcriptionAbortController?.abort(); }