diff --git a/apps/mobile/README.md b/apps/mobile/README.md index 78ff5e7..bcd941e 100644 --- a/apps/mobile/README.md +++ b/apps/mobile/README.md @@ -30,6 +30,12 @@ npm ci npm run web ``` +Voice prompts use the iOS and Android system speech-recognition services, which +may use network transcription depending on the device and its settings. Because +that support includes native code, this app cannot run in Expo Go; test it in a +development build with `npm run ios` or `npm run android`. On web, dictation is +shown when the browser provides the Web Speech API. + ## Validate From the repo root: diff --git a/apps/mobile/app.json b/apps/mobile/app.json index 86df94f..ef3a77c 100644 --- a/apps/mobile/app.json +++ b/apps/mobile/app.json @@ -58,7 +58,14 @@ { "photosPermission": "Allow Brio to attach photos as Hermes context.", "cameraPermission": "Allow Brio to take a photo and attach it as Hermes context.", - "microphonePermission": false + "microphonePermission": "Allow Brio to use the microphone for voice prompts." + } + ], + [ + "expo-speech-recognition", + { + "microphonePermission": "Allow Brio to use the microphone for voice prompts.", + "speechRecognitionPermission": "Allow Brio to turn your speech into editable prompt text." } ], [ diff --git a/apps/mobile/package-lock.json b/apps/mobile/package-lock.json index f855bf6..a4b4d65 100644 --- a/apps/mobile/package-lock.json +++ b/apps/mobile/package-lock.json @@ -27,6 +27,7 @@ "expo-router": "~57.0.15", "expo-secure-store": "~57.0.1", "expo-sharing": "~57.0.14", + "expo-speech-recognition": "^56.0.4", "expo-splash-screen": "~57.0.7", "expo-status-bar": "~57.0.1", "expo-symbols": "~57.0.2", @@ -8595,6 +8596,17 @@ "react-native": "*" } }, + "node_modules/expo-speech-recognition": { + "version": "56.0.4", + "resolved": "https://registry.npmjs.org/expo-speech-recognition/-/expo-speech-recognition-56.0.4.tgz", + "integrity": "sha512-6Sh6aeKSG4W81TYfHUL8sgOxO++YrUi5Zhpmu5tiWurm6NP5hFL1SHd6+GbtV7Fyo4Cc35AofR9QxiSROZMJcQ==", + "license": "MIT", + "peerDependencies": { + "expo": "*", + "react": "*", + "react-native": "*" + } + }, "node_modules/expo-splash-screen": { "version": "57.0.7", "resolved": "https://registry.npmjs.org/expo-splash-screen/-/expo-splash-screen-57.0.7.tgz", diff --git a/apps/mobile/package.json b/apps/mobile/package.json index c761b97..463ef2f 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -22,6 +22,7 @@ "expo-router": "~57.0.15", "expo-secure-store": "~57.0.1", "expo-sharing": "~57.0.14", + "expo-speech-recognition": "^56.0.4", "expo-splash-screen": "~57.0.7", "expo-status-bar": "~57.0.1", "expo-symbols": "~57.0.2", diff --git a/apps/mobile/src/components/composer-controls.tsx b/apps/mobile/src/components/composer-controls.tsx index 3d4d051..3729f07 100644 --- a/apps/mobile/src/components/composer-controls.tsx +++ b/apps/mobile/src/components/composer-controls.tsx @@ -1,8 +1,11 @@ import { useQuery } from '@tanstack/react-query'; import * as DocumentPicker from 'expo-document-picker'; import * as ImagePicker from 'expo-image-picker'; +import { useIsFocused } from 'expo-router'; +import { SymbolView } from 'expo-symbols'; import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from 'react'; import { + AppState, Modal, Platform, Pressable, @@ -36,6 +39,22 @@ import { } from '@/lib/brio'; import { applyCompletion, completionToken, composerContentFromSharedPayloads } from '@/lib/composer-model'; import { uploadComposerAttachment, type AttachmentSource } from '@/lib/composer'; +import { + abortSpeechRecognition, + claimSpeechRecognition, + isSpeechRecognitionAvailable, + ownsSpeechRecognition, + releaseSpeechRecognitionBeforeStart, + requestSpeechRecognitionPermissions, + startSpeechRecognition, + stopSpeechRecognition, +} from '@/lib/speech-recognition-controller'; +import { + appendSpeechTranscript, + mergeSpeechSegment, + normalizeSpeechRecognitionLocale, + speechRecognitionErrorMessage, +} from '@/lib/speech-recognition-model'; import type { ComposerAttachment, PromptDeliveryMode } from '@/state/composer-store-model'; type UploadState = { @@ -46,6 +65,8 @@ type UploadState = { error?: string; }; +type SpeechState = 'idle' | 'starting' | 'listening' | 'stopping'; + const COMPOSER_MOTION = { duration: 230, easing: Easing.bezier(0.2, 0, 0, 1), @@ -64,6 +85,7 @@ export function ComposerControls({ hydrated, keyboardVisible, forceExpanded = false, + visible = true, modelControl, onAddAttachment, onDraftChange, @@ -85,6 +107,7 @@ export function ComposerControls({ hydrated: boolean; keyboardVisible?: boolean; forceExpanded?: boolean; + visible?: boolean; modelControl?: ReactNode; onAddAttachment: (attachment: ComposerAttachment) => Promise; onDraftChange: (text: string) => void; @@ -98,9 +121,14 @@ export function ComposerControls({ }) { const colors = useTheme(); const incomingShare = useIncomingShareContext(); + const isFocused = useIsFocused(); const processedShare = useRef(null); const inputRef = useRef(null); const previousKeyboardVisible = useRef(keyboardVisible); + const speechBaseDraft = useRef(''); + const committedSpeech = useRef(''); + const speechOwner = useRef(Symbol('composer-speech-owner')); + const speechRequest = useRef(0); const [pickerOpen, setPickerOpen] = useState(false); const [historyOpen, setHistoryOpen] = useState(false); const [commandsOpen, setCommandsOpen] = useState(false); @@ -108,6 +136,7 @@ export function ComposerControls({ const [inputHeight, setInputHeight] = useState(42); const [uploads, setUploads] = useState([]); const [error, setError] = useState(''); + const [speechState, setSpeechState] = useState('idle'); const token = completionToken(draft); const capabilities = useQuery({ @@ -130,17 +159,21 @@ export function ComposerControls({ enabled: commandsOpen, staleTime: 60_000, }); + const speechActive = speechState !== 'idle'; + const hasPrompt = Boolean(draft.trim()) || attachments.length > 0; const canSend = hydrated && !sendDisabled && - (Boolean(draft.trim()) || attachments.length > 0) && - uploads.length === 0; + hasPrompt && + uploads.length === 0 && + !speechActive; const expanded = forceExpanded || inputFocused || pickerOpen || historyOpen || commandsOpen || + speechActive || attachments.length > 0 || uploads.length > 0 || Boolean(error) || @@ -181,6 +214,37 @@ export function ComposerControls({ return () => clearTimeout(timer); }, [inputFocused, keyboardVisible]); + useEffect(() => { + if (isFocused) return; + speechRequest.current += 1; + abortSpeechRecognition(speechOwner.current); + if (!ownsSpeechRecognition(speechOwner.current)) setSpeechState('idle'); + }, [isFocused]); + + useEffect(() => { + if (visible) return; + speechRequest.current += 1; + abortSpeechRecognition(speechOwner.current); + if (!ownsSpeechRecognition(speechOwner.current)) setSpeechState('idle'); + }, [visible]); + + useEffect(() => { + const subscription = AppState.addEventListener('change', (state) => { + // iOS permission sheets can transiently report `inactive`; only a real + // background transition should cancel the permission/start sequence. + if (state !== 'background') return; + speechRequest.current += 1; + abortSpeechRecognition(speechOwner.current); + if (!ownsSpeechRecognition(speechOwner.current)) setSpeechState('idle'); + }); + return () => subscription.remove(); + }, []); + + useEffect(() => () => { + speechRequest.current += 1; + abortSpeechRecognition(speechOwner.current); + }, [connection.id, profile, sessionId]); + const uploadSources = useCallback(async (sources: AttachmentSource[]) => { setPickerOpen(false); setError(''); @@ -230,7 +294,7 @@ export function ComposerControls({ }, [attachments, capabilities.data, connection, onAddAttachment, profile, sessionId]); useEffect(() => { - if (incomingShare.isResolving || incomingShare.error || incomingShare.sharedPayloads.length === 0) return; + if (speechActive || incomingShare.isResolving || incomingShare.error || incomingShare.sharedPayloads.length === 0) return; const payloads = incomingShare.resolvedSharedPayloads.length ? incomingShare.resolvedSharedPayloads : incomingShare.sharedPayloads; @@ -244,7 +308,7 @@ export function ComposerControls({ incomingShare.consumeSharedPayloads(); if (!succeeded) setError('Some shared files could not be imported. Share them again to retry.'); })(); - }, [draft, incomingShare, onDraftChange, uploadSources]); + }, [draft, incomingShare, onDraftChange, speechActive, uploadSources]); useEffect(() => { if (Platform.OS !== 'web' || typeof document === 'undefined') return; @@ -317,10 +381,87 @@ export function ComposerControls({ if (focused === inputFocused) return; setInputFocused(focused); }; + const startSpeech = async () => { + const owner = speechOwner.current; + const request = ++speechRequest.current; + setError(''); + inputRef.current?.blur(); + setInputFocused(false); + setSpeechState('starting'); + try { + if (!isSpeechRecognitionAvailable()) { + setError('Speech recognition is not available on this device or browser.'); + setSpeechState('idle'); + return; + } + const permission = await requestSpeechRecognitionPermissions(); + if (request !== speechRequest.current) return; + if (!permission.granted) { + setError('Microphone and speech recognition access are required. Enable them in device settings.'); + setSpeechState('idle'); + return; + } + if (AppState.currentState !== 'active' || !visible || !isFocused) { + setSpeechState('idle'); + return; + } + const claimed = claimSpeechRecognition(owner, { + onStart: () => setSpeechState('listening'), + onResult: (event) => { + const segment = event.results[0]?.transcript ?? ''; + if (!segment.trim()) return; + if (event.isFinal) committedSpeech.current = mergeSpeechSegment(committedSpeech.current, segment); + const transcript = event.isFinal + ? committedSpeech.current + : mergeSpeechSegment(committedSpeech.current, segment); + onDraftChange(appendSpeechTranscript(speechBaseDraft.current, transcript)); + }, + onNoMatch: () => setError("I didn't hear any speech. Tap the mic and try again."), + onError: (event) => { + const message = speechRecognitionErrorMessage(event.error); + if (message) setError(message); + setSpeechState('stopping'); + }, + onEnd: () => setSpeechState('idle'), + }); + if (!claimed) { + setError('Dictation is already active in another composer.'); + setSpeechState('idle'); + return; + } + speechBaseDraft.current = draft; + committedSpeech.current = ''; + startSpeechRecognition(owner, { + addsPunctuation: true, + continuous: false, + interimResults: true, + lang: normalizeSpeechRecognitionLocale(Intl.DateTimeFormat().resolvedOptions().locale), + maxAlternatives: 1, + }); + } catch (reason) { + releaseSpeechRecognitionBeforeStart(owner); + setSpeechState('idle'); + setError(reason instanceof Error ? reason.message : 'Could not start speech recognition.'); + } + }; + const stopSpeech = () => { + if (speechState === 'starting') { + speechRequest.current += 1; + abortSpeechRecognition(speechOwner.current); + if (!ownsSpeechRecognition(speechOwner.current)) setSpeechState('idle'); + return; + } + if (speechState === 'stopping') { + abortSpeechRecognition(speechOwner.current); + return; + } + setSpeechState('stopping'); + stopSpeechRecognition(speechOwner.current); + }; return ( <> - {suggestions.data?.items.length && token ? ( + {!speechActive && suggestions.data?.items.length && token ? ( {suggestions.data.items.slice(0, 12).map((item) => ( onDraftChange(applyCompletion(draft, token, item.text))} style={styles.completionRow}> @@ -389,6 +530,7 @@ export function ComposerControls({ { @@ -403,7 +545,7 @@ export function ComposerControls({ setInputHeight(nextHeight); }} onFocus={() => setFocused(true)} - placeholder={active ? 'Ask a follow-up…' : 'Ask Hermes anything…'} + placeholder={speechActive ? 'Listening…' : active ? 'Ask a follow-up…' : 'Ask Hermes anything…'} placeholderTextColor={colors.textTertiary} ref={inputRef} scrollEnabled={expanded && inputHeight >= 96} @@ -420,24 +562,28 @@ export function ComposerControls({ - onSend('queue')} - style={({ pressed }) => [ - styles.sendPressable, - { - backgroundColor: canSend ? colors.accent : colors.backgroundSelected, - opacity: pressed ? 0.72 : 1, - }, - ]}> - - ↑ - - + {!hasPrompt && uploads.length === 0 ? ( + {expanded ? ( @@ -448,12 +594,16 @@ export function ComposerControls({ keyboardShouldPersistTaps="handled" style={styles.toolbarScroller} showsHorizontalScrollIndicator={false}> - setPickerOpen(true)} /> - {modelControl} - setCommandsOpen(true)} /> - {history.length > 0 ? setHistoryOpen(true)} /> : null} - {canUndo ? : null} - {canRedo ? : null} + setPickerOpen(true)} /> + void startSpeech()} + state={speechState} + /> + {speechActive ? null : modelControl} + setCommandsOpen(true)} /> + {history.length > 0 ? setHistoryOpen(true)} /> : null} + {canUndo ? : null} + {canRedo ? : null} {active ? onSend('redirect')} tone="warning" /> : null}