From 962fef476bf5a73f3b986ca2d921090cdbbd5769 Mon Sep 17 00:00:00 2001 From: Rye Date: Sun, 15 Mar 2026 14:15:03 +0100 Subject: [PATCH 01/11] fix random resizing of recorder pop up by setting the width to a constant --- src/app/features/room/AudioMessageRecorder.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/features/room/AudioMessageRecorder.tsx b/src/app/features/room/AudioMessageRecorder.tsx index f324d444..0936588f 100644 --- a/src/app/features/room/AudioMessageRecorder.tsx +++ b/src/app/features/room/AudioMessageRecorder.tsx @@ -50,7 +50,7 @@ export function AudioMessageRecorder({ borderRadius: config.radii.R400, boxShadow: config.shadow.E200, padding: config.space.S400, - minWidth: 300, + width: 300, }} > Audio Message Recorder From 215225bd2844c8637988c2900b913b5df97079d6 Mon Sep 17 00:00:00 2001 From: Rye Date: Sun, 15 Mar 2026 14:56:14 +0100 Subject: [PATCH 02/11] fix audio codec choosing --- .../features/room/AudioMessageRecorder.tsx | 6 +++ src/app/features/room/RoomInput.tsx | 3 +- .../voice-recorder-kit/supportedCodec.ts | 51 +++++++++++++++++++ src/app/plugins/voice-recorder-kit/types.ts | 1 + .../voice-recorder-kit/useVoiceRecorder.ts | 15 ++++-- 5 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 src/app/plugins/voice-recorder-kit/supportedCodec.ts diff --git a/src/app/features/room/AudioMessageRecorder.tsx b/src/app/features/room/AudioMessageRecorder.tsx index 0936588f..073a8cb5 100644 --- a/src/app/features/room/AudioMessageRecorder.tsx +++ b/src/app/features/room/AudioMessageRecorder.tsx @@ -8,6 +8,7 @@ type AudioMessageRecorderProps = { onRequestClose: () => void; onWaveformUpdate: (waveform: number[]) => void; onAudioLengthUpdate: (length: number) => void; + onAudioCodecUpdate?: (codec: string) => void; }; // We use a react voice recorder library to handle the recording of audio messages, as it provides a simple API and handles the complexities of recording audio in the browser. @@ -19,6 +20,7 @@ export function AudioMessageRecorder({ onRequestClose, onWaveformUpdate, onAudioLengthUpdate, + onAudioCodecUpdate, }: AudioMessageRecorderProps) { const containerRef = useRef(null); const isDismissedRef = useRef(false); @@ -60,16 +62,20 @@ export function AudioMessageRecorder({ audioFile, waveform, audioLength, + audioCodec, }: { audioFile: Blob; waveform: number[]; audioLength: number; + audioCodec: string; }) => { if (isDismissedRef.current) return; // closes the recorder and sends the audio file back to the parent component to be uploaded and sent as a message onRecordingComplete(audioFile); onWaveformUpdate(waveform); onAudioLengthUpdate(audioLength); + // Pass the audio codec to the parent component + if (onAudioCodecUpdate) onAudioCodecUpdate(audioCodec); }} buttonBackgroundColor={color.SurfaceVariant.Container} buttonHoverBackgroundColor={color.SurfaceVariant.ContainerHover} diff --git a/src/app/features/room/RoomInput.tsx b/src/app/features/room/RoomInput.tsx index b280cf5d..39b389f6 100644 --- a/src/app/features/room/RoomInput.tsx +++ b/src/app/features/room/RoomInput.tsx @@ -159,6 +159,7 @@ import { } from './msgContent'; import { CommandAutocomplete } from './CommandAutocomplete'; import { AudioMessageRecorder } from './AudioMessageRecorder'; +import { getSupportedAudioExtension } from '$plugins/voice-recorder-kit/supportedCodec'; const getReplyContent = (replyDraft: IReplyDraft | undefined): IEventRelation => { if (!replyDraft) return {}; @@ -1096,7 +1097,7 @@ export const RoomInput = forwardRef( onRecordingComplete={(audioBlob) => { const file = new File( [audioBlob], - `sable-audio-message-${Date.now()}.ogg`, + `sable-audio-message-${Date.now()}.${getSupportedAudioExtension(audioBlob.type)}`, { type: audioBlob.type, } diff --git a/src/app/plugins/voice-recorder-kit/supportedCodec.ts b/src/app/plugins/voice-recorder-kit/supportedCodec.ts new file mode 100644 index 00000000..f78de6dc --- /dev/null +++ b/src/app/plugins/voice-recorder-kit/supportedCodec.ts @@ -0,0 +1,51 @@ +/** Codecs to test for */ +const codecs = [ + 'audio/ogg;codecs=speex', + 'audio/ogg;codecs=opus', + 'audio/ogg;codecs=vorbis', + 'audio/ogg', + 'audio/webm;codecs=opus', + 'audio/webm;codecs=vorbis', + 'audio/webm', + 'audio/mp4;codecs=aac', + 'audio/mp4', + 'audio/mpeg', + 'audio/wav;codecs=1', + 'audio/wav', +]; + +/** + * Checks for supported audio codecs in the current browser and returns the first supported codec. + * If no supported codec is found, it returns null. + */ +export function getSupportedAudioCodec(): string | null { + const supportedCodec = codecs.find((codec) => MediaRecorder.isTypeSupported(codec)); + return supportedCodec || null; +} + +/** + * Returns the appropriate file extension for a given audio codec. + * This is used to ensure that the recorded audio file has the correct extension based on the codec used for recording. + */ +export function getSupportedAudioExtension(codec: string): string { + switch (codec) { + case 'audio/ogg;codecs=opus': + case 'audio/ogg;codecs=vorbis': + case 'audio/ogg;codecs=speex': + case 'audio/ogg': + return 'ogg'; + case 'audio/webm;codecs=opus': + case 'audio/webm': + return 'webm'; + case 'audio/mp4': + case 'audio/mp4;codecs=aac': + return 'mp4'; + case 'audio/mpeg': + return 'mp3'; + case 'audio/wav;codecs=1': + case 'audio/wav': + return 'wav'; + default: + return 'dat'; // default extension for unknown codecs + } +} diff --git a/src/app/plugins/voice-recorder-kit/types.ts b/src/app/plugins/voice-recorder-kit/types.ts index 42cccd27..9834303c 100644 --- a/src/app/plugins/voice-recorder-kit/types.ts +++ b/src/app/plugins/voice-recorder-kit/types.ts @@ -7,6 +7,7 @@ export type VoiceRecorderStopPayload = { audioUrl: string; waveform: number[]; audioLength: number; + audioCodec: string; }; export type UseVoiceRecorderOptions = { diff --git a/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts b/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts index 45476246..08b76cd8 100644 --- a/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts +++ b/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts @@ -5,6 +5,7 @@ import type { RecorderState, VoiceRecorderStopPayload, } from './types'; +import { getSupportedAudioCodec, getSupportedAudioExtension } from './supportedCodec'; const BAR_COUNT = 40; const WAVEFORM_POINT_COUNT = 100; @@ -237,6 +238,12 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const codec = getSupportedAudioCodec(); + if (!codec) { + setError('No supported audio codec found for recording.'); + cleanupStream(); + return; + } streamRef.current = stream; chunksRef.current = []; previousChunksRef.current = []; @@ -245,7 +252,7 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic setupAudioGraph(stream); startRecordingTimer(); - const mediaRecorder = new MediaRecorder(stream); + const mediaRecorder = new MediaRecorder(stream, { mimeType: codec }); mediaRecorderRef.current = mediaRecorder; mediaRecorder.ondataavailable = (event: BlobEvent) => { @@ -278,7 +285,7 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic if (chunksRef.current.length === 0) return; - const blob = new Blob(chunksRef.current, { type: 'audio/ogg' }); + const blob = new Blob(chunksRef.current, { type: codec }); if (lastUrlRef.current) { URL.revokeObjectURL(lastUrlRef.current); } @@ -286,7 +293,9 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic lastUrlRef.current = url; setAudioUrl(url); - const file = new File([blob], `voice-${Date.now()}.ogg`, { type: 'audio/ogg' }); + const file = new File([blob], `voice-${Date.now()}.${getSupportedAudioExtension(codec)}`, { + type: codec, + }); setAudioFile(file); const waveformData = downsampleWaveform(waveformSamplesRef.current, WAVEFORM_POINT_COUNT); From 6daa6150d0085c781bd98a41163b683c1df4efe3 Mon Sep 17 00:00:00 2001 From: Rye Date: Sun, 15 Mar 2026 14:58:02 +0100 Subject: [PATCH 03/11] fix a few js errors oops --- src/app/features/room/RoomInput.tsx | 2 +- src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/features/room/RoomInput.tsx b/src/app/features/room/RoomInput.tsx index 39b389f6..777e5d2f 100644 --- a/src/app/features/room/RoomInput.tsx +++ b/src/app/features/room/RoomInput.tsx @@ -149,6 +149,7 @@ import { usePowerLevelsContext } from '$hooks/usePowerLevels'; import { useRoomCreators } from '$hooks/useRoomCreators'; import { useRoomPermissions } from '$hooks/useRoomPermissions'; import { AutocompleteNotice } from '$components/editor/autocomplete/AutocompleteNotice'; +import { getSupportedAudioExtension } from '$plugins/voice-recorder-kit/supportedCodec'; import { SchedulePickerDialog } from './schedule-send'; import * as css from './schedule-send/SchedulePickerDialog.css'; import { @@ -159,7 +160,6 @@ import { } from './msgContent'; import { CommandAutocomplete } from './CommandAutocomplete'; import { AudioMessageRecorder } from './AudioMessageRecorder'; -import { getSupportedAudioExtension } from '$plugins/voice-recorder-kit/supportedCodec'; const getReplyContent = (replyDraft: IReplyDraft | undefined): IEventRelation => { if (!replyDraft) return {}; diff --git a/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts b/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts index 08b76cd8..ef4fc8d1 100644 --- a/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts +++ b/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts @@ -143,6 +143,7 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic audioUrl: url, waveform: waveformData, audioLength, + audioCodec: file.type, }; onStop(payload); }, From 2bd3c37b6d24cdc48511db61582235577c7dd8c8 Mon Sep 17 00:00:00 2001 From: Rye Date: Sun, 15 Mar 2026 15:31:07 +0100 Subject: [PATCH 04/11] enhance audio message content structure and add MSC1767 compatibility --- src/app/features/room/msgContent.ts | 36 ++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/src/app/features/room/msgContent.ts b/src/app/features/room/msgContent.ts index 5536f7d0..12012429 100644 --- a/src/app/features/room/msgContent.ts +++ b/src/app/features/room/msgContent.ts @@ -155,28 +155,58 @@ export const getAudioMsgContent = ( audioLength?: number ): AudioMsgContent => { const { file, encInfo } = item; - const content: IContent = { + let content: IContent = { msgtype: MsgType.Audio, filename: file.name, - body: file.name, + body: item.body && item.body.length > 0 ? item.body : 'a voice message', format: 'org.matrix.custom.html', - formatted_body: file.name, + formatted_body: item.body && item.body.length > 0 ? item.body : 'a voice message', info: { mimetype: file.type, size: file.size, + duration: item.metadata.markedAsSpoiler || !audioLength ? 0 : audioLength * 1000, }, + + // Element-compatible unstable extensible-event keys 'org.matrix.msc1767.audio': { waveform: waveform?.map((v) => Math.round(v * 1024)), // scale waveform values to fit in 10 bits (0-1024) for more efficient storage, as per MSC1767 spec duration: item.metadata.markedAsSpoiler || !audioLength ? 0 : audioLength * 1000, // if marked as spoiler, set duration to 0 to hide it in clients that support msc1767 }, + 'org.matrix.msc1767.text': item.body && item.body.length > 0 ? item.body : 'a voice message', + 'org.matrix.msc3245.voice.v2': { + duration: !audioLength ? 0 : audioLength, + waveform: waveform?.map((v) => Math.round(v * 1024)), + }, }; if (encInfo) { content.file = { ...encInfo, url: mxc, }; + content = { + ...content, + + // Element-compatible unstable extensible-event keys + 'org.matrix.msc1767.file': { + name: file.name, + mimetype: file.type, + size: file.size, + file: content.file, + }, + }; } else { content.url = mxc; + content = { + ...content, + + // Element-compatible unstable extensible-event keys + 'org.matrix.msc1767.file': { + name: file.name, + mimetype: file.type, + size: file.size, + url: content.url, + }, + }; } if (item.body && item.body.length > 0) content.body = item.body; if (item.formatted_body && item.formatted_body.length > 0) { From 126894a346aaf5066f7b0eb0d229c84e37e60811 Mon Sep 17 00:00:00 2001 From: Rye Date: Sun, 15 Mar 2026 15:54:38 +0100 Subject: [PATCH 05/11] improve useVoiceRecorder to improve audio codec handling and enhance waveform sample documentation --- .../voice-recorder-kit/useVoiceRecorder.ts | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts b/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts index ef4fc8d1..2233fcaa 100644 --- a/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts +++ b/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts @@ -32,9 +32,21 @@ function downsampleWaveform(samples: number[], targetCount: number): number[] { return result; } +/** + * Custom React hook for recording voice messages using the MediaRecorder API. + * It manages the recording state, audio data, and provides functions to control the recording process (start, pause, stop, resume, play, etc.). + * It also handles audio visualization by analyzing the audio stream and generating levels for a visualizer. + * The hook supports multiple audio codecs and generates appropriate file extensions based on the supported codec. + */ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoiceRecorderReturn { const { autoStart = true, onStop, onDelete } = options; + /** + * The audio codec we will use + * we will choose depending on the browser support + */ + const audioCodec = getSupportedAudioCodec(); + const [isRecording, setIsRecording] = useState(false); const [isStopped, setIsStopped] = useState(false); const [isTemporaryStopped, setIsTemporaryStopped] = useState(false); @@ -68,9 +80,17 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic const isRestartingRef = useRef(false); const isTemporaryStopRef = useRef(false); const temporaryPreviewUrlRef = useRef(null); - // waveform samples collected during recording, used to generate waveform on stop. We collect all samples and downsample at the end to get a more accurate waveform, especially for short recordings. We use a ref to avoid causing re-renders on every sample. + /** + * waveform samples collected during recording, used to generate waveform on stop. + * We collect all samples and downsample at the end to get a more accurate waveform, especially for short recordings. + * We use a ref to avoid causing re-renders on every sample. + */ const waveformSamplesRef = useRef([]); - // Flag to indicate whether we should be collecting waveform samples. We need this because there can be a short delay between starting recording and the audio graph being set up, during which we might get some samples that we don't want to include in the waveform. + /** + * Flag to indicate whether we should be collecting waveform samples. + * We need this because there can be a short delay between starting recording + * and the audio graph being set up, during which we might get some samples that we don't want to include in the waveform. + */ const isCollectingWaveformRef = useRef(false); const cleanupStream = useCallback(() => { @@ -456,7 +476,7 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic chunksRef.current.length > 0 ? chunksRef.current : previousChunksRef.current; if (allChunks.length > 0) { - const blob = new Blob(allChunks, { type: 'audio/ogg' }); + const blob = new Blob(allChunks, { type: getSupportedAudioCodec() || 'audio/webm' }); urlToPlay = URL.createObjectURL(blob); temporaryPreviewUrlRef.current = urlToPlay; } @@ -602,7 +622,7 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic if (chunksRef.current.length === 0) return; - const blob = new Blob(chunksRef.current, { type: 'audio/webm' }); + const blob = new Blob(chunksRef.current, { type: audioCodec || 'audio/webm' }); if (lastUrlRef.current) { URL.revokeObjectURL(lastUrlRef.current); } @@ -610,7 +630,11 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic lastUrlRef.current = url; setAudioUrl(url); - const file = new File([blob], `voice-${Date.now()}.webm`, { type: 'audio/webm' }); + const file = new File( + [blob], + `voice-${Date.now()}.${getSupportedAudioExtension(blob.type)}`, + { type: blob.type } + ); setAudioFile(file); const waveformData = downsampleWaveform(waveformSamplesRef.current, WAVEFORM_POINT_COUNT); @@ -649,6 +673,7 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic isResumingRef.current = false; } }, [ + audioCodec, cleanupAudioContext, cleanupStream, emitStopPayload, From d4ad5cad5c62d720b30c0630d490e16f5f56ebd6 Mon Sep 17 00:00:00 2001 From: Rye Date: Sun, 15 Mar 2026 18:11:02 +0100 Subject: [PATCH 06/11] refactor supported codecs for improved Safari compatibility and correct file extensions --- .../voice-recorder-kit/supportedCodec.ts | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/app/plugins/voice-recorder-kit/supportedCodec.ts b/src/app/plugins/voice-recorder-kit/supportedCodec.ts index f78de6dc..e8f3dece 100644 --- a/src/app/plugins/voice-recorder-kit/supportedCodec.ts +++ b/src/app/plugins/voice-recorder-kit/supportedCodec.ts @@ -1,17 +1,25 @@ /** Codecs to test for */ const codecs = [ - 'audio/ogg;codecs=speex', + // silly webkit-prefixed codecs for Safari support, because safari, apparently lies when asked what it supports + 'audio/mp4;codecs=mp4a.40.2', + 'audio/mp4;codecs=mp4a.40.5', + 'audio/aac', + // Firefox 'audio/ogg;codecs=opus', 'audio/ogg;codecs=vorbis', 'audio/ogg', + // Chromium / Firefox 'audio/webm;codecs=opus', - 'audio/webm;codecs=vorbis', 'audio/webm', + // fallback + 'audio/wav;codecs=1', + 'audio/wav', + // other Codecs + 'audio/ogg;codecs=speex', + 'audio/webm;codecs=vorbis', 'audio/mp4;codecs=aac', 'audio/mp4', 'audio/mpeg', - 'audio/wav;codecs=1', - 'audio/wav', ]; /** @@ -19,6 +27,11 @@ const codecs = [ * If no supported codec is found, it returns null. */ export function getSupportedAudioCodec(): string | null { + const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); + + if (isSafari && MediaRecorder.isTypeSupported('audio/mp4')) { + return 'audio/mp4'; + } const supportedCodec = codecs.find((codec) => MediaRecorder.isTypeSupported(codec)); return supportedCodec || null; } @@ -39,12 +52,18 @@ export function getSupportedAudioExtension(codec: string): string { return 'webm'; case 'audio/mp4': case 'audio/mp4;codecs=aac': - return 'mp4'; + return 'm4a'; case 'audio/mpeg': return 'mp3'; case 'audio/wav;codecs=1': case 'audio/wav': return 'wav'; + // silly webkit stuff + case 'audio/mp4;codecs=mp4a.40.2': + case 'audio/mp4;codecs=mp4a.40.5': + return 'm4a'; + case 'audio/aac': + return 'aac'; default: return 'dat'; // default extension for unknown codecs } From b02e345b5e4b575102204a489ff621432a335744 Mon Sep 17 00:00:00 2001 From: Rye Date: Sun, 15 Mar 2026 18:40:37 +0100 Subject: [PATCH 07/11] element compat --- src/app/features/room/msgContent.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/app/features/room/msgContent.ts b/src/app/features/room/msgContent.ts index 12012429..e5de20f2 100644 --- a/src/app/features/room/msgContent.ts +++ b/src/app/features/room/msgContent.ts @@ -177,6 +177,8 @@ export const getAudioMsgContent = ( duration: !audioLength ? 0 : audioLength, waveform: waveform?.map((v) => Math.round(v * 1024)), }, + // for element compat + 'org.matrix.msc3245.voice': {}, }; if (encInfo) { content.file = { From 8a517b1bdb23b1bf22b97c575735b10ad9ef2ee3 Mon Sep 17 00:00:00 2001 From: Rye Date: Sun, 15 Mar 2026 18:43:35 +0100 Subject: [PATCH 08/11] add changeset --- .changeset/fix_voice_message_element_compat.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fix_voice_message_element_compat.md diff --git a/.changeset/fix_voice_message_element_compat.md b/.changeset/fix_voice_message_element_compat.md new file mode 100644 index 00000000..57270a55 --- /dev/null +++ b/.changeset/fix_voice_message_element_compat.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +fix of compatibility of voice messages with element clients and style misshaps From 7926c99d2e2f95c7928588be54edbee4cf26e3fe Mon Sep 17 00:00:00 2001 From: Rye Date: Sun, 15 Mar 2026 18:49:35 +0100 Subject: [PATCH 09/11] fix: improve cleanup process and ensure correct audio codec usage in useVoiceRecorder --- .../plugins/voice-recorder-kit/useVoiceRecorder.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts b/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts index 2233fcaa..f3fa291a 100644 --- a/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts +++ b/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts @@ -396,6 +396,7 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic mediaRecorder.requestData(); } mediaRecorder.stop(); + cleanupStream(); setIsStopped(true); setIsTemporaryStopped(false); @@ -434,6 +435,7 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic previousChunksRef.current = [...chunksRef.current]; isTemporaryStopRef.current = false; mediaRecorder.stop(); + cleanupStream(); setIsStopped(true); setIsTemporaryStopped(false); setIsPaused(false); @@ -476,7 +478,7 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic chunksRef.current.length > 0 ? chunksRef.current : previousChunksRef.current; if (allChunks.length > 0) { - const blob = new Blob(allChunks, { type: getSupportedAudioCodec() || 'audio/webm' }); + const blob = new Blob(allChunks, { type: audioCodec || 'audio/webm' }); urlToPlay = URL.createObjectURL(blob); temporaryPreviewUrlRef.current = urlToPlay; } @@ -531,12 +533,13 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic } }, [ audioUrl, - cleanupAudioContext, - isPlaying, isPaused, + isPlaying, + audioCodec, + stopTimer, + cleanupAudioContext, setupPlaybackGraph, startPlaybackTimer, - stopTimer, ]); const handlePlay = useCallback(() => { From 9ecfb84dd1be9bccc769744e4929c4f57031049d Mon Sep 17 00:00:00 2001 From: Rye Date: Mon, 16 Mar 2026 14:03:05 +0100 Subject: [PATCH 10/11] fixes for iOS (hopefully) --- .../voice-recorder-kit/supportedCodec.ts | 51 ++++++--- .../voice-recorder-kit/useVoiceRecorder.ts | 100 ++++++++++++++---- 2 files changed, 115 insertions(+), 36 deletions(-) diff --git a/src/app/plugins/voice-recorder-kit/supportedCodec.ts b/src/app/plugins/voice-recorder-kit/supportedCodec.ts index e8f3dece..fd848e95 100644 --- a/src/app/plugins/voice-recorder-kit/supportedCodec.ts +++ b/src/app/plugins/voice-recorder-kit/supportedCodec.ts @@ -1,25 +1,37 @@ -/** Codecs to test for */ -const codecs = [ - // silly webkit-prefixed codecs for Safari support, because safari, apparently lies when asked what it supports +const safariPreferredCodecs = [ + // Safari works best with MP4/AAC but fails when strict codecs are defined on iOS. + // Prioritize the plain container to avoid NotSupportedError during MediaRecorder initialization. + 'audio/mp4', 'audio/mp4;codecs=mp4a.40.2', 'audio/mp4;codecs=mp4a.40.5', + 'audio/mp4;codecs=aac', 'audio/aac', + // Fallbacks + 'audio/wav;codecs=1', + 'audio/wav', + 'audio/mpeg', +]; + +const defaultPreferredCodecs = [ + // Chromium / Firefox stable path. + 'audio/webm;codecs=opus', + 'audio/webm', // Firefox 'audio/ogg;codecs=opus', 'audio/ogg;codecs=vorbis', 'audio/ogg', - // Chromium / Firefox - 'audio/webm;codecs=opus', - 'audio/webm', - // fallback + // Fallbacks 'audio/wav;codecs=1', 'audio/wav', - // other Codecs - 'audio/ogg;codecs=speex', - 'audio/webm;codecs=vorbis', + 'audio/mpeg', + // Keep MP4/AAC as late fallback for non-Safari browsers. + 'audio/mp4;codecs=mp4a.40.2', + 'audio/mp4;codecs=mp4a.40.5', 'audio/mp4;codecs=aac', 'audio/mp4', - 'audio/mpeg', + 'audio/aac', + 'audio/ogg;codecs=speex', + 'audio/webm;codecs=vorbis', ]; /** @@ -27,12 +39,19 @@ const codecs = [ * If no supported codec is found, it returns null. */ export function getSupportedAudioCodec(): string | null { - const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); - - if (isSafari && MediaRecorder.isTypeSupported('audio/mp4')) { - return 'audio/mp4'; + if (!('MediaRecorder' in globalThis) || !globalThis.MediaRecorder) { + return null; } - const supportedCodec = codecs.find((codec) => MediaRecorder.isTypeSupported(codec)); + + const userAgent = globalThis.navigator?.userAgent ?? ''; + const isIOS = + /iPad|iPhone|iPod/.test(userAgent) || + // eslint-disable-next-line @typescript-eslint/no-deprecated + (globalThis.navigator?.platform === 'MacIntel' && globalThis.navigator?.maxTouchPoints > 1); + const isSafari = /^((?!chrome|android|crios|fxios|edgios).)*safari/i.test(userAgent) || isIOS; + + const preferredCodecs = isSafari ? safariPreferredCodecs : defaultPreferredCodecs; + const supportedCodec = preferredCodecs.find((codec) => MediaRecorder.isTypeSupported(codec)); return supportedCodec || null; } diff --git a/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts b/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts index f3fa291a..a1eab6dd 100644 --- a/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts +++ b/src/app/plugins/voice-recorder-kit/useVoiceRecorder.ts @@ -209,7 +209,7 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic }, []); const setupAudioGraph = useCallback( - (stream: MediaStream) => { + (stream: MediaStream): MediaStream => { const audioContext = new AudioContext(); audioContextRef.current = audioContext; const source = audioContext.createMediaStreamSource(stream); @@ -220,9 +220,17 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic const dataArray = new Uint8Array(bufferLength); analyserRef.current = analyser; dataArrayRef.current = dataArray; + + // Fix for iOS Safari: routing the stream through a MediaStreamDestination + // prevents the AudioContext from "stealing" the track from the MediaRecorder + const destination = audioContext.createMediaStreamDestination(); source.connect(analyser); + analyser.connect(destination); + audioContext.resume().catch(() => {}); animateLevels(); + + return destination.stream; }, [animateLevels] ); @@ -270,10 +278,10 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic previousChunksRef.current = []; waveformSamplesRef.current = []; isCollectingWaveformRef.current = true; - setupAudioGraph(stream); + const recordedStream = setupAudioGraph(stream); startRecordingTimer(); - const mediaRecorder = new MediaRecorder(stream, { mimeType: codec }); + const mediaRecorder = new MediaRecorder(recordedStream, { mimeType: codec }); mediaRecorderRef.current = mediaRecorder; mediaRecorder.ondataavailable = (event: BlobEvent) => { @@ -304,9 +312,20 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic return; } - if (chunksRef.current.length === 0) return; + if (chunksRef.current.length === 0) { + if (isTemporaryStopRef.current) { + setIsTemporaryStopped(true); + setIsStopped(true); + isTemporaryStopRef.current = false; + } else { + setIsStopped(true); + setIsTemporaryStopped(false); + } + return; + } - const blob = new Blob(chunksRef.current, { type: codec }); + const actualType = chunksRef.current[0]?.type || codec || 'audio/webm'; + const blob = new Blob(chunksRef.current, { type: actualType }); if (lastUrlRef.current) { URL.revokeObjectURL(lastUrlRef.current); } @@ -314,9 +333,13 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic lastUrlRef.current = url; setAudioUrl(url); - const file = new File([blob], `voice-${Date.now()}.${getSupportedAudioExtension(codec)}`, { - type: codec, - }); + const file = new File( + [blob], + `voice-${Date.now()}.${getSupportedAudioExtension(actualType)}`, + { + type: actualType, + } + ); setAudioFile(file); const waveformData = downsampleWaveform(waveformSamplesRef.current, WAVEFORM_POINT_COUNT); @@ -333,7 +356,9 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic } }; - mediaRecorder.start(); + // Pass a timeslice to ensure Safari iOS periodically flushes chunks + // Otherwise Safari might fail to emit any chunks when stopped abruptly + mediaRecorder.start(1000); setIsRecording(true); setIsPaused(false); setIsStopped(false); @@ -393,11 +418,21 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic isTemporaryStopRef.current = false; if (mediaRecorder.state === 'recording') { - mediaRecorder.requestData(); + try { + mediaRecorder.requestData(); + } catch { + // ignore + } + } + + try { + mediaRecorder.stop(); + } catch { + // ignore } - mediaRecorder.stop(); - cleanupStream(); + // Let cleanupStream() be handled by mediaRecorder.onstop + // Calling it synchronously here can kill the stream before Safari finishes emitting data setIsStopped(true); setIsTemporaryStopped(false); setIsPaused(false); @@ -434,8 +469,23 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic if (mediaRecorder && mediaRecorder.state !== 'inactive') { previousChunksRef.current = [...chunksRef.current]; isTemporaryStopRef.current = false; - mediaRecorder.stop(); - cleanupStream(); + + if (mediaRecorder.state === 'recording') { + try { + mediaRecorder.requestData(); + } catch { + // ignore + } + } + + try { + mediaRecorder.stop(); + } catch { + // ignore + } + + // Let cleanupStream() be handled by mediaRecorder.onstop + // Calling it synchronously here can kill the stream before Safari finishes emitting data setIsStopped(true); setIsTemporaryStopped(false); setIsPaused(false); @@ -478,7 +528,8 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic chunksRef.current.length > 0 ? chunksRef.current : previousChunksRef.current; if (allChunks.length > 0) { - const blob = new Blob(allChunks, { type: audioCodec || 'audio/webm' }); + const actualType = allChunks[0]?.type || audioCodec || 'audio/webm'; + const blob = new Blob(allChunks, { type: actualType }); urlToPlay = URL.createObjectURL(blob); temporaryPreviewUrlRef.current = urlToPlay; } @@ -596,13 +647,16 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); streamRef.current = stream; - setupAudioGraph(stream); + const recordedStream = setupAudioGraph(stream); // Force update seconds to the correct total time before starting timer setSeconds(pausedTimeRef.current); startRecordingTimer(); - const mediaRecorder = new MediaRecorder(stream); + const codec = getSupportedAudioCodec() || audioCodec; + const mediaRecorder = codec + ? new MediaRecorder(recordedStream, { mimeType: codec }) + : new MediaRecorder(recordedStream); mediaRecorderRef.current = mediaRecorder; mediaRecorder.ondataavailable = (event: BlobEvent) => { @@ -623,9 +677,13 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic isCollectingWaveformRef.current = false; - if (chunksRef.current.length === 0) return; + if (chunksRef.current.length === 0) { + setIsStopped(true); + return; + } - const blob = new Blob(chunksRef.current, { type: audioCodec || 'audio/webm' }); + const actualType = chunksRef.current[0]?.type || audioCodec || 'audio/webm'; + const blob = new Blob(chunksRef.current, { type: actualType }); if (lastUrlRef.current) { URL.revokeObjectURL(lastUrlRef.current); } @@ -646,7 +704,9 @@ export function useVoiceRecorder(options: UseVoiceRecorderOptions = {}): UseVoic emitStopPayload(file, url, waveformData, audioLength); }; - mediaRecorder.start(); + // Pass a timeslice to ensure Safari iOS periodically flushes chunks + // Otherwise Safari might fail to emit any chunks when stopped abruptly + mediaRecorder.start(1000); setIsRecording(true); setIsPaused(false); setIsStopped(false); From 166c8f4835d10e0ab10b209736c5b4da69980e6a Mon Sep 17 00:00:00 2001 From: Rye Date: Mon, 16 Mar 2026 14:10:56 +0100 Subject: [PATCH 11/11] fix file extension mapping --- .../plugins/voice-recorder-kit/supportedCodec.ts | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/app/plugins/voice-recorder-kit/supportedCodec.ts b/src/app/plugins/voice-recorder-kit/supportedCodec.ts index fd848e95..99481bde 100644 --- a/src/app/plugins/voice-recorder-kit/supportedCodec.ts +++ b/src/app/plugins/voice-recorder-kit/supportedCodec.ts @@ -60,27 +60,18 @@ export function getSupportedAudioCodec(): string | null { * This is used to ensure that the recorded audio file has the correct extension based on the codec used for recording. */ export function getSupportedAudioExtension(codec: string): string { - switch (codec) { - case 'audio/ogg;codecs=opus': - case 'audio/ogg;codecs=vorbis': - case 'audio/ogg;codecs=speex': + const baseType = codec.split(';')[0].trim(); + switch (baseType) { case 'audio/ogg': return 'ogg'; - case 'audio/webm;codecs=opus': case 'audio/webm': return 'webm'; case 'audio/mp4': - case 'audio/mp4;codecs=aac': return 'm4a'; case 'audio/mpeg': return 'mp3'; - case 'audio/wav;codecs=1': case 'audio/wav': return 'wav'; - // silly webkit stuff - case 'audio/mp4;codecs=mp4a.40.2': - case 'audio/mp4;codecs=mp4a.40.5': - return 'm4a'; case 'audio/aac': return 'aac'; default: