From 6ec5c1584543b9cb902e413988c71c4708756ec2 Mon Sep 17 00:00:00 2001 From: Anuraj Jit Saikia Date: Thu, 30 Jul 2026 19:36:33 +0530 Subject: [PATCH] =?UTF-8?q?feat(transcription):=20background-only=20?= =?UTF-8?q?=E2=80=94=20remove=20captions=20overlay=20and=20interim=20broad?= =?UTF-8?q?cast?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deletes the LiveCaptions on-stage overlay and its page state, removes the transcript-interim event end-to-end (shared contracts, server room-wide emit, client subscription, panel interim rows). The transcript panel now shows finalized speech turns only. Deepgram sessions keep consuming interims internally to assemble finals; start/stop auth, consent, snapshot-on-join, and download are unchanged. Closes #194 --- client/src/components/LiveCaptions.tsx | 60 ------------------- .../src/components/TranscriptPanel.test.tsx | 52 ++++++++++++++++ client/src/components/TranscriptPanel.tsx | 26 +------- client/src/pages/RoomPage.tsx | 32 ---------- client/tsconfig.json | 2 +- .../transcription/meeting-transcription.ts | 11 +--- server/test/deepgram-session.test.js | 2 +- server/test/meeting-transcription.test.js | 15 +++++ shared/src/events.ts | 7 --- 9 files changed, 74 insertions(+), 133 deletions(-) delete mode 100644 client/src/components/LiveCaptions.tsx create mode 100644 client/src/components/TranscriptPanel.test.tsx diff --git a/client/src/components/LiveCaptions.tsx b/client/src/components/LiveCaptions.tsx deleted file mode 100644 index d7efdc7..0000000 --- a/client/src/components/LiveCaptions.tsx +++ /dev/null @@ -1,60 +0,0 @@ -import { Avatar, Box, Typography } from '@mui/material'; - -interface CaptionSpeaker { - name?: string; - avatar?: string; -} - -interface CaptionEntry { - speaker?: CaptionSpeaker | null; - text?: string; -} - -interface LiveCaptionsProps { - entry?: CaptionEntry | null; - interim?: CaptionEntry | null; -} - -export default function LiveCaptions({ entry, interim }: LiveCaptionsProps) { - const speaker = interim?.text ? interim.speaker : entry?.speaker; - const text = interim?.text || entry?.text; - if (!text) return null; - - return ( - - - {speaker?.name?.[0] || '?'} - - - - {speaker?.name || 'Participant'}{interim?.text ? ' · listening' : ''} - - - {text} - - - - ); -} diff --git a/client/src/components/TranscriptPanel.test.tsx b/client/src/components/TranscriptPanel.test.tsx new file mode 100644 index 0000000..bd5203d --- /dev/null +++ b/client/src/components/TranscriptPanel.test.tsx @@ -0,0 +1,52 @@ +import { render, screen } from '@testing-library/react'; +import { ThemeProvider } from '@mui/material/styles'; +import { beforeAll, describe, expect, it, vi } from 'vitest'; +import theme from '../theme/theme'; +import TranscriptPanel from './TranscriptPanel'; + +beforeAll(() => { + Element.prototype.scrollIntoView = vi.fn(); +}); + +const finalizedEntries = [{ + id: 'final-1', + speaker: { name: 'Alice' }, + ts: new Date('2026-07-30T10:15:00Z'), + text: 'The finalized speech turn.', +}, { + id: 'final-2', + speaker: { name: 'Bob' }, + ts: new Date('2026-07-30T10:16:00Z'), + text: 'The next finalized speech turn.', +}]; + +function renderPanel() { + return render( + + + , + ); +} + +describe('TranscriptPanel', () => { + it('shows finalized speech turns only', () => { + renderPanel(); + + expect(screen.getByText('Alice')).toBeInTheDocument(); + expect(screen.getByText('The finalized speech turn.')).toBeInTheDocument(); + expect(screen.getByText('Bob')).toBeInTheDocument(); + expect(screen.getByText('The next finalized speech turn.')).toBeInTheDocument(); + expect(screen.queryByText(/^listening$/i)).not.toBeInTheDocument(); + expect(screen.queryByRole('status')).not.toBeInTheDocument(); + }); +}); diff --git a/client/src/components/TranscriptPanel.tsx b/client/src/components/TranscriptPanel.tsx index b24b6d0..fe2879d 100644 --- a/client/src/components/TranscriptPanel.tsx +++ b/client/src/components/TranscriptPanel.tsx @@ -17,17 +17,10 @@ interface TranscriptEntry { ts: string | number | Date; text: string; } -interface TranscriptInterim { - utteranceId: string; - speaker?: TranscriptSpeaker; - text: string; -} interface TranscriptPanelProps { open: boolean; entries: TranscriptEntry[]; active: boolean; - interims: TranscriptInterim[]; - contributorStatus: string; contributorError?: string; isHost: boolean; canContribute: boolean; @@ -45,8 +38,6 @@ export default function TranscriptPanel({ open, entries, active, - interims, - contributorStatus, contributorError, isHost, canContribute, @@ -58,7 +49,7 @@ export default function TranscriptPanel({ const bottomRef = useRef(null); useEffect(() => { if (open) bottomRef.current?.scrollIntoView({ behavior: 'smooth' }); - }, [open, entries.length, interims.length]); + }, [open, entries.length]); if (!open) return null; @@ -138,7 +129,7 @@ export default function TranscriptPanel({ ) : null} - {!entries.length && !interims.length ? ( + {!entries.length ? ( The conversation will appear here @@ -165,19 +156,6 @@ export default function TranscriptPanel({ ))} - {interims.map((interim) => ( - - - - - - - {interim.speaker?.name || 'Participant'} · {contributorStatus === 'error' ? 'provider issue' : 'listening'} - - {interim.text} - - - ))}
diff --git a/client/src/pages/RoomPage.tsx b/client/src/pages/RoomPage.tsx index ccbe64f..6dbed25 100644 --- a/client/src/pages/RoomPage.tsx +++ b/client/src/pages/RoomPage.tsx @@ -32,7 +32,6 @@ import ChatPanel from '../components/ChatPanel'; import PeoplePanel from '../components/PeoplePanel'; import MediaReconnectingAlert from '../components/MediaReconnectingAlert'; import TranscriptPanel from '../components/TranscriptPanel'; -import LiveCaptions from '../components/LiveCaptions'; import CallNotifications from '../components/CallNotifications'; import ReactionsOverlay from '../components/ReactionsOverlay'; import type { ChatMessage } from '../components/ChatPanel'; @@ -43,7 +42,6 @@ import type { RoomUser, TranscriptContributorState, TranscriptEntry, - TranscriptInterim, TranscriptState, } from '@a-meet/contracts'; import { playSound, isSoundEnabled, toggleSound } from '../services/sounds'; @@ -113,13 +111,10 @@ export default function RoomPage() { }); const [transcriptEntries, setTranscriptEntries] = useState([]); const [transcriptConfigured, setTranscriptConfigured] = useState(null); - const [transcriptInterims, setTranscriptInterims] = useState>({}); const [contributorState, setContributorState] = useState({ status: 'idle', provider: null, error: '' }); const [transcriptConsent, setTranscriptConsent] = useState(hasTranscriptConsent); const [transcriptConsentOpen, setTranscriptConsentOpen] = useState(false); const pendingTranscriptStartRef = useRef(false); - const [latestCaption, setLatestCaption] = useState(null); - const captionTimerRef = useRef | undefined>(undefined); const [unreadCount, setUnreadCount] = useState(0); // Host asked us to unmute — surfaced as a one-tap prompt (never forced). const [unmuteRequestFrom, setUnmuteRequestFrom] = useState(null); @@ -240,9 +235,6 @@ export default function RoomPage() { audioTrack: localAudioOn ? audioTrack : null, onChunk: sendTranscriptAudio, }); - const transcriptInterimList = Object.values(transcriptInterims).sort((a, b) => a.ts - b.ts); - const latestInterim = transcriptInterimList[transcriptInterimList.length - 1] ?? null; - useEffect(() => { if (!shouldContributeTranscript || !pcmCapture.supported) return undefined; let cancelled = false; @@ -356,9 +348,6 @@ export default function RoomPage() { setMessages((prev) => [...prev, { type: 'event', text: `${u.name} left`, ts: Date.now() }]); pushNote({ kind: 'event', variant: 'leave', name: u.name, avatar: u.avatar }); playSound('leave'); - setTranscriptInterims((current) => Object.fromEntries( - Object.entries(current).filter(([, interim]) => interim.speaker?.id !== u.id), - )); }); socket.on('chat-message', (msg) => { setMessages((prev) => [...prev, { type: 'chat', ...msg }]); @@ -393,7 +382,6 @@ export default function RoomPage() { }); socket.on('transcript-state', (state) => { setTranscriptState(state); - setTranscriptInterims({}); if (state.active) { setActivePanel('transcript'); if (!hasTranscriptConsent() @@ -404,17 +392,6 @@ export default function RoomPage() { }); socket.on('transcript-segment', (entry) => { setTranscriptEntries((current) => mergeTranscriptEntries(current, [entry])); - setLatestCaption(entry); - clearTimeout(captionTimerRef.current); - captionTimerRef.current = setTimeout(() => setLatestCaption(null), 6500); - }); - socket.on('transcript-interim', (interim) => { - setTranscriptInterims((current) => { - const next = { ...current }; - if (interim.text) next[interim.utteranceId] = interim; - else delete next[interim.utteranceId]; - return next; - }); }); socket.on('transcript-contributor-state', (state) => { setContributorState({ @@ -443,7 +420,6 @@ export default function RoomPage() { socket.off('transcript-snapshot'); socket.off('transcript-state'); socket.off('transcript-segment'); - socket.off('transcript-interim'); socket.off('transcript-contributor-state'); socket.off('sfu-meeting-ended'); socket.off('sfu-hand-raise-update', onPeerHandRaise); @@ -454,7 +430,6 @@ export default function RoomPage() { // the packet doesn't flush before disconnect, the grace window still covers it. socket.emit('leave-room', roomId); socket.disconnect(); - clearTimeout(captionTimerRef.current); }; // eslint-disable-next-line react-hooks/exhaustive-deps }, [roomId]); @@ -1285,11 +1260,6 @@ export default function RoomPage() { {/* Bottom-left floating emoji stream (M7.5) */} - - {/* Top overlay: meeting info (left) + participants (right) */} socket.emit('transcript-contributor-state', state), - onInterim: ({ utteranceId, text }) => { - io.to(roomId).emit('transcript-interim', { - utteranceId, - speaker: { id: user.id, name: user.name, avatar: user.avatar || '' }, - text, - ts: Date.now(), - }); - }, + // Deepgram still returns interims so it can assemble a complete final turn. + // They are deliberately not sent to meeting participants. + onInterim: () => {}, onFinal: ({ utteranceId, text, audio }) => { const appended = appendTranscriptSegment(roomId, user, { clientSegmentId: `provider:${utteranceId}`, diff --git a/server/test/deepgram-session.test.js b/server/test/deepgram-session.test.js index 33a1f96..10b340f 100644 --- a/server/test/deepgram-session.test.js +++ b/server/test/deepgram-session.test.js @@ -142,7 +142,7 @@ describe('DeepgramMeetingSession provider lifecycle', () => { }); describe('DeepgramMeetingSession turn assembly', () => { - it('broadcasts interim text and finalizes one audio-backed utterance', () => { + it('accumulates interim text and finalizes one audio-backed utterance', () => { const onInterim = vi.fn(); const onFinal = vi.fn(); const session = new DeepgramMeetingSession({ diff --git a/server/test/meeting-transcription.test.js b/server/test/meeting-transcription.test.js index 6b2a875..87c439f 100644 --- a/server/test/meeting-transcription.test.js +++ b/server/test/meeting-transcription.test.js @@ -144,6 +144,21 @@ describe('startContributor', () => { expect(mockSessionInstance.start).toHaveBeenCalledTimes(1); }); + // Contract-level removal makes an interim emit a type error; this fake-io assertion covers the existing server seam. + it('does not broadcast provider interim results to room participants', async () => { + const socketId = uid(); + const { io, emits } = makeIo(); + const socket = makeSocket(socketId); + env.transcription.deepgramApiKey = 'key'; + + await startContributor({ io, socket, roomId: ROOM }); + + const { onInterim } = DeepgramMeetingSession.mock.calls.at(-1)[0]; + onInterim({ utteranceId: 'utterance-1', text: 'still speaking' }); + + expect(emits).not.toContainEqual(expect.objectContaining({ event: 'transcript-interim' })); + }); + it('cleans up the sessions map if session.start() throws', async () => { const socketId = uid(); const { io } = makeIo(); diff --git a/shared/src/events.ts b/shared/src/events.ts index 84e2d95..f12cd74 100644 --- a/shared/src/events.ts +++ b/shared/src/events.ts @@ -28,12 +28,6 @@ export interface TranscriptEntry { provisional: boolean; revisedAt?: number; } -export interface TranscriptInterim { - utteranceId: string; - speaker: { id: string; name: string; avatar: string }; - text: string; - ts: number; -} export interface TranscriptSnapshot extends TranscriptState { entries: TranscriptEntry[]; configured: boolean; @@ -100,7 +94,6 @@ export interface ServerToClientEvents extends WebRtcServerToClientEvents { 'transcript-snapshot': (snapshot: TranscriptSnapshot) => void; 'transcript-state': (state: TranscriptState) => void; 'transcript-segment': (entry: TranscriptEntry) => void; - 'transcript-interim': (interim: TranscriptInterim) => void; 'transcript-contributor-state': (state: TranscriptContributorState) => void; 'sfu-meeting-ended': () => void; 'sfu-force-muted': () => void;