diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/backend/__init__.py @@ -0,0 +1 @@ + diff --git a/backend/llm.py b/backend/llm.py index e306cd7..b726c9e 100644 --- a/backend/llm.py +++ b/backend/llm.py @@ -6,12 +6,16 @@ factory is the only public API main.py should use. """ +import asyncio import logging from ollama import AsyncClient from pydantic import BaseModel -import config as cfg +try: + from . import config as cfg +except ImportError: + import config as cfg # type: ignore logger = logging.getLogger(__name__) @@ -72,14 +76,17 @@ async def correct( self._history = self._history[-cfg.LLM_HISTORY_MAX:] try: - response = await self._client.chat( - model=cfg.LLM_MODEL, - messages=[ - {"role": "system", "content": _SYSTEM_PROMPT}, - *self._history, - ], - format=_Schema.model_json_schema(), - options={"think": False}, + response = await asyncio.wait_for( + self._client.chat( + model=cfg.LLM_MODEL, + messages=[ + {"role": "system", "content": _SYSTEM_PROMPT}, + *self._history, + ], + format=_Schema.model_json_schema(), + options={"think": False}, + ), + timeout=60.0, ) try: @@ -95,12 +102,15 @@ async def correct( text += "." return text + except asyncio.TimeoutError: + logger.warning("LLM timed out after 60 s; falling back to raw transcript") except Exception: logger.exception("LLM correction failed; falling back to raw transcript") - fallback = transcript.strip().capitalize() - if fallback and fallback[-1] not in ".?!": - fallback += "." - return fallback + + fallback = (transcript or "").strip().capitalize() + if fallback and fallback[-1] not in ".?!": + fallback += "." + return fallback def make_corrector() -> LLMCorrector: diff --git a/backend/main.py b/backend/main.py index 4fd1e12..e6e2717 100644 --- a/backend/main.py +++ b/backend/main.py @@ -42,21 +42,24 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse -import config as cfg -import vsr -import llm - -# Gesture recogniser — lazy import so the VSR backend works even if -# the gesture module's mediapipe install is separate. try: - import sys as _sys - _sys.path.insert(0, str(__import__("pathlib").Path(__file__).parent.parent)) + from . import config as cfg + from . import vsr + from . import llm +except ImportError: + import config as cfg # type: ignore + import vsr # type: ignore + import llm # type: ignore + +# Gesture recogniser — lazy import (mediapipe may be in a separate env) +import sys as _sys, pathlib as _pl +_sys.path.insert(0, str(_pl.Path(__file__).parent.parent)) +try: from gesture.recognizer import GestureRecognizer as _GestureRecognizer _gesture_available = True except Exception as _ge: _gesture_available = False - logger_tmp = __import__("logging").getLogger("silent-speech") - logger_tmp.warning("Gesture module unavailable: %s", _ge) + __import__("logging").getLogger("silent-speech").warning("Gesture module unavailable: %s", _ge) # ── logging ─────────────────────────────────────────────────────────────────── logging.basicConfig( diff --git a/backend/vsr.py b/backend/vsr.py index 9088a03..6283810 100644 --- a/backend/vsr.py +++ b/backend/vsr.py @@ -22,7 +22,10 @@ import numpy as np import torch -import config as cfg +try: + from . import config as cfg +except ImportError: + import config as cfg # type: ignore logger = logging.getLogger(__name__) diff --git a/frontend/src/app/dashboard/page.tsx b/frontend/src/app/dashboard/page.tsx new file mode 100644 index 0000000..7f9c66f --- /dev/null +++ b/frontend/src/app/dashboard/page.tsx @@ -0,0 +1,37 @@ +'use client'; +import { useSession } from 'next-auth/react'; +import { useRouter } from 'next/navigation'; +import { useEffect } from 'react'; +import Dashboard from '@/components/dashboard/Dashboard'; + +export default function DashboardPage() { + const { data: session, status } = useSession(); + const router = useRouter(); + + useEffect(() => { + if (status === 'unauthenticated') router.replace('/login'); + }, [status, router]); + + if (status === 'loading' || !session) { + return ( +
+ LOADING… +
+ ); + } + + const u = session.user; + return ( + + ); +} diff --git a/frontend/src/components/AuthFlow.tsx b/frontend/src/components/AuthFlow.tsx index b95ae20..ee95fcb 100644 --- a/frontend/src/components/AuthFlow.tsx +++ b/frontend/src/components/AuthFlow.tsx @@ -16,12 +16,12 @@ export default function AuthFlow({ initialMode = 'login' }: { initialMode?: 'log useEffect(() => { if (session) { - router.push('/'); + router.push('/dashboard'); } }, [session, router]); const handleGoogleSignIn = () => { - signIn('google', { callbackUrl: '/' }); + signIn('google', { callbackUrl: '/dashboard' }); }; const isFlipped = mode === 'signup'; diff --git a/frontend/src/components/GestureCapture.tsx b/frontend/src/components/GestureCapture.tsx index 7a1ecb2..b314170 100644 --- a/frontend/src/components/GestureCapture.tsx +++ b/frontend/src/components/GestureCapture.tsx @@ -315,7 +315,7 @@ export default function GestureCapture() { {/* Back link */} - { (e.currentTarget as HTMLAnchorElement).style.color = 'var(--fg-0)'; (e.currentTarget as HTMLAnchorElement).style.borderColor = 'var(--fg-2)'; }} onMouseLeave={e => { (e.currentTarget as HTMLAnchorElement).style.color = 'var(--fg-3)'; (e.currentTarget as HTMLAnchorElement).style.borderColor = 'var(--fg-4)'; }} > - ← Back to home + ← Back to dashboard + + ); +} diff --git a/frontend/src/components/dashboard/ResultPopup.tsx b/frontend/src/components/dashboard/ResultPopup.tsx new file mode 100644 index 0000000..dcf9969 --- /dev/null +++ b/frontend/src/components/dashboard/ResultPopup.tsx @@ -0,0 +1,262 @@ +'use client'; +import { useState, useCallback, useEffect } from 'react'; +import type { ResultData } from './types'; + +interface Props { + result: ResultData | null; + show: boolean; + onClose: () => void; + onTryAgain: () => void; +} + +function SpeakerIcon() { + return ( + + + + + ); +} +function StopIcon() { + return ( + + + + ); +} + +export default function ResultPopup({ result, show, onClose, onTryAgain }: Props) { + const [showCandidates, setShowCandidates] = useState(false); + const [speaking, setSpeaking] = useState(false); + const [mounted, setMounted] = useState(false); + + useEffect(() => { + if (show) { + setTimeout(() => setMounted(true), 10); + setShowCandidates(false); + } else { + setMounted(false); + if (speaking) { window.speechSynthesis?.cancel(); setSpeaking(false); } + } + }, [show, speaking]); + + const speak = useCallback(() => { + if (!result || !('speechSynthesis' in window)) return; + if (speaking) { window.speechSynthesis.cancel(); setSpeaking(false); return; } + const txt = result.corrected || result.raw || ''; + if (!txt) return; + const utt = new SpeechSynthesisUtterance(txt); + utt.onstart = () => setSpeaking(true); + utt.onend = () => setSpeaking(false); + utt.onerror = () => setSpeaking(false); + window.speechSynthesis.speak(utt); + }, [result, speaking]); + + if (!show && !mounted) return null; + + const displayText = result ? (result.corrected || result.raw || '(no output)') : ''; + + return ( +
{ if (e.target === e.currentTarget) onClose(); }} + style={{ + position: 'fixed', inset: 0, zIndex: 200, + display: 'flex', flexDirection: 'column', justifyContent: 'flex-end', + background: `rgba(6,6,8,${mounted ? '0.75' : '0'})`, + backdropFilter: mounted ? 'blur(8px)' : 'none', + transition: 'background 300ms ease, backdrop-filter 300ms ease', + }} + > + {/* Panel */} +
+ {/* Corner marks */} + {(['tl','tr'] as const).map(pos => ( + + ))} + + {/* Header */} +
+
+ + + TRANSCRIPT READY · {result?.input === 'webcam' ? 'LIVE SESSION' : 'VIDEO FILE'} + +
+ +
+ + {/* Main transcript + speaker */} +
+
+ {displayText} +
+ +
+ + {/* Raw VSR (if different) */} + {result?.raw && result.corrected && result.raw !== result.corrected && ( +
+
+ Raw VSR output +
+
+ {result.raw} +
+
+ )} + + {/* N-best */} + {result && result.candidates.length > 0 && ( +
+ + {showCandidates && ( +
+ {result.candidates.map((c, i) => ( +
+ + {String(i + 1).padStart(2, '0')} + + + {c.text} + + + {c.score.toFixed(2)} + +
+ ))} +
+ )} +
+ )} + + {/* Divider */} +
+ + {/* Actions */} +
+ + +
+
+
+ ); +} diff --git a/frontend/src/components/dashboard/UploadArea.tsx b/frontend/src/components/dashboard/UploadArea.tsx new file mode 100644 index 0000000..3ef555a --- /dev/null +++ b/frontend/src/components/dashboard/UploadArea.tsx @@ -0,0 +1,192 @@ +'use client'; +import { useRef, useState, useCallback } from 'react'; + +interface Props { + onFile: (file: File) => void; + onBack: () => void; +} + +function Corner({ v, h, size = 12 }: { v: 'top' | 'bottom'; h: 'left' | 'right'; size?: number }) { + return ( + + ); +} + +function formatSize(bytes: number) { + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)} KB`; + return `${(bytes / 1024 / 1024).toFixed(1)} MB`; +} + +export default function UploadArea({ onFile, onBack }: Props) { + const inputRef = useRef(null); + const [dragging, setDragging] = useState(false); + const [selected, setSelected] = useState(null); + + const pick = useCallback((file: File) => { + if (file.type.startsWith('video/') || file.name.match(/\.(mp4|webm|avi|mov|mkv)$/i)) { + setSelected(file); + } + }, []); + + const handleDrop = useCallback((e: React.DragEvent) => { + e.preventDefault(); + setDragging(false); + const file = e.dataTransfer.files[0]; + if (file) pick(file); + }, [pick]); + + const handleInput = useCallback((e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (file) pick(file); + }, [pick]); + + return ( +
+ {/* Eyebrow */} +
+ + VIDEO FILE UPLOAD + +
+ + {/* Drop zone */} +
!selected && inputRef.current?.click()} + onDragOver={e => { e.preventDefault(); setDragging(true); }} + onDragLeave={() => setDragging(false)} + onDrop={handleDrop} + style={{ + position: 'relative', + width: '100%', maxWidth: 520, minHeight: 220, + border: `2px dashed ${dragging ? 'var(--accent)' : selected ? 'rgba(184,216,248,0.35)' : 'var(--fg-4)'}`, + borderRadius: 12, + background: dragging + ? 'rgba(184,216,248,0.05)' + : selected + ? 'rgba(184,216,248,0.03)' + : 'linear-gradient(180deg, var(--bg-2), var(--bg-1))', + display: 'flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', + gap: 16, padding: 36, + cursor: selected ? 'default' : 'pointer', + transition: 'border-color 200ms ease, background 200ms ease', + }} + > + + + + + + {selected ? ( + <> +
+ + + + +
+
+
+ {selected.name} +
+
+ {formatSize(selected.size)} · {selected.type || 'video'} +
+
+ + + ) : ( + <> +
+ + + + +
+
+
+ {dragging ? 'Drop to upload' : 'Drop your video here'} +
+
+ MP4 · WEBM · AVI · MOV · MKV +
+
+
+ — or click to browse — +
+ + )} +
+ + + + {/* Controls */} +
+ + + {selected && ( + + )} +
+
+ ); +} diff --git a/frontend/src/components/dashboard/WebcamCapture.tsx b/frontend/src/components/dashboard/WebcamCapture.tsx new file mode 100644 index 0000000..d5996cc --- /dev/null +++ b/frontend/src/components/dashboard/WebcamCapture.tsx @@ -0,0 +1,245 @@ +'use client'; +import { useRef, useEffect, useState, useCallback } from 'react'; + +interface Props { + wsReady: boolean; + onFrame: (buf: ArrayBuffer) => void; + onStartRecording: () => void; + onStopRecording: () => void; + onBack: () => void; +} + +function Corner({ v, h }: { v: 'top' | 'bottom'; h: 'left' | 'right' }) { + return ( + + ); +} + +export default function WebcamCapture({ wsReady, onFrame, onStartRecording, onStopRecording, onBack }: Props) { + const videoRef = useRef(null); + const canvasRef = useRef(null); + const intervalRef = useRef | null>(null); + const [hasStream, setHasStream] = useState(false); + const [recording, setRecording] = useState(false); + const [frameCount, setFrameCount] = useState(0); + const [permErr, setPermErr] = useState(null); + + useEffect(() => { + let stream: MediaStream; + navigator.mediaDevices + .getUserMedia({ video: { width: 640, height: 480, frameRate: 25 }, audio: false }) + .then(s => { + stream = s; + if (videoRef.current) { + videoRef.current.srcObject = s; + videoRef.current.play().catch(() => {}); + } + setHasStream(true); + }) + .catch(err => setPermErr(err.message ?? 'Camera permission denied')); + + return () => { stream?.getTracks().forEach(t => t.stop()); }; + }, []); + + const startRecording = useCallback(() => { + if (!hasStream || !wsReady) return; + setRecording(true); + setFrameCount(0); + onStartRecording(); + + intervalRef.current = setInterval(() => { + const video = videoRef.current; + const canvas = canvasRef.current; + if (!video || !canvas) return; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + ctx.drawImage(video, 0, 0, 640, 480); + canvas.toBlob(blob => { + if (!blob) return; + blob.arrayBuffer().then(buf => { + onFrame(buf); + setFrameCount(c => c + 1); + }); + }, 'image/jpeg', 0.85); + }, 1000 / 25); + }, [hasStream, wsReady, onFrame, onStartRecording]); + + const stopRecording = useCallback(() => { + if (intervalRef.current) { clearInterval(intervalRef.current); intervalRef.current = null; } + setRecording(false); + onStopRecording(); + }, [onStopRecording]); + + useEffect(() => () => { if (intervalRef.current) clearInterval(intervalRef.current); }, []); + + const canRecord = hasStream && wsReady && !recording; + + return ( +
+ {/* Eyebrow */} +
+ + LIVE WEBCAM INPUT + +
+ + {/* Video frame */} +
+ + + + + + {/* Status badge */} +
+ + + {recording ? `● REC · ${frameCount} frames` : wsReady ? 'READY' : 'CONNECTING…'} + +
+ + {permErr ? ( +
+ {permErr} +
+ ) : ( +
+ ); +} diff --git a/frontend/src/components/dashboard/types.ts b/frontend/src/components/dashboard/types.ts new file mode 100644 index 0000000..0975632 --- /dev/null +++ b/frontend/src/components/dashboard/types.ts @@ -0,0 +1,15 @@ +export interface HistoryEntry { + id: string; + timestamp: number; + input: 'webcam' | 'upload'; + raw: string; + corrected: string; + candidates: Array<{ text: string; score: number }>; +} + +export interface ResultData { + raw: string; + corrected: string; + candidates: Array<{ text: string; score: number }>; + input: 'webcam' | 'upload'; +}