From f8e3c8d1a10bc66efa9cf34f3405da5235144409 Mon Sep 17 00:00:00 2001 From: Sanskaar Date: Thu, 7 May 2026 09:14:08 +0530 Subject: [PATCH 1/3] feat(gesture): move gesture button into auth-protected dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add /dashboard page with feature cards for Lip Reading and Gesture Recognition — only accessible when logged in (redirects to /login if not) - Login/signup now redirects to /dashboard instead of home (both useEffect redirect and Google callbackUrl) - Remove gesture button from Hero landing page — it belongs behind auth Co-Authored-By: Claude Sonnet 4.6 --- frontend/src/app/dashboard/page.tsx | 248 +++++++++++++++++++++++++++ frontend/src/components/AuthFlow.tsx | 4 +- frontend/src/components/Hero.tsx | 6 - 3 files changed, 250 insertions(+), 8 deletions(-) create mode 100644 frontend/src/app/dashboard/page.tsx diff --git a/frontend/src/app/dashboard/page.tsx b/frontend/src/app/dashboard/page.tsx new file mode 100644 index 0000000..fc6ee16 --- /dev/null +++ b/frontend/src/app/dashboard/page.tsx @@ -0,0 +1,248 @@ +'use client'; +import { useSession, signOut } from 'next-auth/react'; +import { useRouter } from 'next/navigation'; +import { useEffect } from 'react'; +import Link from 'next/link'; + +export default function DashboardPage() { + const { data: session, status } = useSession(); + const router = useRouter(); + + useEffect(() => { + if (status === 'unauthenticated') router.replace('/login'); + }, [status, router]); + + if (status === 'loading' || !session) { + return ( +
+ LOADING… +
+ ); + } + + const user = session.user; + + return ( +
+ {/* Nav */} + + + {/* Main */} +
+ {/* Header */} +
+
+ + DASHBOARD +
+

+ Welcome back{user?.name ? `, ${user.name.split(' ')[0]}` : ''}. +

+

+ Choose a mode to get started. +

+
+ + {/* Feature cards */} +
+ {/* Lip Reading */} + } + accent="var(--accent)" + /> + + {/* Gesture Recognition */} + } + accent="#a0e0b0" + /> +
+ + {/* Footer note */} +
+ NO AUDIO RECORDED + · + ON-DEVICE INFERENCE + · + + ← BACK TO SITE + +
+
+
+ ); +} + + +function FeatureCard({ + href, tag, title, description, icon, accent, +}: { + href: string; tag: string; title: string; + description: string; icon: React.ReactNode; accent: string; +}) { + return ( + +
{ + const el = e.currentTarget as HTMLDivElement; + el.style.borderColor = accent + '55'; + el.style.background = 'var(--bg-2)'; + el.style.transform = 'translateY(-2px)'; + el.style.boxShadow = `0 8px 32px ${accent}18`; + }} + onMouseLeave={e => { + const el = e.currentTarget as HTMLDivElement; + el.style.borderColor = 'var(--fg-4)'; + el.style.background = 'var(--bg-1)'; + el.style.transform = ''; + el.style.boxShadow = ''; + }} + > + {/* Accent corner line */} +
+ + {/* Icon */} +
+ {icon} +
+ + {/* Tag */} +
+ {tag} +
+ + {/* Title */} +

+ {title} +

+ + {/* Description */} +

+ {description} +

+ + {/* CTA arrow */} +
+ OPEN + + + +
+
+ + ); +} + +function LipIcon() { + return ( + + + + + + + ); +} + +function HandIcon() { + return ( + + + + ); +} diff --git a/frontend/src/components/AuthFlow.tsx b/frontend/src/components/AuthFlow.tsx index b95ae20..ee95fcb 100644 --- a/frontend/src/components/AuthFlow.tsx +++ b/frontend/src/components/AuthFlow.tsx @@ -16,12 +16,12 @@ export default function AuthFlow({ initialMode = 'login' }: { initialMode?: 'log useEffect(() => { if (session) { - router.push('/'); + router.push('/dashboard'); } }, [session, router]); const handleGoogleSignIn = () => { - signIn('google', { callbackUrl: '/' }); + signIn('google', { callbackUrl: '/dashboard' }); }; const isFlipped = mode === 'signup'; diff --git a/frontend/src/components/Hero.tsx b/frontend/src/components/Hero.tsx index f5eaa0c..d009eee 100644 --- a/frontend/src/components/Hero.tsx +++ b/frontend/src/components/Hero.tsx @@ -265,12 +265,6 @@ export default function Hero() { - - - - - Gesture recognition - Read the vision From 86020c724fef41c407df3f70990de21ba906e09a Mon Sep 17 00:00:00 2001 From: Sanskaar Date: Thu, 7 May 2026 09:20:51 +0530 Subject: [PATCH 2/3] feat(gesture): restore original dashboard layout + add gesture card MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Restore full dashboard: sidebar with profile/history, top bar, 3-card idle screen (Live Webcam · Upload Video · Gesture Detection) - Gesture Detection card navigates to /gesture page - /dashboard page.tsx passes session user to Dashboard component - GestureCapture: back link goes to /dashboard instead of home Co-Authored-By: Claude Sonnet 4.6 --- frontend/src/app/dashboard/page.tsx | 231 +--------- frontend/src/components/GestureCapture.tsx | 4 +- .../src/components/dashboard/Dashboard.tsx | 416 ++++++++++++++++++ .../src/components/dashboard/HistoryPanel.tsx | 190 ++++++++ .../components/dashboard/LoadingScreen.tsx | 116 +++++ .../src/components/dashboard/ResultPopup.tsx | 262 +++++++++++ .../src/components/dashboard/UploadArea.tsx | 192 ++++++++ .../components/dashboard/WebcamCapture.tsx | 245 +++++++++++ frontend/src/components/dashboard/types.ts | 15 + 9 files changed, 1448 insertions(+), 223 deletions(-) create mode 100644 frontend/src/components/dashboard/Dashboard.tsx create mode 100644 frontend/src/components/dashboard/HistoryPanel.tsx create mode 100644 frontend/src/components/dashboard/LoadingScreen.tsx create mode 100644 frontend/src/components/dashboard/ResultPopup.tsx create mode 100644 frontend/src/components/dashboard/UploadArea.tsx create mode 100644 frontend/src/components/dashboard/WebcamCapture.tsx create mode 100644 frontend/src/components/dashboard/types.ts diff --git a/frontend/src/app/dashboard/page.tsx b/frontend/src/app/dashboard/page.tsx index fc6ee16..7f9c66f 100644 --- a/frontend/src/app/dashboard/page.tsx +++ b/frontend/src/app/dashboard/page.tsx @@ -1,8 +1,8 @@ 'use client'; -import { useSession, signOut } from 'next-auth/react'; +import { useSession } from 'next-auth/react'; import { useRouter } from 'next/navigation'; import { useEffect } from 'react'; -import Link from 'next/link'; +import Dashboard from '@/components/dashboard/Dashboard'; export default function DashboardPage() { const { data: session, status } = useSession(); @@ -24,225 +24,14 @@ export default function DashboardPage() { ); } - const user = session.user; - - return ( -
- {/* Nav */} - - - {/* Main */} -
- {/* Header */} -
-
- - DASHBOARD -
-

- Welcome back{user?.name ? `, ${user.name.split(' ')[0]}` : ''}. -

-

- Choose a mode to get started. -

-
- - {/* Feature cards */} -
- {/* Lip Reading */} - } - accent="var(--accent)" - /> - - {/* Gesture Recognition */} - } - accent="#a0e0b0" - /> -
- - {/* Footer note */} -
- NO AUDIO RECORDED - · - ON-DEVICE INFERENCE - · - - ← BACK TO SITE - -
-
-
- ); -} - - -function FeatureCard({ - href, tag, title, description, icon, accent, -}: { - href: string; tag: string; title: string; - description: string; icon: React.ReactNode; accent: string; -}) { - return ( - -
{ - const el = e.currentTarget as HTMLDivElement; - el.style.borderColor = accent + '55'; - el.style.background = 'var(--bg-2)'; - el.style.transform = 'translateY(-2px)'; - el.style.boxShadow = `0 8px 32px ${accent}18`; - }} - onMouseLeave={e => { - const el = e.currentTarget as HTMLDivElement; - el.style.borderColor = 'var(--fg-4)'; - el.style.background = 'var(--bg-1)'; - el.style.transform = ''; - el.style.boxShadow = ''; - }} - > - {/* Accent corner line */} -
- - {/* Icon */} -
- {icon} -
- - {/* Tag */} -
- {tag} -
- - {/* Title */} -

- {title} -

- - {/* Description */} -

- {description} -

- - {/* CTA arrow */} -
- OPEN - - - -
-
- - ); -} - -function LipIcon() { - return ( - - - - - - - ); -} - -function HandIcon() { + const u = session.user; return ( - - - + ); } diff --git a/frontend/src/components/GestureCapture.tsx b/frontend/src/components/GestureCapture.tsx index 7a1ecb2..b314170 100644 --- a/frontend/src/components/GestureCapture.tsx +++ b/frontend/src/components/GestureCapture.tsx @@ -315,7 +315,7 @@ export default function GestureCapture() {
{/* Back link */} - { (e.currentTarget as HTMLAnchorElement).style.color = 'var(--fg-0)'; (e.currentTarget as HTMLAnchorElement).style.borderColor = 'var(--fg-2)'; }} onMouseLeave={e => { (e.currentTarget as HTMLAnchorElement).style.color = 'var(--fg-3)'; (e.currentTarget as HTMLAnchorElement).style.borderColor = 'var(--fg-4)'; }} > - ← Back to home + ← Back to dashboard +
+ ); +} diff --git a/frontend/src/components/dashboard/ResultPopup.tsx b/frontend/src/components/dashboard/ResultPopup.tsx new file mode 100644 index 0000000..dcf9969 --- /dev/null +++ b/frontend/src/components/dashboard/ResultPopup.tsx @@ -0,0 +1,262 @@ +'use client'; +import { useState, useCallback, useEffect } from 'react'; +import type { ResultData } from './types'; + +interface Props { + result: ResultData | null; + show: boolean; + onClose: () => void; + onTryAgain: () => void; +} + +function SpeakerIcon() { + return ( + + + + + ); +} +function StopIcon() { + return ( + + + + ); +} + +export default function ResultPopup({ result, show, onClose, onTryAgain }: Props) { + const [showCandidates, setShowCandidates] = useState(false); + const [speaking, setSpeaking] = useState(false); + const [mounted, setMounted] = useState(false); + + useEffect(() => { + if (show) { + setTimeout(() => setMounted(true), 10); + setShowCandidates(false); + } else { + setMounted(false); + if (speaking) { window.speechSynthesis?.cancel(); setSpeaking(false); } + } + }, [show, speaking]); + + const speak = useCallback(() => { + if (!result || !('speechSynthesis' in window)) return; + if (speaking) { window.speechSynthesis.cancel(); setSpeaking(false); return; } + const txt = result.corrected || result.raw || ''; + if (!txt) return; + const utt = new SpeechSynthesisUtterance(txt); + utt.onstart = () => setSpeaking(true); + utt.onend = () => setSpeaking(false); + utt.onerror = () => setSpeaking(false); + window.speechSynthesis.speak(utt); + }, [result, speaking]); + + if (!show && !mounted) return null; + + const displayText = result ? (result.corrected || result.raw || '(no output)') : ''; + + return ( +
{ if (e.target === e.currentTarget) onClose(); }} + style={{ + position: 'fixed', inset: 0, zIndex: 200, + display: 'flex', flexDirection: 'column', justifyContent: 'flex-end', + background: `rgba(6,6,8,${mounted ? '0.75' : '0'})`, + backdropFilter: mounted ? 'blur(8px)' : 'none', + transition: 'background 300ms ease, backdrop-filter 300ms ease', + }} + > + {/* Panel */} +
+ {/* Corner marks */} + {(['tl','tr'] as const).map(pos => ( + + ))} + + {/* Header */} +
+
+ + + TRANSCRIPT READY · {result?.input === 'webcam' ? 'LIVE SESSION' : 'VIDEO FILE'} + +
+ +
+ + {/* Main transcript + speaker */} +
+
+ {displayText} +
+ +
+ + {/* Raw VSR (if different) */} + {result?.raw && result.corrected && result.raw !== result.corrected && ( +
+
+ Raw VSR output +
+
+ {result.raw} +
+
+ )} + + {/* N-best */} + {result && result.candidates.length > 0 && ( +
+ + {showCandidates && ( +
+ {result.candidates.map((c, i) => ( +
+ + {String(i + 1).padStart(2, '0')} + + + {c.text} + + + {c.score.toFixed(2)} + +
+ ))} +
+ )} +
+ )} + + {/* Divider */} +
+ + {/* Actions */} +
+ + +
+
+
+ ); +} diff --git a/frontend/src/components/dashboard/UploadArea.tsx b/frontend/src/components/dashboard/UploadArea.tsx new file mode 100644 index 0000000..3ef555a --- /dev/null +++ b/frontend/src/components/dashboard/UploadArea.tsx @@ -0,0 +1,192 @@ +'use client'; +import { useRef, useState, useCallback } from 'react'; + +interface Props { + onFile: (file: File) => void; + onBack: () => void; +} + +function Corner({ v, h, size = 12 }: { v: 'top' | 'bottom'; h: 'left' | 'right'; size?: number }) { + return ( + + ); +} + +function formatSize(bytes: number) { + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)} KB`; + return `${(bytes / 1024 / 1024).toFixed(1)} MB`; +} + +export default function UploadArea({ onFile, onBack }: Props) { + const inputRef = useRef(null); + const [dragging, setDragging] = useState(false); + const [selected, setSelected] = useState(null); + + const pick = useCallback((file: File) => { + if (file.type.startsWith('video/') || file.name.match(/\.(mp4|webm|avi|mov|mkv)$/i)) { + setSelected(file); + } + }, []); + + const handleDrop = useCallback((e: React.DragEvent) => { + e.preventDefault(); + setDragging(false); + const file = e.dataTransfer.files[0]; + if (file) pick(file); + }, [pick]); + + const handleInput = useCallback((e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (file) pick(file); + }, [pick]); + + return ( +
+ {/* Eyebrow */} +
+ + VIDEO FILE UPLOAD + +
+ + {/* Drop zone */} +
!selected && inputRef.current?.click()} + onDragOver={e => { e.preventDefault(); setDragging(true); }} + onDragLeave={() => setDragging(false)} + onDrop={handleDrop} + style={{ + position: 'relative', + width: '100%', maxWidth: 520, minHeight: 220, + border: `2px dashed ${dragging ? 'var(--accent)' : selected ? 'rgba(184,216,248,0.35)' : 'var(--fg-4)'}`, + borderRadius: 12, + background: dragging + ? 'rgba(184,216,248,0.05)' + : selected + ? 'rgba(184,216,248,0.03)' + : 'linear-gradient(180deg, var(--bg-2), var(--bg-1))', + display: 'flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', + gap: 16, padding: 36, + cursor: selected ? 'default' : 'pointer', + transition: 'border-color 200ms ease, background 200ms ease', + }} + > + + + + + + {selected ? ( + <> +
+ + + + +
+
+
+ {selected.name} +
+
+ {formatSize(selected.size)} · {selected.type || 'video'} +
+
+ + + ) : ( + <> +
+ + + + +
+
+
+ {dragging ? 'Drop to upload' : 'Drop your video here'} +
+
+ MP4 · WEBM · AVI · MOV · MKV +
+
+
+ — or click to browse — +
+ + )} +
+ + + + {/* Controls */} +
+ + + {selected && ( + + )} +
+
+ ); +} diff --git a/frontend/src/components/dashboard/WebcamCapture.tsx b/frontend/src/components/dashboard/WebcamCapture.tsx new file mode 100644 index 0000000..d5996cc --- /dev/null +++ b/frontend/src/components/dashboard/WebcamCapture.tsx @@ -0,0 +1,245 @@ +'use client'; +import { useRef, useEffect, useState, useCallback } from 'react'; + +interface Props { + wsReady: boolean; + onFrame: (buf: ArrayBuffer) => void; + onStartRecording: () => void; + onStopRecording: () => void; + onBack: () => void; +} + +function Corner({ v, h }: { v: 'top' | 'bottom'; h: 'left' | 'right' }) { + return ( + + ); +} + +export default function WebcamCapture({ wsReady, onFrame, onStartRecording, onStopRecording, onBack }: Props) { + const videoRef = useRef(null); + const canvasRef = useRef(null); + const intervalRef = useRef | null>(null); + const [hasStream, setHasStream] = useState(false); + const [recording, setRecording] = useState(false); + const [frameCount, setFrameCount] = useState(0); + const [permErr, setPermErr] = useState(null); + + useEffect(() => { + let stream: MediaStream; + navigator.mediaDevices + .getUserMedia({ video: { width: 640, height: 480, frameRate: 25 }, audio: false }) + .then(s => { + stream = s; + if (videoRef.current) { + videoRef.current.srcObject = s; + videoRef.current.play().catch(() => {}); + } + setHasStream(true); + }) + .catch(err => setPermErr(err.message ?? 'Camera permission denied')); + + return () => { stream?.getTracks().forEach(t => t.stop()); }; + }, []); + + const startRecording = useCallback(() => { + if (!hasStream || !wsReady) return; + setRecording(true); + setFrameCount(0); + onStartRecording(); + + intervalRef.current = setInterval(() => { + const video = videoRef.current; + const canvas = canvasRef.current; + if (!video || !canvas) return; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + ctx.drawImage(video, 0, 0, 640, 480); + canvas.toBlob(blob => { + if (!blob) return; + blob.arrayBuffer().then(buf => { + onFrame(buf); + setFrameCount(c => c + 1); + }); + }, 'image/jpeg', 0.85); + }, 1000 / 25); + }, [hasStream, wsReady, onFrame, onStartRecording]); + + const stopRecording = useCallback(() => { + if (intervalRef.current) { clearInterval(intervalRef.current); intervalRef.current = null; } + setRecording(false); + onStopRecording(); + }, [onStopRecording]); + + useEffect(() => () => { if (intervalRef.current) clearInterval(intervalRef.current); }, []); + + const canRecord = hasStream && wsReady && !recording; + + return ( +
+ {/* Eyebrow */} +
+ + LIVE WEBCAM INPUT + +
+ + {/* Video frame */} +
+ + + + + + {/* Status badge */} +
+ + + {recording ? `● REC · ${frameCount} frames` : wsReady ? 'READY' : 'CONNECTING…'} + +
+ + {permErr ? ( +
+ {permErr} +
+ ) : ( +
+ ); +} diff --git a/frontend/src/components/dashboard/types.ts b/frontend/src/components/dashboard/types.ts new file mode 100644 index 0000000..0975632 --- /dev/null +++ b/frontend/src/components/dashboard/types.ts @@ -0,0 +1,15 @@ +export interface HistoryEntry { + id: string; + timestamp: number; + input: 'webcam' | 'upload'; + raw: string; + corrected: string; + candidates: Array<{ text: string; score: number }>; +} + +export interface ResultData { + raw: string; + corrected: string; + candidates: Array<{ text: string; score: number }>; + input: 'webcam' | 'upload'; +} From 4e6511b457a10a3f3579a2a1ec147661b133548a Mon Sep 17 00:00:00 2001 From: Sanskaar Date: Thu, 7 May 2026 09:30:33 +0530 Subject: [PATCH 3/3] fix(backend): LLM timeout + relative imports so result reaches frontend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - llm.py: wrap chat() in asyncio.wait_for(timeout=60) — without this the LLM hangs forever and the result message is never sent to the browser - llm.py: move fallback outside both except blocks so TimeoutError doesn't silently return None - llm.py, vsr.py, main.py: try relative import first, fall back to bare — works both as `uvicorn backend.main:app` (package) and direct `python main.py` - backend/__init__.py: ensure package marker exists Co-Authored-By: Claude Sonnet 4.6 --- backend/__init__.py | 1 + backend/llm.py | 36 +++++++++++++++++++++++------------- backend/main.py | 23 +++++++++++++---------- backend/vsr.py | 5 ++++- 4 files changed, 41 insertions(+), 24 deletions(-) create mode 100644 backend/__init__.py diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/backend/__init__.py @@ -0,0 +1 @@ + diff --git a/backend/llm.py b/backend/llm.py index e306cd7..b726c9e 100644 --- a/backend/llm.py +++ b/backend/llm.py @@ -6,12 +6,16 @@ factory is the only public API main.py should use. """ +import asyncio import logging from ollama import AsyncClient from pydantic import BaseModel -import config as cfg +try: + from . import config as cfg +except ImportError: + import config as cfg # type: ignore logger = logging.getLogger(__name__) @@ -72,14 +76,17 @@ async def correct( self._history = self._history[-cfg.LLM_HISTORY_MAX:] try: - response = await self._client.chat( - model=cfg.LLM_MODEL, - messages=[ - {"role": "system", "content": _SYSTEM_PROMPT}, - *self._history, - ], - format=_Schema.model_json_schema(), - options={"think": False}, + response = await asyncio.wait_for( + self._client.chat( + model=cfg.LLM_MODEL, + messages=[ + {"role": "system", "content": _SYSTEM_PROMPT}, + *self._history, + ], + format=_Schema.model_json_schema(), + options={"think": False}, + ), + timeout=60.0, ) try: @@ -95,12 +102,15 @@ async def correct( text += "." return text + except asyncio.TimeoutError: + logger.warning("LLM timed out after 60 s; falling back to raw transcript") except Exception: logger.exception("LLM correction failed; falling back to raw transcript") - fallback = transcript.strip().capitalize() - if fallback and fallback[-1] not in ".?!": - fallback += "." - return fallback + + fallback = (transcript or "").strip().capitalize() + if fallback and fallback[-1] not in ".?!": + fallback += "." + return fallback def make_corrector() -> LLMCorrector: diff --git a/backend/main.py b/backend/main.py index 4fd1e12..e6e2717 100644 --- a/backend/main.py +++ b/backend/main.py @@ -42,21 +42,24 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse -import config as cfg -import vsr -import llm - -# Gesture recogniser — lazy import so the VSR backend works even if -# the gesture module's mediapipe install is separate. try: - import sys as _sys - _sys.path.insert(0, str(__import__("pathlib").Path(__file__).parent.parent)) + from . import config as cfg + from . import vsr + from . import llm +except ImportError: + import config as cfg # type: ignore + import vsr # type: ignore + import llm # type: ignore + +# Gesture recogniser — lazy import (mediapipe may be in a separate env) +import sys as _sys, pathlib as _pl +_sys.path.insert(0, str(_pl.Path(__file__).parent.parent)) +try: from gesture.recognizer import GestureRecognizer as _GestureRecognizer _gesture_available = True except Exception as _ge: _gesture_available = False - logger_tmp = __import__("logging").getLogger("silent-speech") - logger_tmp.warning("Gesture module unavailable: %s", _ge) + __import__("logging").getLogger("silent-speech").warning("Gesture module unavailable: %s", _ge) # ── logging ─────────────────────────────────────────────────────────────────── logging.basicConfig( diff --git a/backend/vsr.py b/backend/vsr.py index 9088a03..6283810 100644 --- a/backend/vsr.py +++ b/backend/vsr.py @@ -22,7 +22,10 @@ import numpy as np import torch -import config as cfg +try: + from . import config as cfg +except ImportError: + import config as cfg # type: ignore logger = logging.getLogger(__name__)